Debian Codegraph @

recipes/tls_mag.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 import json
4 import re
5
6 from calibre.ebooks.BeautifulSoup import BeautifulSoup
7 from calibre.web.feeds.news import BasicNewsRecipe
8
9
10 def re_html(y):
11 if y:
12 soup = BeautifulSoup(y.rstrip())
13 return soup.text
14 return ''
15
16
17 class TLS(BasicNewsRecipe):
18 title = 'Times Literary Supplement'
19 __author__ = 'unkn0wn'
20 description = (
21 'TLS, world’s leading journal for literature and ideas. Every week, we publish book reviews, book extracts, '
22 'essays and poems from leading writers from around the world. We cover far more than just literature, featuring '
23 'major articles on subjects from anthropology to zoology, philosophy to politics, comedy to psychology. Each week, '
24 'we also review the latest in fiction, film, opera, theatre, dance, radio and television.'
25 )
26 encoding = 'utf-8'
27 language = 'en_GB'
28 remove_empty_feeds = True
29
30 extra_css = '''
31 .label { font-size:small; color:#404040; }
32 .figc { font-size:small; text-align:center; }
33 .desc { font-style:italic; color:#202020; }
34 .auth { font-size:small; }
35 em, blockquote { color:#202020; }
36 .det { font-size:small; color:#202020; }
37 '''
38
39 recipe_specific_options = {
40 'date': {
41 'short': 'The date of the edition to download\nlower case Month-DD-YYYY format',
42 'long': 'For example, july-12-2024',
43 'default': 'current-issue',
44 }
45 }
46
47 def get_browser(self, *args, **kwargs):
48 # Default Chrome UA is served a CloudFront CAPTCHA page that
49 # lacks the rel=shortlink Link header parse_index needs to
50 # extract the WordPress issue id. Safari UA gets the real page.
51 kwargs['user_agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15'
52 return BasicNewsRecipe.get_browser(self, *args, **kwargs)
53
54 def parse_index(self):
55 issue = 'https://www.the-tls.com/issues/current-issue/'
56
57 d = self.recipe_specific_options.get('date')
58 if d and isinstance(d, str):
59 issue = 'https://www.the-tls.com/issues/' + d + '/'
60
61 soup = self.index_to_soup(issue)
62 issue_id = soup.find('link', rel='shortlink')['href'].partition('=')[-1]
63 url = 'https://www.the-tls.com/wp-json/tls/v2/contents-page/' + issue_id
64 raw = self.index_to_soup(url, raw=True)
65 data = json.loads(raw)
66 self.cover_url = data['featuredimage']['full_image'].split('?')[0] + '?w=600'
67 self.timefmt = ' [' + data['issuedateline']['issuedate'] + ']'
68 if data['issuedateline']['issuenumber']:
69 self.description = 'Issue ' + data['issuedateline']['issuenumber']
70
71 feeds = []
72
73 if data['featuredarticle']:
74 self.log('A note from the Editor')
75 feeds.append(('A note from the Editor', [self.get_cont(data['featuredarticle'])]))
76
77 cont = data['contents']
78 for c in cont:
79 section = re_html(cont[c]['articleheader']['title'])
80 if not section:
81 continue
82 self.log(section)
83 articles = []
84 for arts in cont[c]['articleslist']:
85 articles.append(self.get_cont(arts))
86 if articles:
87 feeds.append((section, articles))
88 return feeds
89
90 def get_cont(self, x):
91 url = x['url']
92 title = re_html(x['headline'])
93 desc = re_html(x['standfirst'])
94 if x['byline']['text']:
95 desc = 'By ' + re_html(x['byline']['text']) + ' | ' + desc
96 self.log(' ', title, '\n\t', desc, '\n\t', url)
97 return {'title': title, 'description': desc, 'url': url}
98
99 def preprocess_html(self, soup):
100 for img in soup.findAll('img', attrs={'src': True}):
101 img['src'] = img['src'].split('?')[0] + '?w=600'
102 return soup
103
104 def preprocess_raw_html(self, raw, *a):
105 pg = re.search(r'var tlsPageObject = ({.+)', raw).group(1)
106 data = json.JSONDecoder().raw_decode(pg)[0]
107 if 'articleIntroPrimary' in data:
108 prim = data['articleIntroPrimary']
109 title = '<h1>' + prim['headline'] + '</h1>\n'
110 desc = '<p class="desc">' + prim['standfirst'] + '</p>\n'
111
112 bks = auth = lede = ''
113
114 label = '<div class="label">{}</div>\n'
115 l = prim['label']
116 if l['category']['text'] and l['articletype']:
117 label = label.format(l['articletype'] + ' | ' + l['category']['text'])
118 elif l['articletype']:
119 label = label.format(l['articletype'])
120 elif l['category']['text']:
121 label = label.format(l['category']['text'])
122
123 if prim['byline']['text']:
124 auth = '<p class="auth"><a href="{}">'.format(prim['byline']['link']) + prim['byline']['text'] + '</a></p>\n'
125
126 if prim.get('bookdetails'):
127 bks += '<br>'
128 for a in prim['bookdetails']:
129 for x, y in a.items():
130 if isinstance(y, str):
131 if x == 'imageurl':
132 bks += '<img src="{}">'.format(y)
133 elif y:
134 bks += '<div class="det">' + y + '</div>\n'
135 bks += '<br>'
136 else:
137 prim = title = desc = label = auth = lede = bks = ''
138
139 if 'article_data_leadimage' in data:
140 i = data['article_data_leadimage']
141 if i.get('full_image'):
142 lede = '<br><img src="{}"><div class="figc">{}</div>'.format(i['full_image'] + '?w600', i['imagecaption'] + ' <i>' + i['imagecredit'] + '</i>')
143 cont = self.index_to_soup('https://www.the-tls.com/wp-json/wp/v2/tls_articles/' + data['ID'], raw=True)
144 c_data = json.loads(cont)
145 body = c_data['content']['rendered']
146
147 html = '<html><body><div>' + label + title + desc + auth + lede + bks + body + '</div></body></html>'
148 return BeautifulSoup(html).prettify()