Debian Codegraph @

recipes/spectator_magazine.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 from calibre.web.feeds.news import BasicNewsRecipe, classes
4
5
6 def absurl(url):
7 if url.startswith('/'):
8 url = 'https://www.spectator.co.uk' + url
9 return url
10
11
12 class spectator(BasicNewsRecipe):
13 title = 'Spectator Magazine'
14 __author__ = 'unkn0wn'
15 description = 'The Spectator was established in 1828, and is the best-written and most influential weekly in the English language.'
16 language = 'en_GB'
17 no_stylesheets = True
18 remove_attributes = ['height', 'width', 'style']
19 ignore_duplicate_articles = {'url'}
20 masthead_url = 'https://upload.wikimedia.org/wikipedia/commons/thumb/c/c7/The_Spectator_logo.svg/320px-The_Spectator_logo.svg.png'
21 encoding = 'utf-8'
22 remove_empty_feeds = True
23 resolve_internal_links = True
24
25 extra_css = '''
26 .writers-link__text, .author-bio__content {font-size:small; color:#404040;}
27 #fig-c {text-align:center; font-size:small;}
28 blockquote, em, i {color:#202020;}
29 img {display:block; margin:0 auto;}
30 '''
31
32 keep_only_tags = [classes('writers-link entry-header__author entry-header__title entry-header__thumbnail entry-content author-bio__content ')]
33
34 remove_tags = [
35 dict(name=['svg', 'button']),
36 classes('entry-meta audio-read-block insert--most-popular ad-slot ad-slot--in-content ad-content subscription-banner '),
37 ]
38
39 def preprocess_html(self, soup):
40 for fc in soup.findAll('figcaption'):
41 fc['id'] = 'fig-c'
42 return soup
43
44 def get_browser(self, *args, **kwargs):
45 kwargs['user_agent'] = 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
46 br = BasicNewsRecipe.get_browser(self, *args, **kwargs)
47 br.addheaders += [('Referer', 'https://www.google.com/'), ('X-Forwarded-For', '66.249.66.1')]
48 return br
49
50 recipe_specific_options = {'date': {'short': 'The date of the edition to download (DD-MM-YYYY format)', 'long': 'For example, 20-07-2024'}}
51
52 def parse_index(self):
53 index = 'https://www.spectator.co.uk/magazine'
54 d = self.recipe_specific_options.get('date')
55 if d and isinstance(d, str):
56 index = index + '/' + d + '/'
57 soup = self.index_to_soup(index)
58 self.cover_url = soup.find(**classes('magazine-header__container')).img['src'].split('?')[0]
59 issue = self.tag_to_string(soup.find(**classes('magazine-header__title'))).strip()
60 time = soup.find('time')
61 self.title = 'The Spectator ' + issue
62 self.timefmt = ' [' + self.tag_to_string(time) + ']'
63 self.log('Downloading Issue: ', self.title, self.timefmt)
64 nav_div = soup.find('ul', **classes('archive-entry__nav-list'))
65 section_list = []
66
67 for x in nav_div.findAll(['a']):
68 section_list.append((self.tag_to_string(x).strip(), absurl(x['href'])))
69 feeds = []
70
71 # For each section title, fetch the article urls
72 for section in section_list:
73 section_title = section[0]
74 section_url = section[1]
75 self.log(section_title, section_url)
76 soup = self.index_to_soup(section_url)
77 articles = self.articles_from_soup(soup)
78 if articles:
79 feeds.append((section_title, articles))
80 return feeds
81
82 def articles_from_soup(self, soup):
83 ans = []
84 for div in soup.findAll('div', **classes('mosaic__tile mosaic__tile--lead-up')):
85 a = div.find('a', href=True, attrs={'class': 'article__title-link'})
86 url = absurl(a['href'])
87 title = self.tag_to_string(a).strip()
88 teaser = div.find('p', **classes('article__excerpt-text'))
89 desc = ''
90 if teaser:
91 desc = self.tag_to_string(teaser).strip()
92 obj = div.find('a', **classes('article__author article__author--link'))
93 if obj:
94 desc = self.tag_to_string(obj).strip() + ' | ' + desc
95 sec = div.findParent('div').find('a', attrs={'class': 'magazine-issue__entry-link'})
96 if sec:
97 desc = self.tag_to_string(sec).strip() + ' | ' + desc
98
99 self.log('\t', title, '\n\t', desc, '\n\t\t', url)
100 ans.append({'title': title, 'description': desc, 'url': url})
101 return ans