Debian Codegraph @

recipes/hamilton_spectator.recipe

raw
1 #!/usr/bin/env python
2 from calibre.web.feeds.news import BasicNewsRecipe, classes
3
4 '''
5 Hamilton Spectator Calibre Recipe
6 '''
7
8
9 def absurl(url):
10 if url.startswith('/'):
11 return 'https://www.thespec.com' + url
12
13
14 class HamiltonSpectator(BasicNewsRecipe):
15 title = 'Hamilton Spectator'
16 max_articles_per_feed = 50
17 __author__ = 'unkn0wn'
18 publisher = 'thespec.com'
19 description = 'Ontario Canada Newspaper'
20 category = 'News, Ontario, Canada'
21 remove_javascript = True
22 use_embedded_content = False
23 no_stylesheets = True
24 language = 'en_CA'
25 encoding = 'utf-8'
26 remove_attributes = ['style', 'height', 'width']
27 masthead_url = 'https://bloximages.chicago2.vip.townnews.com/thespec.com/content/tncms/custom/image/c0094646-1108-11ee-8af0-b3954ce40e5e.png'
28
29 ignore_duplicate_articles = {'title', 'url'}
30
31 extra_css = '''
32 .caption { font-size:small; text-align:center; }
33 .authorList, .endnote_contrib { font-size:small; }
34 '''
35
36 keep_only_tags = [classes('headline asset-summary authorList articleMainArt asset-body')]
37
38 remove_tags = [
39 dict(name=['svg', 'button']),
40 dict(attrs={'id': ['tncms-region-article_instory_top', 'tncms-region-article_bottom', 'asset-video-primary']}),
41 classes(
42 'tnt-blurred-image share-container subscriber-offers access-offers-in-page '
43 'access-offers-wrapper tnt-ads-container adLabelWrapperManual shareIcons '
44 'articleFeedbackCTA comments-container card-image'
45 ),
46 ]
47
48 def preprocess_html(self, soup):
49 for img in soup.findAll('img', attrs={'data-srcset': True}):
50 for x in img['data-srcset'].split(','):
51 if '640w' in x.split():
52 img['src'] = x.split()[0]
53 return soup
54
55 def parse_index(self):
56 index = 'https://www.thespec.com/'
57 sections = ['news', 'politics', 'opinion', 'business', 'sports', 'life', 'entertainment']
58 feeds = []
59 soup = self.index_to_soup(index)
60 for sec in sections:
61 section = sec.capitalize()
62 self.log(section)
63 articles = []
64 for a in soup.findAll('a', attrs={'href': lambda x: x and x.startswith('/' + sec + '/')}):
65 url = absurl(a['href'].split('#')[0])
66 if url in {index + sec + '/', index + sec}:
67 continue
68 if not url.endswith('.html'):
69 continue
70 title = self.tag_to_string(a)
71 self.log('\t', title, '\n\t\t', url)
72 articles.append({'title': title, 'url': url})
73 if articles:
74 feeds.append((section, articles))
75 return feeds