Debian Codegraph @

recipes/globe_and_mail.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 # License: GPLv3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
4
5 from calibre.web.feeds.news import BasicNewsRecipe
6
7
8 def absolutize(href):
9 if href.startswith('/'):
10 href = 'https://www.theglobeandmail.com' + href
11 return href
12
13
14 def classes(classes):
15 q = frozenset(classes.split(' '))
16 return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)})
17
18
19 class GlobeMail(BasicNewsRecipe):
20 title = 'The Globe and Mail'
21 __author__ = 'Kovid Goyal'
22 encoding = 'utf-8'
23 publisher = 'Globe & Mail'
24 language = 'en_CA'
25 ignore_duplicate_articles = {'title', 'url'}
26 no_stylesheets = True
27 remove_attributes = ['style']
28 delay = 1
29
30 keep_only_tags = [classes('article-title c-article-meta l-article l-article-title')]
31 remove_tags = [
32 classes('c-ad pb-f-commercial-dfp-ads pb-f-article-actions pb-f-article-meta article-toolbar'),
33 dict(name='source'),
34 ]
35
36 def parse_index(self):
37 ans = []
38 for section in 'canada opinion politics sports life arts world real-estate'.split():
39 if self.test and len(ans) >= self.test[0]:
40 break
41 soup = self.index_to_soup('https://www.theglobeandmail.com/{}/'.format(section))
42 self.log('Processing section:', section)
43 articles = list(self.parse_gm_section(soup))
44 if articles:
45 ans.append((section.capitalize(), articles))
46 return ans
47
48 def parse_gm_section(self, soup):
49 for a in soup.findAll('a', href=True, attrs={'data-lt-pos': lambda x: x and x.endswith('featured')}):
50 headline = a.find('div', 'c-card__hed-text')
51 title = self.tag_to_string(headline)
52 url = absolutize(a['href'])
53 self.log(' ', title, 'at', url)
54 yield {'title': title, 'url': url}