Debian Codegraph @

recipes/afr.recipe

raw
1 #!/usr/bin/env python
2 from datetime import date
3
4 from calibre.web.feeds.news import BasicNewsRecipe
5
6
7 def absurl(url):
8 if url.startswith('/'):
9 return 'https://www.afr.com' + url
10
11
12 class afr(BasicNewsRecipe):
13 title = 'Australian Financial Review'
14 __author__ = 'unkn0wn'
15 description = (
16 'For more than 65 years The Australian Financial Review has been the authority on business,'
17 ' finance and investment news in Australia. It has a reputation for independent, award-winning '
18 "journalism and is essential reading for Australia's business and investor community."
19 )
20 masthead_url = 'https://www.nineforbrands.com.au/wp-content/uploads/2020/08/AFR-DHOSP-Logo-black-RGB.png'
21 encoding = 'utf-8'
22 language = 'en_AU'
23
24 use_embedded_content = False
25 timefmt = ' [%d %b %Y]'
26 max_articles_per_feed = 25
27 no_stylesheets = True
28 remove_attributes = ['style', 'height', 'width']
29
30 keep_only_tags = [dict(name=['article', 'main'], attrs={'id': 'content'})]
31
32 remove_tags = [
33 dict(attrs={'data-testid': ['ArticleTools', 'ArticleBreadcrumb-Links', 'ad-wrapper', 'ArticleFooter', 'ArticleTags', 'beyondwords-player-wrapper']}),
34 dict(name=['button', 'aside', 'svg']),
35 ]
36
37 remove_tags_after = [dict(name='aside', attrs={'id': 'stickyContainer'})]
38
39 extra_css = '''
40 #img-cap {font-size:small; text-align:center;}
41 [data-testid="AuthorNames"], [data-testid="ArticleTimestamp"] {font-size:small;}
42 '''
43
44 ignore_duplicate_articles = {'title', 'url'}
45 resolve_internal_links = True
46 remove_empty_feeds = True
47
48 def preprocess_html(self, soup):
49 for img in soup.findAll('img', attrs={'data-src': True}):
50 img['src'] = img['data-src']
51 for fig in soup.findAll('figcaption'):
52 fig['id'] = 'img-cap'
53 return soup
54
55 def parse_index(self):
56 index = 'https://www.afr.com/'
57 sections = [
58 'companies',
59 'market',
60 'politics',
61 'policy',
62 'world',
63 'wealth',
64 'street-talk',
65 'chaticleer',
66 'rear-window',
67 'life-and-luxury',
68 'technology',
69 'property',
70 'work-and-careers',
71 ]
72 feeds = []
73 soup = self.index_to_soup(index)
74 for sec in sections:
75 section = sec.capitalize()
76 self.log(section)
77 articles = []
78 for a in soup.findAll('a', attrs={'href': lambda x: x and x.startswith('/' + sec + '/')}):
79 url = absurl(a['href'].split('?')[0])
80 if url in {index + sec + '/', index + sec}:
81 continue
82 if date.today().strftime('%Y') not in url:
83 continue
84 title = self.tag_to_string(a)
85 self.log('\t', title, '\n\t\t', url)
86 articles.append({'title': title, 'url': url})
87 if articles:
88 feeds.append((section, articles))
89 return feeds