Debian Codegraph @

recipes/abc_au.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 from __future__ import absolute_import, division, print_function, unicode_literals
4
5 __license__ = 'GPL v3'
6 __copyright__ = '2020, Pat Stapleton <pat.stapleton at gmail.com>'
7 '''
8 Recipe for ABC News Australia (online)
9 '''
10 from calibre.web.feeds.news import BasicNewsRecipe
11
12
13 class ABCNews(BasicNewsRecipe):
14 title = 'ABC News'
15 language = 'en_AU'
16 __author__ = 'Pat Stapleton'
17 description = 'From the Australian Broadcasting Corporation. The ABC is owned and funded by the Australian Government, but is editorially independent.'
18 masthead_url = 'https://www.abc.net.au/cm/lb/8212706/data/news-logo-2017---desktop-print-data.png'
19 cover_url = 'https://www.abc.net.au/news/linkableblob/8413676/data/abc-news-og-data.jpg'
20 cover_margins = (0, 20, '#000000')
21 scale_news_images_to_device = True
22 oldest_article = 2 # days
23 simultaneous_downloads = 10
24 max_articles_per_feed = 100
25 publication_type = 'newspaper'
26
27 recipe_specific_options = {
28 'days': {
29 'short': 'Oldest article to download from this news source. In days ',
30 'long': 'For example, 0.5, gives you articles from the past 12 hours',
31 'default': str(oldest_article),
32 }
33 }
34
35 def __init__(self, *args, **kwargs):
36 BasicNewsRecipe.__init__(self, *args, **kwargs)
37 d = self.recipe_specific_options.get('days')
38 if d and isinstance(d, str):
39 self.oldest_article = float(d)
40
41 # auto_cleanup = True # enable this as a backup option if recipe stops working
42
43 # use_embedded_content = False # if set to true will assume that all the article content is within the feed (i.e. won't try to fetch more data)
44
45 no_stylesheets = True
46 remove_javascript = True
47
48 keep_only_tags = [dict(id='content')] # the article content is contained in <main id="content" /> tag
49
50 # ************************************
51 # Regular expressions for remove_tags:
52 # ************************************
53 # remove aside tag - used for overlapping boxes within article
54 # aside_reg_exp = '^.*aside.*$'
55
56 # ************************************
57 # Clear out all the unwanted html tags:
58 # ************************************
59 remove_tags = [
60 # dict(name='aside', attrs={'name': re.compile(aside_reg_exp, re.IGNORECASE)})
61 {'name': ['meta', 'link', 'noscript', 'aside']},
62 {
63 'attrs': {
64 'data-component': [
65 'Ticker',
66 'PublishedDate',
67 'Timestamp',
68 'Link',
69 'ShareLink',
70 'ShareUtility',
71 'RelatedStories',
72 'ArticleTopStories',
73 'ArticleTopStoriesCard',
74 'ArticleJustInStories',
75 'RelatedTopics',
76 'Player',
77 'ArticleSidebar',
78 'TopStoriesSidebar',
79 'UtilityBar',
80 ]
81 }
82 },
83 ]
84
85 # ************************************
86 # Tidy up the output to look neat for reading
87 # ************************************
88 remove_attributes = ['width', 'height', 'style']
89 extra_css = '.byline{font-size:smaller;margin-bottom:10px;}.inline-caption{display:block;font-size:smaller;text-decoration: none;}'
90
91 # ************************************
92 # Fix images (dynamically generated by ABC news)
93 # ************************************
94 def preprocess_html(self, soup):
95 for img in soup.findAll('img', attrs={'data-src': True}):
96 for x in img['data-src'].split():
97 if '/' in x:
98 img['src'] = x
99 return soup
100
101 compress_news_images = True
102
103 feeds = [
104 ('Top Stories', 'https://www.abc.net.au/news/feed/45910/rss.xml'),
105 ('Politics', 'https://www.abc.net.au/news/feed/1534/rss.xml'),
106 ('World', 'https://www.abc.net.au/news/feed/4405318/rss.xml'),
107 ('Business', 'https://www.abc.net.au/news/feed/51892/rss.xml'),
108 ('Analysis', 'https://www.abc.net.au/news/feed/7571268/rss.xml'),
109 ('Sport', 'https://www.abc.net.au/news/feed/2942460/rss.xml'),
110 ('Science', 'https://www.abc.net.au/news/feed/8132426/rss.xml'),
111 ('Health', 'https://www.abc.net.au/news/feed/9167762/rss.xml'),
112 ('Arts and Entertainment', 'https://www.abc.net.au/news/feed/472/rss.xml'),
113 ('Fact Check', 'https://www.abc.net.au/news/feed/5306468/rss.xml'),
114 # ('Adelaide', 'https://www.abc.net.au/news/feed/8057540/rss.xml'), #enable by removing # at start of line
115 # ('Brisbane', 'https://www.abc.net.au/news/feed/8053540/rss.xml'), #enable by removing # at start of line
116 # ('Canberra', 'https://www.abc.net.au/news/feed/8057234/rss.xml'), #enable by removing # at start of line
117 # ('Darwin', 'https://www.abc.net.au/news/feed/8057648/rss.xml'), #enable by removing # at start of line
118 # ('Hobart', 'https://www.abc.net.au/news/feed/8054562/rss.xml'), #enable by removing # at start of line
119 # ('Melbourne', 'https://www.abc.net.au/news/feed/8057136/rss.xml'), #enable by removing # at start of line
120 # ('Perth', 'https://www.abc.net.au/news/feed/8057096/rss.xml'), #enable by removing # at start of line
121 # ('Sydney', 'https://www.abc.net.au/news/feed/8055316/rss.xml'), #enable by removing # at start of line
122 ]