| 1 |
#!/usr/bin/env python |
| 2 |
# vim:fileencoding=utf-8 |
| 3 |
from __future__ import absolute_import, division, print_function, unicode_literals |
| 4 |
|
| 5 |
__license__ = 'GPL v3' |
| 6 |
__copyright__ = '2020, Pat Stapleton <pat.stapleton at gmail.com>' |
| 7 |
''' |
| 8 |
Recipe for ABC News Australia (online) |
| 9 |
''' |
| 10 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 11 |
|
| 12 |
|
| 13 |
class ABCNews(BasicNewsRecipe): |
| 14 |
title = 'ABC News' |
| 15 |
language = 'en_AU' |
| 16 |
__author__ = 'Pat Stapleton' |
| 17 |
description = 'From the Australian Broadcasting Corporation. The ABC is owned and funded by the Australian Government, but is editorially independent.' |
| 18 |
masthead_url = 'https://www.abc.net.au/cm/lb/8212706/data/news-logo-2017---desktop-print-data.png' |
| 19 |
cover_url = 'https://www.abc.net.au/news/linkableblob/8413676/data/abc-news-og-data.jpg' |
| 20 |
cover_margins = (0, 20, '#000000') |
| 21 |
scale_news_images_to_device = True |
| 22 |
oldest_article = 2 # days |
| 23 |
simultaneous_downloads = 10 |
| 24 |
max_articles_per_feed = 100 |
| 25 |
publication_type = 'newspaper' |
| 26 |
|
| 27 |
recipe_specific_options = { |
| 28 |
'days': { |
| 29 |
'short': 'Oldest article to download from this news source. In days ', |
| 30 |
'long': 'For example, 0.5, gives you articles from the past 12 hours', |
| 31 |
'default': str(oldest_article), |
| 32 |
} |
| 33 |
} |
| 34 |
|
| 35 |
def __init__(self, *args, **kwargs): |
| 36 |
BasicNewsRecipe.__init__(self, *args, **kwargs) |
| 37 |
d = self.recipe_specific_options.get('days') |
| 38 |
if d and isinstance(d, str): |
| 39 |
self.oldest_article = float(d) |
| 40 |
|
| 41 |
# auto_cleanup = True # enable this as a backup option if recipe stops working |
| 42 |
|
| 43 |
# use_embedded_content = False # if set to true will assume that all the article content is within the feed (i.e. won't try to fetch more data) |
| 44 |
|
| 45 |
no_stylesheets = True |
| 46 |
remove_javascript = True |
| 47 |
|
| 48 |
keep_only_tags = [dict(id='content')] # the article content is contained in <main id="content" /> tag |
| 49 |
|
| 50 |
# ************************************ |
| 51 |
# Regular expressions for remove_tags: |
| 52 |
# ************************************ |
| 53 |
# remove aside tag - used for overlapping boxes within article |
| 54 |
# aside_reg_exp = '^.*aside.*$' |
| 55 |
|
| 56 |
# ************************************ |
| 57 |
# Clear out all the unwanted html tags: |
| 58 |
# ************************************ |
| 59 |
remove_tags = [ |
| 60 |
# dict(name='aside', attrs={'name': re.compile(aside_reg_exp, re.IGNORECASE)}) |
| 61 |
{'name': ['meta', 'link', 'noscript', 'aside']}, |
| 62 |
{ |
| 63 |
'attrs': { |
| 64 |
'data-component': [ |
| 65 |
'Ticker', |
| 66 |
'PublishedDate', |
| 67 |
'Timestamp', |
| 68 |
'Link', |
| 69 |
'ShareLink', |
| 70 |
'ShareUtility', |
| 71 |
'RelatedStories', |
| 72 |
'ArticleTopStories', |
| 73 |
'ArticleTopStoriesCard', |
| 74 |
'ArticleJustInStories', |
| 75 |
'RelatedTopics', |
| 76 |
'Player', |
| 77 |
'ArticleSidebar', |
| 78 |
'TopStoriesSidebar', |
| 79 |
'UtilityBar', |
| 80 |
] |
| 81 |
} |
| 82 |
}, |
| 83 |
] |
| 84 |
|
| 85 |
# ************************************ |
| 86 |
# Tidy up the output to look neat for reading |
| 87 |
# ************************************ |
| 88 |
remove_attributes = ['width', 'height', 'style'] |
| 89 |
extra_css = '.byline{font-size:smaller;margin-bottom:10px;}.inline-caption{display:block;font-size:smaller;text-decoration: none;}' |
| 90 |
|
| 91 |
# ************************************ |
| 92 |
# Fix images (dynamically generated by ABC news) |
| 93 |
# ************************************ |
| 94 |
def preprocess_html(self, soup): |
| 95 |
for img in soup.findAll('img', attrs={'data-src': True}): |
| 96 |
for x in img['data-src'].split(): |
| 97 |
if '/' in x: |
| 98 |
img['src'] = x |
| 99 |
return soup |
| 100 |
|
| 101 |
compress_news_images = True |
| 102 |
|
| 103 |
feeds = [ |
| 104 |
('Top Stories', 'https://www.abc.net.au/news/feed/45910/rss.xml'), |
| 105 |
('Politics', 'https://www.abc.net.au/news/feed/1534/rss.xml'), |
| 106 |
('World', 'https://www.abc.net.au/news/feed/4405318/rss.xml'), |
| 107 |
('Business', 'https://www.abc.net.au/news/feed/51892/rss.xml'), |
| 108 |
('Analysis', 'https://www.abc.net.au/news/feed/7571268/rss.xml'), |
| 109 |
('Sport', 'https://www.abc.net.au/news/feed/2942460/rss.xml'), |
| 110 |
('Science', 'https://www.abc.net.au/news/feed/8132426/rss.xml'), |
| 111 |
('Health', 'https://www.abc.net.au/news/feed/9167762/rss.xml'), |
| 112 |
('Arts and Entertainment', 'https://www.abc.net.au/news/feed/472/rss.xml'), |
| 113 |
('Fact Check', 'https://www.abc.net.au/news/feed/5306468/rss.xml'), |
| 114 |
# ('Adelaide', 'https://www.abc.net.au/news/feed/8057540/rss.xml'), #enable by removing # at start of line |
| 115 |
# ('Brisbane', 'https://www.abc.net.au/news/feed/8053540/rss.xml'), #enable by removing # at start of line |
| 116 |
# ('Canberra', 'https://www.abc.net.au/news/feed/8057234/rss.xml'), #enable by removing # at start of line |
| 117 |
# ('Darwin', 'https://www.abc.net.au/news/feed/8057648/rss.xml'), #enable by removing # at start of line |
| 118 |
# ('Hobart', 'https://www.abc.net.au/news/feed/8054562/rss.xml'), #enable by removing # at start of line |
| 119 |
# ('Melbourne', 'https://www.abc.net.au/news/feed/8057136/rss.xml'), #enable by removing # at start of line |
| 120 |
# ('Perth', 'https://www.abc.net.au/news/feed/8057096/rss.xml'), #enable by removing # at start of line |
| 121 |
# ('Sydney', 'https://www.abc.net.au/news/feed/8055316/rss.xml'), #enable by removing # at start of line |
| 122 |
] |