Debian Codegraph @

recipes/the_oz.recipe

raw
1 #!/usr/bin/env python
2 __license__ = 'GPL v3'
3 __copyright__ = '2009, Matthew Briggs'
4 __docformat__ = 'restructuredtext en'
5
6 '''
7 http://www.theaustralian.news.com.au/
8 '''
9
10 from calibre.web.feeds.news import BasicNewsRecipe
11
12
13 def classes(classes):
14 q = frozenset(classes.split(' '))
15 return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)})
16
17
18 class DailyTelegraph(BasicNewsRecipe):
19 title = 'The Australian'
20 __author__ = 'Kovid Goyal'
21 description = (
22 'National broadsheet newspaper from down under - colloquially known as The Oz'
23 '. You will need to have a subscription to '
24 'http://www.theaustralian.com.au to get full articles.'
25 )
26 language = 'en_AU'
27
28 oldest_article = 2
29 needs_subscription = 'optional'
30 max_articles_per_feed = 30
31 remove_javascript = True
32 no_stylesheets = True
33 encoding = 'utf8'
34 remove_empty_feeds = True
35 ignore_duplicate_articles = {'url'}
36
37 keep_only_tags = [
38 classes('story-headline'),
39 dict(id='story'),
40 ]
41 remove_tags = [
42 dict(id='comments'),
43 classes(
44 'story-info story-header-tools module-controls story-sidebar'
45 ' story-footer story-footer-tools story-extras story-related vms-nav'
46 ' vms-endcard vms-discover share-tools story-comments-link'
47 ' vms-controls ooyala-player vms-countdown vms-header comments'
48 ),
49 ]
50
51 feeds = [
52 ('National News', 'https://www.news.com.au/content-feeds/latest-news-national/'),
53 ('World News', 'https://www.news.com.au/content-feeds/latest-news-world/'),
54 ('Entertainment', 'https://www.news.com.au/content-feeds/latest-news-entertainment/'),
55 ('Technology', 'https://www.news.com.au/content-feeds/latest-news-technology/'),
56 ('Lifestyle', 'https://www.news.com.au/content-feeds/latest-news-lifestyle/'),
57 ('Sport', 'https://www.news.com.au/content-feeds/latest-news-sport/'),
58 ('Finance', 'https://www.news.com.au/content-feeds/latest-news-finance/'),
59 ('Travel', 'https://www.news.com.au/content-feeds/latest-news-travel/'),
60 ]
61
62 def get_browser(self):
63 return BasicNewsRecipe.get_browser(self, user_agent='common_words/based')
64
65 def get_article_url(self, article):
66 ans = article.link
67 if '/video/' in ans:
68 return
69 return ans
70
71 def preprocess_html(self, soup):
72 for img in soup.findAll('img', attrs={'data-src': True}):
73 img['src'] = img['data-src']
74 return soup