Debian Codegraph @

recipes/independent.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 from calibre import random_user_agent
4 from calibre.ebooks.BeautifulSoup import Tag
5 from calibre.web.feeds.recipes import BasicNewsRecipe
6
7
8 def classes(classes):
9 q = frozenset(classes.split(' '))
10 return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)})
11
12
13 def new_tag(soup, name, attrs=()):
14 impl = getattr(soup, 'new_tag', None)
15 if impl is not None:
16 return impl(name, attrs=dict(attrs))
17 return Tag(soup, name, attrs=attrs or None)
18
19
20 class TheIndependentNew(BasicNewsRecipe):
21 title = 'The Independent'
22 __author__ = 'Krittika Goyal'
23 description = (
24 'The latest in UK News and World News from The '
25 'Independent. Wide range of international and local news, sports '
26 'news, commentary and opinion pieces.Independent News - Breaking news '
27 'that matters. Your daily comprehensive news source - The '
28 'Independent Newspaper'
29 )
30 publisher = 'The Independent'
31 oldest_article = 2.0
32 ignore_duplicate_articles = {'title', 'url'}
33 category = 'news, UK'
34 no_stylesheets = True
35 use_embedded_content = False
36 remove_empty_feeds = True
37 language = 'en_GB'
38 publication_type = 'newspaper'
39 encoding = 'utf-8'
40 compress_news_images = True
41
42 recipe_specific_options = {
43 'days': {
44 'short': 'Oldest article to download from this news source. In days ',
45 'long': 'For example, 0.5, gives you articles from the past 12 hours',
46 'default': str(oldest_article),
47 }
48 }
49
50 def __init__(self, *args, **kwargs):
51 BasicNewsRecipe.__init__(self, *args, **kwargs)
52 d = self.recipe_specific_options.get('days')
53 if d and isinstance(d, str):
54 self.oldest_article = float(d)
55
56 keep_only_tags = [
57 dict(id=['articleHeader', 'main']),
58 classes('headline sub-headline breadcrumb author publish-date hero-image body-content'),
59 ]
60 remove_tags = [
61 classes(
62 'inline-related inline-readmore ad-wrapper icon-gallery i-gallery article-im-prompt apester-media'
63 ' social-share i-amphtml-replaced-content gallery-btn related'
64 ),
65 dict(name='amp-live-list'),
66 ]
67 remove_attributes = ['style']
68
69 def get_browser(self, *a, **kw):
70 # This site returns images in JPEG-XR format if the user agent is IE
71 if not hasattr(self, 'non_ie_ua'):
72 try:
73 self.non_ie_ua = random_user_agent(allow_ie=False)
74 except TypeError:
75 self.non_ie_ua = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.111 Safari/537.36'
76 kw['user_agent'] = self.non_ie_ua
77 br = BasicNewsRecipe.get_browser(self, *a, **kw)
78 return br
79
80 def preprocess_html(self, soup):
81 for img in soup.findAll('amp-img'):
82 img.name = 'img'
83 img['srcset'] = ''
84
85 for div in soup.findAll(attrs={'class': 'full-gallery'}):
86 imgs = {}
87 for li in div.findAll('li', attrs={'data-gallery-item': True, 'data-original': True}):
88 imgs[li['data-gallery-item']] = li['data-original']
89 li.extract()
90 for li in div.findAll('li', attrs={'data-gallery-legend': True}):
91 src = imgs.get(li['data-gallery-legend'])
92 if src is not None:
93 img = new_tag(soup, 'img')
94 img['src'] = src
95 img['style'] = 'display:block'
96 li.append(img)
97 return soup
98
99 feeds = [
100 ('News - UK', 'http://www.independent.co.uk/news/uk/rss'),
101 ('News - World', 'http://www.independent.co.uk/news/world/rss'),
102 ('News - Business', 'http://www.independent.co.uk/news/business/rss'),
103 ('News - People', 'http://www.independent.co.uk/news/people/rss'),
104 ('News - Science', 'http://www.independent.co.uk/news/science/rss'),
105 ('News - Media', 'http://www.independent.co.uk/news/media/rss'),
106 ('News - Education', 'http://www.independent.co.uk/news/education/rss'),
107 ('News - Obituaries', 'http://www.independent.co.uk/news/obituaries/rss'),
108 ('News - Corrections', 'http://www.independent.co.uk/news/corrections/rss'),
109 ('Voices', 'http://www.independent.co.uk/voices/rss'),
110 ('Environment', 'http://www.independent.co.uk/environment/rss'),
111 ('Sport - Athletics', 'http://www.independent.co.uk/sport/general/athletics/rss'),
112 ('Sport - Cricket', 'http://www.independent.co.uk/sport/cricket/rss'),
113 ('Sport - Football', 'http://www.independent.co.uk/sport/football/rss'),
114 ('Sport - Golf', 'http://www.independent.co.uk/sport/golf/rss'),
115 ('Sport - Motor racing', 'http://www.independent.co.uk/sport/motor-racing/rss'),
116 ('Sport - Olympics', 'http://www.independent.co.uk/sport/olympics/rss'),
117 ('Sport - Racing', 'http://www.independent.co.uk/sport/racing/rss'),
118 ('Sport - Rugby League', 'http://www.independent.co.uk/sport/general/rugby-league/rss'),
119 ('Sport - Rugby Union', 'http://www.independent.co.uk/sport/rugby/rugby-union/rss'),
120 ('Sport - Sailing', 'http://www.independent.co.uk/sport/general/sailing/rss'),
121 ('Sport - Tennis', 'http://www.independent.co.uk/sport/tennis/rss'),
122 ('Sport - Others', 'http://www.independent.co.uk/sport/general/others/rss'),
123 ('Life & Style - Fashion', 'http://www.independent.co.uk/life-style/fashion/rss'),
124 ('Life & Style -Food & Drink', 'http://www.independent.co.uk/life-style/food-and-drink/rss'),
125 ('Life & Style - Health and Families', 'http://www.independent.co.uk/life-style/health-and-families/rss'),
126 ('Life & Style - History', 'http://www.independent.co.uk/life-style/history/rss'),
127 ('Life & Style - Gadgets & Tech', 'http://www.independent.co.uk/life-style/gadgets-and-tech/rss'),
128 ('Life & Style - Motoring', 'http://www.independent.co.uk/life-style/motoring/rss'),
129 ('Arts & Ents - Art', 'http://www.independent.co.uk/arts-entertainment/art/rss'),
130 ('Arts & Ents - Architecture', 'http://www.independent.co.uk/arts-entertainment/architecture/rss'),
131 ('Arts & Ents - Music', 'http://www.independent.co.uk/arts-entertainment/music/rss'),
132 ('Arts & Ents - Classical', 'http://www.independent.co.uk/arts-entertainment/classical/rss'),
133 ('Arts & Ents - Films', 'http://www.independent.co.uk/arts-entertainment/films/rss'),
134 ('Arts & Ents - TV', 'http://www.independent.co.uk/arts-entertainment/tv/rss'),
135 ('Arts & Ents - Theatre and Dance', 'http://www.independent.co.uk/arts-entertainment/theatre-dance/rss'),
136 ('Arts & Ents - Comedy', 'http://www.independent.co.uk/arts-entertainment/comedy/rss'),
137 ('Arts & Ents - Books', 'http://www.independent.co.uk/arts-entertainment/books/rss'),
138 ('Travel', 'http://www.independent.co.uk/travel/rss'),
139 ('Money', 'http://www.independent.co.uk/money/rss'),
140 ('IndyBest', 'http://www.independent.co.uk/extras/indybest/rss'),
141 ]