| 1 |
#!/usr/bin/env python |
| 2 |
# vim:fileencoding=utf-8 |
| 3 |
from calibre import random_user_agent |
| 4 |
from calibre.ebooks.BeautifulSoup import Tag |
| 5 |
from calibre.web.feeds.recipes import BasicNewsRecipe |
| 6 |
|
| 7 |
|
| 8 |
def classes(classes): |
| 9 |
q = frozenset(classes.split(' ')) |
| 10 |
return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)}) |
| 11 |
|
| 12 |
|
| 13 |
def new_tag(soup, name, attrs=()): |
| 14 |
impl = getattr(soup, 'new_tag', None) |
| 15 |
if impl is not None: |
| 16 |
return impl(name, attrs=dict(attrs)) |
| 17 |
return Tag(soup, name, attrs=attrs or None) |
| 18 |
|
| 19 |
|
| 20 |
class TheIndependentNew(BasicNewsRecipe): |
| 21 |
title = 'The Independent' |
| 22 |
__author__ = 'Krittika Goyal' |
| 23 |
description = ( |
| 24 |
'The latest in UK News and World News from The ' |
| 25 |
'Independent. Wide range of international and local news, sports ' |
| 26 |
'news, commentary and opinion pieces.Independent News - Breaking news ' |
| 27 |
'that matters. Your daily comprehensive news source - The ' |
| 28 |
'Independent Newspaper' |
| 29 |
) |
| 30 |
publisher = 'The Independent' |
| 31 |
oldest_article = 2.0 |
| 32 |
ignore_duplicate_articles = {'title', 'url'} |
| 33 |
category = 'news, UK' |
| 34 |
no_stylesheets = True |
| 35 |
use_embedded_content = False |
| 36 |
remove_empty_feeds = True |
| 37 |
language = 'en_GB' |
| 38 |
publication_type = 'newspaper' |
| 39 |
encoding = 'utf-8' |
| 40 |
compress_news_images = True |
| 41 |
|
| 42 |
recipe_specific_options = { |
| 43 |
'days': { |
| 44 |
'short': 'Oldest article to download from this news source. In days ', |
| 45 |
'long': 'For example, 0.5, gives you articles from the past 12 hours', |
| 46 |
'default': str(oldest_article), |
| 47 |
} |
| 48 |
} |
| 49 |
|
| 50 |
def __init__(self, *args, **kwargs): |
| 51 |
BasicNewsRecipe.__init__(self, *args, **kwargs) |
| 52 |
d = self.recipe_specific_options.get('days') |
| 53 |
if d and isinstance(d, str): |
| 54 |
self.oldest_article = float(d) |
| 55 |
|
| 56 |
keep_only_tags = [ |
| 57 |
dict(id=['articleHeader', 'main']), |
| 58 |
classes('headline sub-headline breadcrumb author publish-date hero-image body-content'), |
| 59 |
] |
| 60 |
remove_tags = [ |
| 61 |
classes( |
| 62 |
'inline-related inline-readmore ad-wrapper icon-gallery i-gallery article-im-prompt apester-media' |
| 63 |
' social-share i-amphtml-replaced-content gallery-btn related' |
| 64 |
), |
| 65 |
dict(name='amp-live-list'), |
| 66 |
] |
| 67 |
remove_attributes = ['style'] |
| 68 |
|
| 69 |
def get_browser(self, *a, **kw): |
| 70 |
# This site returns images in JPEG-XR format if the user agent is IE |
| 71 |
if not hasattr(self, 'non_ie_ua'): |
| 72 |
try: |
| 73 |
self.non_ie_ua = random_user_agent(allow_ie=False) |
| 74 |
except TypeError: |
| 75 |
self.non_ie_ua = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.111 Safari/537.36' |
| 76 |
kw['user_agent'] = self.non_ie_ua |
| 77 |
br = BasicNewsRecipe.get_browser(self, *a, **kw) |
| 78 |
return br |
| 79 |
|
| 80 |
def preprocess_html(self, soup): |
| 81 |
for img in soup.findAll('amp-img'): |
| 82 |
img.name = 'img' |
| 83 |
img['srcset'] = '' |
| 84 |
|
| 85 |
for div in soup.findAll(attrs={'class': 'full-gallery'}): |
| 86 |
imgs = {} |
| 87 |
for li in div.findAll('li', attrs={'data-gallery-item': True, 'data-original': True}): |
| 88 |
imgs[li['data-gallery-item']] = li['data-original'] |
| 89 |
li.extract() |
| 90 |
for li in div.findAll('li', attrs={'data-gallery-legend': True}): |
| 91 |
src = imgs.get(li['data-gallery-legend']) |
| 92 |
if src is not None: |
| 93 |
img = new_tag(soup, 'img') |
| 94 |
img['src'] = src |
| 95 |
img['style'] = 'display:block' |
| 96 |
li.append(img) |
| 97 |
return soup |
| 98 |
|
| 99 |
feeds = [ |
| 100 |
('News - UK', 'http://www.independent.co.uk/news/uk/rss'), |
| 101 |
('News - World', 'http://www.independent.co.uk/news/world/rss'), |
| 102 |
('News - Business', 'http://www.independent.co.uk/news/business/rss'), |
| 103 |
('News - People', 'http://www.independent.co.uk/news/people/rss'), |
| 104 |
('News - Science', 'http://www.independent.co.uk/news/science/rss'), |
| 105 |
('News - Media', 'http://www.independent.co.uk/news/media/rss'), |
| 106 |
('News - Education', 'http://www.independent.co.uk/news/education/rss'), |
| 107 |
('News - Obituaries', 'http://www.independent.co.uk/news/obituaries/rss'), |
| 108 |
('News - Corrections', 'http://www.independent.co.uk/news/corrections/rss'), |
| 109 |
('Voices', 'http://www.independent.co.uk/voices/rss'), |
| 110 |
('Environment', 'http://www.independent.co.uk/environment/rss'), |
| 111 |
('Sport - Athletics', 'http://www.independent.co.uk/sport/general/athletics/rss'), |
| 112 |
('Sport - Cricket', 'http://www.independent.co.uk/sport/cricket/rss'), |
| 113 |
('Sport - Football', 'http://www.independent.co.uk/sport/football/rss'), |
| 114 |
('Sport - Golf', 'http://www.independent.co.uk/sport/golf/rss'), |
| 115 |
('Sport - Motor racing', 'http://www.independent.co.uk/sport/motor-racing/rss'), |
| 116 |
('Sport - Olympics', 'http://www.independent.co.uk/sport/olympics/rss'), |
| 117 |
('Sport - Racing', 'http://www.independent.co.uk/sport/racing/rss'), |
| 118 |
('Sport - Rugby League', 'http://www.independent.co.uk/sport/general/rugby-league/rss'), |
| 119 |
('Sport - Rugby Union', 'http://www.independent.co.uk/sport/rugby/rugby-union/rss'), |
| 120 |
('Sport - Sailing', 'http://www.independent.co.uk/sport/general/sailing/rss'), |
| 121 |
('Sport - Tennis', 'http://www.independent.co.uk/sport/tennis/rss'), |
| 122 |
('Sport - Others', 'http://www.independent.co.uk/sport/general/others/rss'), |
| 123 |
('Life & Style - Fashion', 'http://www.independent.co.uk/life-style/fashion/rss'), |
| 124 |
('Life & Style -Food & Drink', 'http://www.independent.co.uk/life-style/food-and-drink/rss'), |
| 125 |
('Life & Style - Health and Families', 'http://www.independent.co.uk/life-style/health-and-families/rss'), |
| 126 |
('Life & Style - History', 'http://www.independent.co.uk/life-style/history/rss'), |
| 127 |
('Life & Style - Gadgets & Tech', 'http://www.independent.co.uk/life-style/gadgets-and-tech/rss'), |
| 128 |
('Life & Style - Motoring', 'http://www.independent.co.uk/life-style/motoring/rss'), |
| 129 |
('Arts & Ents - Art', 'http://www.independent.co.uk/arts-entertainment/art/rss'), |
| 130 |
('Arts & Ents - Architecture', 'http://www.independent.co.uk/arts-entertainment/architecture/rss'), |
| 131 |
('Arts & Ents - Music', 'http://www.independent.co.uk/arts-entertainment/music/rss'), |
| 132 |
('Arts & Ents - Classical', 'http://www.independent.co.uk/arts-entertainment/classical/rss'), |
| 133 |
('Arts & Ents - Films', 'http://www.independent.co.uk/arts-entertainment/films/rss'), |
| 134 |
('Arts & Ents - TV', 'http://www.independent.co.uk/arts-entertainment/tv/rss'), |
| 135 |
('Arts & Ents - Theatre and Dance', 'http://www.independent.co.uk/arts-entertainment/theatre-dance/rss'), |
| 136 |
('Arts & Ents - Comedy', 'http://www.independent.co.uk/arts-entertainment/comedy/rss'), |
| 137 |
('Arts & Ents - Books', 'http://www.independent.co.uk/arts-entertainment/books/rss'), |
| 138 |
('Travel', 'http://www.independent.co.uk/travel/rss'), |
| 139 |
('Money', 'http://www.independent.co.uk/money/rss'), |
| 140 |
('IndyBest', 'http://www.independent.co.uk/extras/indybest/rss'), |
| 141 |
] |