Debian Codegraph @

recipes/msnbc.recipe

raw
1 __license__ = 'GPL v3'
2 __copyright__ = '2010-2011, Darko Miletic <darko.miletic at gmail.com>'
3 '''
4 msnbc.msn.com
5 '''
6
7 from calibre.web.feeds.recipes import BasicNewsRecipe
8
9
10 def classes(classes):
11 q = frozenset(classes.split(' '))
12 return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)})
13
14
15 class MsNBC(BasicNewsRecipe):
16 title = 'MSNBC'
17 __author__ = 'Darko Miletic'
18 description = 'A Fuller Spectrum of News from msnbc.com and nbcnews.com'
19 oldest_article = 2
20 max_articles_per_feed = 100
21 no_stylesheets = True
22 use_embedded_content = False
23 encoding = 'utf8'
24 publisher = 'msnbc.com'
25 language = 'en_US'
26 remove_empty_feeds = True
27 ignore_duplicate_articles = {'title', 'url'}
28 keep_only_tags = [classes('article-hero__container article-body')]
29 remove_tags = [
30 dict(name=['iframe', 'button', 'meta', 'link']),
31 classes('widget_video ad-container related'),
32 dict(attrs={'data-test': ['social-share-inline']}),
33 dict(name='source'),
34 ]
35
36 feeds = [
37 ('Latest', 'https://feeds.nbcnews.com/msnbc/public/news'),
38 ('Top stories', 'https://feeds.nbcnews.com/nbcnews/public/news'),
39 ('Politics', 'https://feeds.nbcnews.com/nbcnews/public/politics'),
40 ('U.S. News', 'https://feeds.nbcnews.com/nbcnews/public/us-news'),
41 ('World', 'https://feeds.nbcnews.com/nbcnews/public/world'),
42 ('Business', 'https://feeds.nbcnews.com/nbcnews/public/business'),
43 ('Opinion', 'https://feeds.nbcnews.com/nbcnews/public/think'),
44 ]
45
46 def get_article_url(self, article):
47 ans = article.get('guid')
48 if '/video/' not in ans:
49 return ans
50
51 def preprocess_html(self, soup):
52 for img in soup.findAll('img', attrs={'data-original': True}):
53 img['src'] = img['data-original']
54 return soup