Debian Codegraph @

recipes/cnetnews.recipe

raw
1 __license__ = 'GPL v3'
2 __copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
3 '''
4 Changelog:
5 2011-09-24
6 Changed cover (drMerry)
7 2011-10-13
8 Updated Cover (drMerry)
9 2014-03-28
10 Update by Armin Geller
11 news.cnet.com
12 further updated by Bonni Salles
13 My updates use the current rss feeds for CNet and eliminates extraneous areas and
14 sets the oldest_article to 1 as the feed generally has the present day and one day before.
15 You may want to set it to download daily for this reason.
16 '''
17
18 from calibre.web.feeds.news import BasicNewsRecipe
19
20
21 def classes(classes):
22 q = frozenset(classes.split(' '))
23 return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)})
24
25
26 class CnetNews(BasicNewsRecipe):
27 title = 'CNET News'
28 __author__ = 'Kovid Goyal'
29 description = 'Tech news and business reports by CNET News. Focused on information technology, core topics include computers, hardware, software, networking, and Internet media.' # noqa: E501
30 publisher = 'CNET'
31 category = 'news, IT, USA'
32 encoding = 'utf-8'
33 language = 'en'
34 oldest_article = 7
35 max_articles_per_feed = 100
36 compress_news_images = True
37 ignore_duplicate_articles = {'title', 'url'}
38 remove_empty_feeds = True
39 use_embedded_content = False
40 no_stylesheets = True
41 recursions = 1
42
43 remove_tags = [
44 dict(name='div', attrs={'id': 'livefyreContainer'}),
45 dict(section=['tag', 'shortcodeRelatedLinks']),
46 dict(attrs={'class': ['ad-inline-product-carousel-top', 'author-social', 'row controls', 'launchGallery', 'topWrap']}),
47 dict(name='a', attrs={'class': 'subHead', 'data-component': 'imageGalleryModal'}),
48 dict(attrs={'data-component': 'sharebar'}),
49 dict(name=['link', 'meta']),
50 classes('playerControls video share-button'),
51 ]
52
53 keep_only_tags = [
54 dict(name='h1'),
55 dict(section='author'),
56 dict(id=['article-body', 'cnetReview']),
57 dict(attrs={'class': 'deal-content'}),
58 ]
59
60 feeds = [
61 # (u'All of CNET', u'http://www.cnet.com/rss/all/'),
62 ('CNET News', 'http://www.cnet.com/rss/news/'),
63 ('CNET Reviews', 'http://www.cnet.com/rss/reviews/'),
64 ('CNET How To', 'http://www.cnet.com/rss/how-to/'),
65 ('CNET iPhone Update', 'http://www.cnet.com/rss/iphone-update/'),
66 ('CNET Crave', 'http://www.cnet.com/rss/crave/'),
67 ('CNET Car Tech', 'http://www.cnet.com/rss/car-tech/'),
68 ('CNET Android Update', 'http://www.cnet.com/rss/android-update/'),
69 ('CNET Gaming', 'http://www.cnet.com/rss/gaming/'),
70 ('CNET Cheapskate', 'http://www.cnet.com/rss/cheapskate/'),
71 ]
72
73 def get_article_url(self, article):
74 ans = BasicNewsRecipe.get_article_url(self, article)
75 if ans and ('cnet.com/videos/' in ans or 'cnet.com/pictures/' in ans):
76 ans = None
77 return ans
78
79 def is_link_wanted(self, url, tag):
80 return tag.findParent(attrs={'class': 'pageNav'}) is not None
81
82 def preprocess_html(self, soup):
83 for attr in 'data-original data-src'.split():
84 for img in soup.findAll('img', attrs={attr: True}):
85 img['src'] = img[attr].strip()
86 return soup
87
88 def postprocess_html(self, soup, first_fetch):
89 for div in soup.findAll(attrs={'class': 'pagination'}):
90 div.extract()
91 for div in soup.findAll(attrs={'data-update-area': 'pagination'}):
92 div.extract()
93 for h1 in soup.findAll(itemprop='headline'):
94 h1.extract()
95 if first_fetch:
96 soup.find('body').insert(1, h1)
97 for img in soup.findAll('img'):
98 img['height'] = img['width'] = ''
99 return soup