Debian Codegraph @

recipes/chr_mon.recipe

raw
1 __license__ = 'GPL v3'
2 __copyright__ = '2012, Darko Miletic <darko.miletic at gmail.com>'
3 '''
4 www.csmonitor.com
5 '''
6
7 import re
8
9 from calibre.web.feeds.news import BasicNewsRecipe
10
11
12 class CSMonitor(BasicNewsRecipe):
13 title = 'The Christian Science Monitor - daily'
14 __author__ = 'Darko Miletic'
15 description = 'The Christian Science Monitor is an international news organization that delivers thoughtful, global coverage via its website, weekly magazine, daily news briefing, and email newsletters.' # noqa: E501
16 publisher = 'The Christian Science Monitor'
17 category = 'news, politics, USA'
18 oldest_article = 2
19 max_articles_per_feed = 200
20 no_stylesheets = True
21 encoding = 'utf8'
22 use_embedded_content = False
23 language = 'en'
24 remove_empty_feeds = True
25 publication_type = 'newspaper'
26 masthead_url = 'http://www.csmonitor.com/extension/csm_base/design/csm_design/images/csmlogo_179x46.gif'
27 extra_css = '''
28 body{font-family: Arial,Tahoma,Verdana,Helvetica,sans-serif }
29 img{margin-bottom: 0.4em; display:block}
30 .head {font-family: Georgia,"Times New Roman",Times,serif}
31 .sByline,.caption{font-size: x-small}
32 .hide{display: none}
33 .sLoc{font-weight: bold}
34 ul{list-style-type: none}
35 '''
36
37 conversion_options = {'comment': description, 'tags': category, 'publisher': publisher, 'language': language}
38
39 remove_tags = [
40 dict(name=['meta', 'link', 'iframe', 'object', 'embed']),
41 dict(attrs={'class': re.compile(r'(^|| )podStoryRel($|| )', re.DOTALL)}),
42 dict(attrs={'class': ['bottom-rel', 'hide']}),
43 dict(attrs={'id': ['pgallerycarousel_enlarge', 'pgallerycarousel_related']}),
44 ]
45 keep_only_tags = [
46 dict(name='h1', attrs={'class': 'head'}),
47 dict(name='h2', attrs={'class': 'subhead'}),
48 dict(attrs={'class': ['sByline', 'thePhoto', 'ui-body-header']}),
49 dict(attrs={'class': re.compile(r'(^|| )sBody($|| )', re.DOTALL)}),
50 ]
51 remove_attributes = ['xmlns:fb']
52
53 feeds = [
54 ('USA', 'http://rss.csmonitor.com/feeds/usa'),
55 ('World', 'http://rss.csmonitor.com/feeds/world'),
56 ('Politics', 'http://rss.csmonitor.com/feeds/politics'),
57 ('Business', 'http://rss.csmonitor.com/feeds/wam'),
58 ('Commentary', 'http://rss.csmonitor.com/feeds/commentary'),
59 ('Books', 'http://rss.csmonitor.com/feeds/books'),
60 ('Arts', 'http://rss.csmonitor.com/feeds/arts'),
61 ('Environment', 'http://rss.csmonitor.com/feeds/environment'),
62 ('Innovation', 'http://rss.csmonitor.com/feeds/scitech'),
63 ('Living', 'http://rss.csmonitor.com/feeds/living'),
64 ('Science', 'http://rss.csmonitor.com/feeds/science'),
65 ('The Culture', 'http://rss.csmonitor.com/feeds/theculture'),
66 ('The Home Forum', 'http://rss.csmonitor.com/feeds/homeforum'),
67 ('Articles', 'http://rss.csmonitor.com/feeds/csarticles'),
68 ]
69
70 def append_page(self, soup):
71 pager = soup.find('div', attrs={'class': 'navigation'})
72 if pager:
73 nexttag = pager.find(attrs={'id': 'next-button'})
74 if nexttag:
75 nurl = 'http://www.csmonitor.com' + nexttag['href']
76 soup2 = self.index_to_soup(nurl)
77 texttag = soup2.find(attrs={'class': re.compile(r'(^|| )sBody($|| )', re.DOTALL)})
78 if texttag:
79 appendtag = soup.find(attrs={'class': re.compile(r'(^|| )sBody($|| )', re.DOTALL)})
80 for citem in texttag.findAll(attrs={'class': [re.compile(r'(^|| )podStoryRel($|| )', re.DOTALL), 'bottom-rel', 'hide']}):
81 citem.extract()
82 self.append_page(soup2)
83 texttag.extract()
84 pager.extract()
85 appendtag.append(texttag)
86
87 def preprocess_html(self, soup):
88 self.append_page(soup)
89 pager = soup.find('div', attrs={'class': 'navigation'})
90 if pager:
91 pager.extract()
92 for item in soup.findAll('a'):
93 limg = item.find('img')
94 if item.string is not None:
95 str = item.string
96 item.replaceWith(str)
97 elif limg:
98 item.name = 'div'
99 item.attrs = []
100 else:
101 str = self.tag_to_string(item)
102 item.replaceWith(str)
103 for item in soup.findAll('img', src=True):
104 if 'scorecardresearch' in item['src']:
105 item.extract()
106 return soup