Debian Codegraph @

recipes/windsor_star.recipe

raw
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 __license__ = 'GPL v3'
5
6 '''
7 www.canada.com
8 '''
9
10 import re
11
12 from calibre.web.feeds.news import BasicNewsRecipe
13
14
15 class CanWestPaper(BasicNewsRecipe):
16 # un-comment the following four lines for the Victoria Times Colonist
17 # # title = u'Victoria Times Colonist'
18 # # url_prefix = 'http://www.timescolonist.com'
19 # # description = u'News from Victoria, BC'
20 # # fp_tag = 'CAN_TC'
21
22 # un-comment the following four lines for the Vancouver Province
23 # # title = u'Vancouver Province'
24 # # url_prefix = 'http://www.theprovince.com'
25 # # description = u'News from Vancouver, BC'
26 # # fp_tag = 'CAN_VP'
27
28 # un-comment the following four lines for the Vancouver Sun
29 # # title = u'Vancouver Sun'
30 # # url_prefix = 'http://www.vancouversun.com'
31 # # description = u'News from Vancouver, BC'
32 # # fp_tag = 'CAN_VS'
33
34 # un-comment the following four lines for the Edmonton Journal
35 # # title = u'Edmonton Journal'
36 # # url_prefix = 'http://www.edmontonjournal.com'
37 # # description = u'News from Edmonton, AB'
38 # # fp_tag = 'CAN_EJ'
39
40 # un-comment the following four lines for the Calgary Herald
41 # # title = u'Calgary Herald'
42 # # url_prefix = 'http://www.calgaryherald.com'
43 # # description = u'News from Calgary, AB'
44 # # fp_tag = 'CAN_CH'
45
46 # un-comment the following four lines for the Regina Leader-Post
47 # # title = u'Regina Leader-Post'
48 # # url_prefix = 'http://www.leaderpost.com'
49 # # description = u'News from Regina, SK'
50 # # fp_tag = ''
51
52 # un-comment the following four lines for the Saskatoon Star-Phoenix
53 # # title = u'Saskatoon Star-Phoenix'
54 # # url_prefix = 'http://www.thestarphoenix.com'
55 # # description = u'News from Saskatoon, SK'
56 # # fp_tag = ''
57
58 # un-comment the following four lines for the Windsor Star
59 title = 'Windsor Star'
60 url_prefix = 'http://www.windsorstar.com'
61 description = 'News from Windsor, ON'
62 fp_tag = 'CAN_'
63
64 # un-comment the following four lines for the Ottawa Citizen
65 # # title = u'Ottawa Citizen'
66 # # url_prefix = 'http://www.ottawacitizen.com'
67 # # description = u'News from Ottawa, ON'
68 # # fp_tag = 'CAN_OC'
69
70 # un-comment the following four lines for the Montreal Gazette
71 # # title = u'Montreal Gazette'
72 # # url_prefix = 'http://www.montrealgazette.com'
73 # # description = u'News from Montreal, QC'
74 # # fp_tag = 'CAN_MG'
75
76 language = 'en_CA'
77 __author__ = 'Nick Redding'
78 no_stylesheets = True
79 timefmt = ' [%b %d]'
80 extra_css = '''
81 .timestamp { font-size:xx-small; display: block; }
82 #storyheader { font-size: medium; }
83 #storyheader h1 { font-size: x-large; }
84 #storyheader h2 { font-size: large; font-style: italic; }
85 .byline { font-size:xx-small; }
86 #photocaption { font-size: small; font-style: italic }
87 #photocredit { font-size: xx-small; }'''
88 keep_only_tags = [dict(name='div', attrs={'id': 'storyheader'}), dict(name='div', attrs={'id': 'storycontent'})]
89 remove_tags = [
90 {'class': 'comments'},
91 dict(name='div', attrs={'class': 'navbar'}),
92 dict(name='div', attrs={'class': 'morelinks'}),
93 dict(name='div', attrs={'class': 'viewmore'}),
94 dict(name='li', attrs={'class': 'email'}),
95 dict(name='div', attrs={'class': 'story_tool_hr'}),
96 dict(name='div', attrs={'class': 'clear'}),
97 dict(name='div', attrs={'class': 'story_tool'}),
98 dict(name='div', attrs={'class': 'copyright'}),
99 dict(name='div', attrs={'class': 'rule_grey_solid'}),
100 dict(name='li', attrs={'class': 'print'}),
101 dict(name='li', attrs={'class': 'share'}),
102 dict(name='ul', attrs={'class': 'bullet'}),
103 ]
104
105 def get_cover_url(self):
106 from datetime import date, timedelta
107
108 if self.fp_tag == '':
109 return None
110 cover = 'http://webmedia.newseum.org/newseum-multimedia/dfp/jpg' + str(date.today().day) + '/lg/' + self.fp_tag + '.jpg'
111 br = BasicNewsRecipe.get_browser(self)
112 daysback = 1
113 try:
114 br.open(cover)
115 except Exception:
116 while daysback < 7:
117 cover = (
118 'http://webmedia.newseum.org/newseum-multimedia/dfp/jpg'
119 + str((date.today() - timedelta(days=daysback)).day)
120 + '/lg/'
121 + self.fp_tag
122 + '.jpg'
123 )
124 br = BasicNewsRecipe.get_browser(self)
125 try:
126 br.open(cover)
127 except Exception:
128 daysback = daysback + 1
129 continue
130 break
131 if daysback == 7:
132 self.log('\nCover unavailable')
133 cover = None
134 return cover
135
136 def fixChars(self, string):
137 # Replace lsquo (\x91)
138 fixed = string.replace('\x91', '‘')
139 # Replace rsquo (\x92)
140 fixed = fixed.replace('\x92', '’')
141 # Replace ldquo (\x93)
142 fixed = fixed.replace('\x93', '“')
143 # Replace rdquo (\x94)
144 fixed = fixed.replace('\x94', '”')
145 # Replace ndash (\x96)
146 fixed = fixed.replace('\x96', '–')
147 # Replace mdash (\x97)
148 fixed = fixed.replace('\x97', '—')
149 fixed = fixed.replace('&#x2019;', '’')
150 return fixed
151
152 def massageNCXText(self, description):
153 return description
154
155 def populate_article_metadata(self, article, soup, first):
156 if first:
157 picdiv = soup.find('body').find('img')
158 if picdiv is not None:
159 self.add_toc_thumbnail(article, re.sub(r'links\\link\d+\\', '', picdiv['src']))
160 xtitle = article.text_summary.strip()
161 if len(xtitle) == 0:
162 desc = soup.find('meta', attrs={'property': 'og:description'})
163 if desc is not None:
164 article.summary = article.text_summary = desc['content']
165
166 def strip_anchors(self, soup):
167 paras = soup.findAll(True)
168 for para in paras:
169 aTags = para.findAll('a')
170 for a in aTags:
171 if a.img is None:
172 a.replaceWith(a.renderContents().decode('cp1252', 'replace'))
173 return soup
174
175 def preprocess_html(self, soup):
176 return self.strip_anchors(soup)
177
178 def parse_index(self):
179 soup = self.index_to_soup(self.url_prefix + '/news/todays-paper/index.html')
180
181 articles = {}
182 key = 'News'
183 ans = ['News']
184
185 # Find each instance of class="sectiontitle", class="featurecontent"
186 for divtag in soup.findAll('div', attrs={'class': ['section_title02', 'featurecontent']}):
187 # self.log(" div class = %s" % divtag['class'])
188 if ''.join(divtag['class']).startswith('section_title'):
189 # div contains section title
190 if not divtag.h3:
191 continue
192 key = self.tag_to_string(divtag.h3, False)
193 ans.append(key)
194 self.log('Section name %s' % key)
195 continue
196 # div contains article data
197 h1tag = divtag.find('h1')
198 if not h1tag:
199 continue
200 atag = h1tag.find('a', href=True)
201 if not atag:
202 continue
203 url = self.url_prefix + '/news/todays-paper/' + atag['href']
204 # self.log("Section %s" % key)
205 # self.log("url %s" % url)
206 title = self.tag_to_string(atag, False)
207 # self.log("title %s" % title)
208 pubdate = ''
209 description = ''
210 ptag = divtag.find('p')
211 if ptag:
212 description = self.tag_to_string(ptag, False)
213 # self.log("description %s" % description)
214 author = ''
215 autag = divtag.find('h4')
216 if autag:
217 author = self.tag_to_string(autag, False)
218 # self.log("author %s" % author)
219 if key not in articles:
220 articles[key] = []
221 articles[key].append(dict(title=title, url=url, date=pubdate, description=description, author=author, content=''))
222
223 ans = [(keyl, articles[keyl]) for keyl in ans if keyl in articles]
224 return ans