Debian Codegraph @

recipes/vancouver_sun.recipe

raw
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3 from __future__ import print_function
4
5 __license__ = 'GPL v3'
6
7 '''
8 www.canada.com
9 '''
10 import re
11
12 from calibre.ebooks.BeautifulSoup import Tag
13 from calibre.web.feeds.recipes import BasicNewsRecipe
14
15
16 def new_tag(soup, name, attrs=()):
17 impl = getattr(soup, 'new_tag', None)
18 if impl is not None:
19 return impl(name, attrs=dict(attrs))
20 return Tag(soup, name, attrs=attrs or None)
21
22
23 class CanWestPaper(BasicNewsRecipe):
24 compress_news_images = True
25 postmedia_index_pages = [
26 ('Headlines', '/index.html'),
27 ('Ottawa & Area', '/news/ottawa/index.html'),
28 ('Vancouver', '/news/vancouver/index.html'),
29 ('Calgary', '/news/calgary/index.html'),
30 ('Edmonton', '/news/edmonton/index.html'),
31 ('Montreal', '/news/montreal/index.html'),
32 ('Fraser Valley', '/news/fraser-valley/index.html'),
33 ('British Columbia', '/news/bc/index.html'),
34 ('Alberta', '/news/alberta/index.html'),
35 ('Canada', '/news/canada/index.html'),
36 ('National', '/news/national/index.html'),
37 ('Politics', '/news/politics/index.html'),
38 ('Insight', '/news/insight/index.html'),
39 ('Special Reports', '/news/specialreports/index.html'),
40 ('Gangs', '/news/gangs/index.html'),
41 ('Education', '/news/education/index.html'),
42 ('Health', '/news/health/index.html'),
43 ('Environment', '/news/environment/index.html'),
44 ('World', '/news/world/index.html'),
45 ('Police Blotter', '/news/crime-and-justice/index.html'),
46 ('Crime', '/news/blotter/index.html'),
47 ('Around Town', '/news/topic.html?t=keyword&q=Around+Town'),
48 ('Diplomatica', '/news/diplomatica/index.html'),
49 ('Opinion', '/opinion/index.html'),
50 ('Columnists', '/columnists/index.html'),
51 ('Editorials', '/opinion/editorials/index.html'),
52 ('Letters', '/opinion/letters/index.html'),
53 ('Business', '/business/index.html'),
54 ('Sports', '/sports/index.html'),
55 ('Arts', '/entertainment/index.html'),
56 ('Life', '/life/index.html'),
57 ('Technology', '/technology/index.html'),
58 ('Travel', '/travel/index.html'),
59 ('Health', '/health/index.html'),
60 ]
61
62 # un-comment the following six lines for the Vancouver Province
63 # # title = u'Vancouver Province'
64 # # url_prefix = 'http://www.theprovince.com'
65 # # description = u'News from Vancouver, BC'
66 # # std_logo_url = 'http://www.theprovince.com/images/logo_theprovince.jpg'
67 # # logo_url = 'vplogo.jpg'
68 # # fp_tag = 'CAN_TP'
69
70 # un-comment the following six lines for the Vancouver Sun
71 title = 'Vancouver Sun'
72 url_prefix = 'http://www.vancouversun.com'
73 description = 'News from Vancouver, BC'
74 std_logo_url = 'http://www.vancouversun.com/images/logo_vancouversun.jpg'
75 logo_url = 'vslogo.jpg'
76 fp_tag = 'CAN_VS'
77
78 # un-comment the following six lines for the Calgary Herald
79 # # title = u'Calgary Herald'
80 # # url_prefix = 'http://www.calgaryherald.com'
81 # # description = u'News from Calgary, AB'
82 # # std_logo_url = 'http://www.calgaryherald.com/images/logo_calgaryherald.jpg'
83 # # logo_url = 'chlogo.jpg'
84 # # fp_tag = 'CAN_CH'
85
86 # un-comment the following six lines for the Edmonton Journal
87 # # title = u'Edmonton Journal'
88 # # url_prefix = 'http://www.edmontonjournal.com'
89 # # description = u'News from Edmonton, AB'
90 # # std_logo_url = 'http://www.edmontonjournal.com/images/logo_edmontonjournal.jpg'
91 # # logo_url = 'ejlogo.jpg'
92 # # fp_tag = 'CAN_EJ'
93
94 # un-comment the following six lines for the Ottawa Citizen
95 # # title = u'Ottawa Citizen'
96 # # url_prefix = 'http://www.ottawacitizen.com'
97 # # description = u'News from Ottawa, ON'
98 # # std_logo_url = 'http://www.ottawacitizen.com/images/logo_ottawacitizen.jpg'
99 # # logo_url = 'oclogo.jpg'
100 # # fp_tag = 'CAN_OC'
101
102 # un-comment the following six lines for the Montreal Gazette
103 # # title = u'Montreal Gazette'
104 # # url_prefix = 'http://www.montrealgazette.com'
105 # # description = u'News from Montreal, QC'
106 # # std_logo_url = 'http://www.montrealgazette.com/images/logo_montrealgazette.jpg'
107 # # logo_url = 'mglogo.jpg'
108 # # fp_tag = 'CAN_MG'
109
110 Kindle_Fire = False
111 masthead_url = std_logo_url
112
113 url_list = []
114 language = 'en_CA'
115 __author__ = 'Nick Redding'
116 no_stylesheets = True
117 timefmt = ' [%b %d]'
118 encoding = 'utf-8'
119 extra_css = '''
120 .timestamp { font-size:xx-small; display: block; }
121 #storyheader { font-size: medium; }
122 #storyheader h1 { font-size: x-large; }
123 #storyheader h2 { font-size: small; font-style: italic; }
124 .byline { font-size:xx-small; }
125 #photocaption { font-size: xx-small; font-style: italic; font-weight: normal; }
126 .photocaption { font-size: xx-small; font-style: italic; font-weight: normal; }
127 #photocredit { font-size: xx-small; font-weight: normal; }'''
128
129 keep_only_tags = [dict(name='div', attrs={'id': re.compile(r'story')})]
130
131 remove_tags = [
132 {'class': 'comments'},
133 {'class': 'comment-intro'},
134 {'class': 'storytab'},
135 dict(name='div', attrs={'class': 'section_title'}),
136 dict(name='div', attrs={'class': 'sharebar'}),
137 dict(name='div', attrs={'class': 'navbar'}),
138 dict(name='div', attrs={'class': 'morelinks'}),
139 dict(name='h2', attrs={'id': 'photocredit'}),
140 dict(name='div', attrs={'class': 'viewmore'}),
141 dict(name='li', attrs={'class': 'email'}),
142 dict(name='div', attrs={'class': 'story_tool_hr'}),
143 dict(name='div', attrs={'class': 'clear'}),
144 dict(name='div', attrs={'class': 'story_tool'}),
145 dict(name='div', attrs={'class': 'copyright'}),
146 dict(name='div', attrs={'class': 'rule_grey_solid'}),
147 dict(name='div', attrs={'id': 'soundoff'}),
148 dict(name='div', attrs={'id': re.compile(r'flyer')}),
149 dict(name='li', attrs={'class': 'print'}),
150 dict(name='li', attrs={'class': 'share'}),
151 dict(name='ul', attrs={'class': 'bullet'}),
152 ]
153
154 def get_cover_url(self):
155 from datetime import date, timedelta
156
157 cover = 'http://webmedia.newseum.org/newseum-multimedia/dfp/jpg' + str(date.today().day) + '/lg/' + self.fp_tag + '.jpg'
158 br = BasicNewsRecipe.get_browser(self)
159 daysback = 1
160 try:
161 br.open(cover)
162 except Exception:
163 while daysback < 7:
164 cover = (
165 'http://webmedia.newseum.org/newseum-multimedia/dfp/jpg'
166 + str((date.today() - timedelta(days=daysback)).day)
167 + '/lg/'
168 + self.fp_tag
169 + '.jpg'
170 )
171 br = BasicNewsRecipe.get_browser(self)
172 try:
173 br.open(cover)
174 except Exception:
175 daysback = daysback + 1
176 continue
177 break
178 if daysback == 7:
179 self.log('\nCover unavailable')
180 cover = None
181 return cover
182
183 def fixChars(self, string):
184 # Replace lsquo (\x91)
185 fixed = string.replace('\x91', '‘')
186 # Replace rsquo (\x92)
187 fixed = fixed.replace('\x92', '’')
188 # Replace ldquo (\x93)
189 fixed = fixed.replace('\x93', '“')
190 # Replace rdquo (\x94)
191 fixed = fixed.replace('\x94', '”')
192 # Replace ndash (\x96)
193 fixed = fixed.replace('\x96', '–')
194 # Replace mdash (\x97)
195 fixed = fixed.replace('\x97', '—')
196 fixed = fixed.replace('&#x2019;', '’')
197 return fixed
198
199 def massageNCXText(self, description):
200 return description
201
202 def populate_article_metadata(self, article, soup, first):
203 if first:
204 picdiv = soup.find('body').find('img')
205 if picdiv is not None:
206 self.add_toc_thumbnail(article, re.sub(r'links\\link\d+\\', '', picdiv['src']))
207 xtitle = article.text_summary.strip()
208 if len(xtitle) == 0:
209 desc = soup.find('meta', attrs={'property': 'og:description'})
210 if desc is not None:
211 article.summary = article.text_summary = desc['content']
212
213 def strip_anchors(self, soup):
214 paras = soup.findAll(True)
215 for para in paras:
216 aTags = para.findAll('a')
217 for a in aTags:
218 if a.img is None:
219 a.replaceWith(a.renderContents().decode('cp1252', 'replace'))
220 return soup
221
222 def preprocess_html(self, soup):
223 # delete empty id attributes--they screw up the TOC for unknown reasons
224 divtags = soup.findAll('div', attrs={'id': ''})
225 if divtags:
226 for div in divtags:
227 del div['id']
228
229 pgall = soup.find('div', attrs={'id': 'storyphoto'})
230 if pgall is not None: # photo gallery perhaps
231 if soup.find('div', attrs={'id': 'storycontent'}) is None:
232 allpics = new_tag(soup, 'div')
233 first_img = pgall.find('div', 'storyimage')
234 if first_img is not None:
235 first_img.extract()
236 tlist = pgall.find('div', attrs={'id': 'relatedthumbs'})
237 if tlist is not None:
238 for atag in tlist.findAll('a'):
239 img = new_tag(soup, 'img')
240 srcpre, sep, srcpost = atag.img['src'].partition('?')
241 img['src'] = srcpre
242 pdesc = new_tag(soup, 'p')
243 pdesc.insert(0, atag.img['alt'])
244 pdesc['class'] = 'photocaption'
245 div = new_tag(soup, 'div')
246 div.insert(0, pdesc)
247 div.insert(0, img)
248 allpics.append(div)
249 pgall.replaceWith(allpics)
250
251 for pg in soup.findAll('div', attrs={'id': 'storyphoto'}):
252 pg.extract()
253 return self.strip_anchors(soup)
254
255 def parse_index(self):
256
257 articles = {}
258 ans = []
259
260 def handle_article(adiv, key):
261 if adiv.name in {'h1', 'h3'}:
262 h1tag = adiv
263 else:
264 h1tag = adiv.h1
265 if h1tag is None:
266 h1tag = adiv.h3
267 if h1tag is not None:
268 atag = h1tag.a
269 if atag is not None:
270 url = atag['href']
271 if url.startswith('/'):
272 url = self.url_prefix + url
273 if not url.startswith(self.url_prefix):
274 print('Rejected ' + url)
275 return
276 if url in self.url_list:
277 print('Rejected dup ' + url)
278 return
279 self.url_list.append(url)
280 title = self.tag_to_string(atag, False)
281 if 'VIDEO' in title.upper():
282 return
283 if 'GALLERY' in title.upper():
284 return
285 if 'PHOTOS' in title.upper():
286 return
287 dtag = adiv.find('div', 'content')
288 description = ''
289 print('URL ' + url)
290 print('TITLE ' + title)
291 if dtag is not None:
292 stag = dtag.span
293 if stag is not None:
294 if ''.join(stag['class']) != 'timestamp':
295 description = self.tag_to_string(stag, False)
296 else:
297 description = self.tag_to_string(dtag, False)
298 print('DESCRIPTION: ' + description)
299 if key not in articles:
300 articles[key] = []
301 articles[key].append(dict(title=title, url=url, date='', description=description, author='', content=''))
302
303 def parse_web_index(key, keyurl):
304 print('Section: ' + key + ': ' + self.url_prefix + keyurl)
305 try:
306 soup = self.index_to_soup(self.url_prefix + keyurl)
307 except Exception:
308 print('Section: ' + key + ' NOT FOUND')
309 return
310 ans.append(key)
311 mainsoup = soup.find('div', 'bodywrapper')
312 footer = mainsoup.find(attrs={'id': 'footerfeature'})
313 if footer is not None:
314 footer.extract()
315 for wdiv in mainsoup.findAll(attrs={'class': ['genericfeature']}):
316 wdiv.extract()
317 for wdiv in mainsoup.findAll(attrs={'class': ['headline', 'featurecontent']}):
318 handle_article(wdiv, key)
319
320 for k, url in self.postmedia_index_pages:
321 parse_web_index(k, url)
322 ans = [(key, articles[key]) for key in ans if key in articles]
323 return ans