| 1 |
#!/usr/bin/env python |
| 2 |
# -*- coding: utf-8 -*- |
| 3 |
from __future__ import print_function |
| 4 |
|
| 5 |
__license__ = 'GPL v3' |
| 6 |
|
| 7 |
''' |
| 8 |
www.canada.com |
| 9 |
''' |
| 10 |
import re |
| 11 |
|
| 12 |
from calibre.ebooks.BeautifulSoup import Tag |
| 13 |
from calibre.web.feeds.recipes import BasicNewsRecipe |
| 14 |
|
| 15 |
|
| 16 |
def new_tag(soup, name, attrs=()): |
| 17 |
impl = getattr(soup, 'new_tag', None) |
| 18 |
if impl is not None: |
| 19 |
return impl(name, attrs=dict(attrs)) |
| 20 |
return Tag(soup, name, attrs=attrs or None) |
| 21 |
|
| 22 |
|
| 23 |
class CanWestPaper(BasicNewsRecipe): |
| 24 |
postmedia_index_pages = [ |
| 25 |
('Headlines', '/index.html'), |
| 26 |
('Ottawa & Area', '/news/ottawa/index.html'), |
| 27 |
('Vancouver', '/news/vancouver/index.html'), |
| 28 |
('Calgary', '/news/calgary/index.html'), |
| 29 |
('Edmonton', '/news/edmonton/index.html'), |
| 30 |
('Montreal', '/news/montreal/index.html'), |
| 31 |
('Fraser Valley', '/news/fraser-valley/index.html'), |
| 32 |
('British Columbia', '/news/bc/index.html'), |
| 33 |
('Alberta', '/news/alberta/index.html'), |
| 34 |
('Canada', '/news/canada/index.html'), |
| 35 |
('National', '/news/national/index.html'), |
| 36 |
('Politics', '/news/politics/index.html'), |
| 37 |
('Insight', '/news/insight/index.html'), |
| 38 |
('Special Reports', '/news/specialreports/index.html'), |
| 39 |
('Gangs', '/news/gangs/index.html'), |
| 40 |
('Education', '/news/education/index.html'), |
| 41 |
('Health', '/news/health/index.html'), |
| 42 |
('Environment', '/news/environment/index.html'), |
| 43 |
('World', '/news/world/index.html'), |
| 44 |
('Police Blotter', '/news/crime-and-justice/index.html'), |
| 45 |
('Crime', '/news/blotter/index.html'), |
| 46 |
('Around Town', '/news/topic.html?t=keyword&q=Around+Town'), |
| 47 |
('Diplomatica', '/news/diplomatica/index.html'), |
| 48 |
('Opinion', '/opinion/index.html'), |
| 49 |
('Columnists', '/columnists/index.html'), |
| 50 |
('Editorials', '/opinion/editorials/index.html'), |
| 51 |
('Letters', '/opinion/letters/index.html'), |
| 52 |
('Business', '/business/index.html'), |
| 53 |
('Sports', '/sports/index.html'), |
| 54 |
('Arts', '/entertainment/index.html'), |
| 55 |
('Life', '/life/index.html'), |
| 56 |
('Technology', '/technology/index.html'), |
| 57 |
('Travel', '/travel/index.html'), |
| 58 |
('Health', '/health/index.html'), |
| 59 |
] |
| 60 |
|
| 61 |
# un-comment the following six lines for the Vancouver Province |
| 62 |
# # title = u'Vancouver Province' |
| 63 |
# # url_prefix = 'http://www.theprovince.com' |
| 64 |
# # description = u'News from Vancouver, BC' |
| 65 |
# # std_logo_url = 'http://www.theprovince.com/images/logo_theprovince.jpg' |
| 66 |
# # logo_url = 'vplogo.jpg' |
| 67 |
# # fp_tag = 'CAN_TP' |
| 68 |
|
| 69 |
# un-comment the following six lines for the Vancouver Sun |
| 70 |
# # title = u'Vancouver Sun' |
| 71 |
# # url_prefix = 'http://www.vancouversun.com' |
| 72 |
# # description = u'News from Vancouver, BC' |
| 73 |
# # std_logo_url = 'http://www.vancouversun.com/images/logo_vancouversun.jpg' |
| 74 |
# # logo_url = 'vslogo.jpg' |
| 75 |
# # fp_tag = 'CAN_VS' |
| 76 |
|
| 77 |
# un-comment the following six lines for the Calgary Herald |
| 78 |
# # title = u'Calgary Herald' |
| 79 |
# # url_prefix = 'http://www.calgaryherald.com' |
| 80 |
# # description = u'News from Calgary, AB' |
| 81 |
# # std_logo_url = 'http://www.calgaryherald.com/images/logo_calgaryherald.jpg' |
| 82 |
# # logo_url = 'chlogo.jpg' |
| 83 |
# # fp_tag = 'CAN_CH' |
| 84 |
|
| 85 |
# un-comment the following six lines for the Edmonton Journal |
| 86 |
# # title = u'Edmonton Journal' |
| 87 |
# # url_prefix = 'http://www.edmontonjournal.com' |
| 88 |
# # description = u'News from Edmonton, AB' |
| 89 |
# # std_logo_url = 'http://www.edmontonjournal.com/images/logo_edmontonjournal.jpg' |
| 90 |
# # logo_url = 'ejlogo.jpg' |
| 91 |
# # fp_tag = 'CAN_EJ' |
| 92 |
|
| 93 |
# un-comment the following six lines for the Ottawa Citizen |
| 94 |
title = 'Ottawa Citizen' |
| 95 |
url_prefix = 'http://www.ottawacitizen.com' |
| 96 |
description = 'News from Ottawa, ON' |
| 97 |
std_logo_url = 'http://www.ottawacitizen.com/images/logo_ottawacitizen.jpg' |
| 98 |
logo_url = 'oclogo.jpg' |
| 99 |
fp_tag = 'CAN_OC' |
| 100 |
|
| 101 |
# un-comment the following six lines for the Montreal Gazette |
| 102 |
# # title = u'Montreal Gazette' |
| 103 |
# # url_prefix = 'http://www.montrealgazette.com' |
| 104 |
# # description = u'News from Montreal, QC' |
| 105 |
# # std_logo_url = 'http://www.montrealgazette.com/images/logo_montrealgazette.jpg' |
| 106 |
# # logo_url = 'mglogo.jpg' |
| 107 |
# # fp_tag = 'CAN_MG' |
| 108 |
|
| 109 |
Kindle_Fire = False |
| 110 |
masthead_url = std_logo_url |
| 111 |
|
| 112 |
url_list = [] |
| 113 |
language = 'en_CA' |
| 114 |
__author__ = 'Nick Redding' |
| 115 |
no_stylesheets = True |
| 116 |
timefmt = ' [%b %d]' |
| 117 |
encoding = 'utf-8' |
| 118 |
extra_css = ''' |
| 119 |
.timestamp { font-size:xx-small; display: block; } |
| 120 |
#storyheader { font-size: medium; } |
| 121 |
#storyheader h1 { font-size: x-large; } |
| 122 |
#storyheader h2 { font-size: small; font-style: italic; } |
| 123 |
.byline { font-size:xx-small; } |
| 124 |
#photocaption { font-size: xx-small; font-style: italic; font-weight: normal; } |
| 125 |
.photocaption { font-size: xx-small; font-style: italic; font-weight: normal; } |
| 126 |
#photocredit { font-size: xx-small; font-weight: normal; }''' |
| 127 |
|
| 128 |
keep_only_tags = [dict(name='div', attrs={'id': re.compile(r'story')})] |
| 129 |
|
| 130 |
remove_tags = [ |
| 131 |
{'class': 'comments'}, |
| 132 |
{'class': 'comment-intro'}, |
| 133 |
{'class': 'storytab'}, |
| 134 |
dict(name='div', attrs={'class': 'section_title'}), |
| 135 |
dict(name='div', attrs={'class': 'sharebar'}), |
| 136 |
dict(name='div', attrs={'class': 'navbar'}), |
| 137 |
dict(name='div', attrs={'class': 'morelinks'}), |
| 138 |
dict(name='h2', attrs={'id': 'photocredit'}), |
| 139 |
dict(name='div', attrs={'class': 'viewmore'}), |
| 140 |
dict(name='li', attrs={'class': 'email'}), |
| 141 |
dict(name='div', attrs={'class': 'story_tool_hr'}), |
| 142 |
dict(name='div', attrs={'class': 'clear'}), |
| 143 |
dict(name='div', attrs={'class': 'story_tool'}), |
| 144 |
dict(name='div', attrs={'class': 'copyright'}), |
| 145 |
dict(name='div', attrs={'class': 'rule_grey_solid'}), |
| 146 |
dict(name='div', attrs={'id': 'soundoff'}), |
| 147 |
dict(name='div', attrs={'id': re.compile(r'flyer')}), |
| 148 |
dict(name='li', attrs={'class': 'print'}), |
| 149 |
dict(name='li', attrs={'class': 'share'}), |
| 150 |
dict(name='ul', attrs={'class': 'bullet'}), |
| 151 |
] |
| 152 |
|
| 153 |
def get_cover_url(self): |
| 154 |
from datetime import date, timedelta |
| 155 |
|
| 156 |
cover = 'http://webmedia.newseum.org/newseum-multimedia/dfp/jpg' + str(date.today().day) + '/lg/' + self.fp_tag + '.jpg' |
| 157 |
br = BasicNewsRecipe.get_browser(self) |
| 158 |
daysback = 1 |
| 159 |
try: |
| 160 |
br.open(cover) |
| 161 |
except Exception: |
| 162 |
while daysback < 7: |
| 163 |
cover = ( |
| 164 |
'http://webmedia.newseum.org/newseum-multimedia/dfp/jpg' |
| 165 |
+ str((date.today() - timedelta(days=daysback)).day) |
| 166 |
+ '/lg/' |
| 167 |
+ self.fp_tag |
| 168 |
+ '.jpg' |
| 169 |
) |
| 170 |
br = BasicNewsRecipe.get_browser(self) |
| 171 |
try: |
| 172 |
br.open(cover) |
| 173 |
except Exception: |
| 174 |
daysback = daysback + 1 |
| 175 |
continue |
| 176 |
break |
| 177 |
if daysback == 7: |
| 178 |
self.log('\nCover unavailable') |
| 179 |
cover = None |
| 180 |
return cover |
| 181 |
|
| 182 |
def fixChars(self, string): |
| 183 |
# Replace lsquo (\x91) |
| 184 |
fixed = string.replace('\x91', '‘') |
| 185 |
# Replace rsquo (\x92) |
| 186 |
fixed = fixed.replace('\x92', '’') |
| 187 |
# Replace ldquo (\x93) |
| 188 |
fixed = fixed.replace('\x93', '“') |
| 189 |
# Replace rdquo (\x94) |
| 190 |
fixed = fixed.replace('\x94', '”') |
| 191 |
# Replace ndash (\x96) |
| 192 |
fixed = fixed.replace('\x96', '–') |
| 193 |
# Replace mdash (\x97) |
| 194 |
fixed = fixed.replace('\x97', '—') |
| 195 |
fixed = fixed.replace('’', '’') |
| 196 |
return fixed |
| 197 |
|
| 198 |
def massageNCXText(self, description): |
| 199 |
return description |
| 200 |
|
| 201 |
def populate_article_metadata(self, article, soup, first): |
| 202 |
if first: |
| 203 |
picdiv = soup.find('body').find('img') |
| 204 |
if picdiv is not None: |
| 205 |
self.add_toc_thumbnail(article, re.sub(r'links\\link\d+\\', '', picdiv['src'])) |
| 206 |
xtitle = article.text_summary.strip() |
| 207 |
if len(xtitle) == 0: |
| 208 |
desc = soup.find('meta', attrs={'property': 'og:description'}) |
| 209 |
if desc is not None: |
| 210 |
article.summary = article.text_summary = desc['content'] |
| 211 |
|
| 212 |
def strip_anchors(self, soup): |
| 213 |
paras = soup.findAll(True) |
| 214 |
for para in paras: |
| 215 |
aTags = para.findAll('a') |
| 216 |
for a in aTags: |
| 217 |
if a.img is None: |
| 218 |
a.replaceWith(a.renderContents().decode('cp1252', 'replace')) |
| 219 |
return soup |
| 220 |
|
| 221 |
def preprocess_html(self, soup): |
| 222 |
# delete empty id attributes--they screw up the TOC for unknown reasons |
| 223 |
divtags = soup.findAll('div', attrs={'id': ''}) |
| 224 |
if divtags: |
| 225 |
for div in divtags: |
| 226 |
del div['id'] |
| 227 |
|
| 228 |
pgall = soup.find('div', attrs={'id': 'storyphoto'}) |
| 229 |
if pgall is not None: # photo gallery perhaps |
| 230 |
if soup.find('div', attrs={'id': 'storycontent'}) is None: |
| 231 |
allpics = new_tag(soup, 'div') |
| 232 |
first_img = pgall.find('div', 'storyimage') |
| 233 |
if first_img is not None: |
| 234 |
first_img.extract() |
| 235 |
tlist = pgall.find('div', attrs={'id': 'relatedthumbs'}) |
| 236 |
if tlist is not None: |
| 237 |
for atag in tlist.findAll('a'): |
| 238 |
img = new_tag(soup, 'img') |
| 239 |
srcpre, sep, srcpost = atag.img['src'].partition('?') |
| 240 |
img['src'] = srcpre |
| 241 |
pdesc = new_tag(soup, 'p') |
| 242 |
pdesc.insert(0, atag.img['alt']) |
| 243 |
pdesc['class'] = 'photocaption' |
| 244 |
div = new_tag(soup, 'div') |
| 245 |
div.insert(0, pdesc) |
| 246 |
div.insert(0, img) |
| 247 |
allpics.append(div) |
| 248 |
pgall.replaceWith(allpics) |
| 249 |
|
| 250 |
for pg in soup.findAll('div', attrs={'id': 'storyphoto'}): |
| 251 |
pg.extract() |
| 252 |
return self.strip_anchors(soup) |
| 253 |
|
| 254 |
def parse_index(self): |
| 255 |
|
| 256 |
articles = {} |
| 257 |
ans = [] |
| 258 |
|
| 259 |
def handle_article(adiv, key): |
| 260 |
if adiv.name in {'h1', 'h3'}: |
| 261 |
h1tag = adiv |
| 262 |
else: |
| 263 |
h1tag = adiv.h1 |
| 264 |
if h1tag is None: |
| 265 |
h1tag = adiv.h3 |
| 266 |
if h1tag is not None: |
| 267 |
atag = h1tag.a |
| 268 |
if atag is not None: |
| 269 |
url = atag['href'] |
| 270 |
if url.startswith('/'): |
| 271 |
url = self.url_prefix + url |
| 272 |
if not url.startswith(self.url_prefix): |
| 273 |
print('Rejected ' + url) |
| 274 |
return |
| 275 |
if url in self.url_list: |
| 276 |
print('Rejected dup ' + url) |
| 277 |
return |
| 278 |
self.url_list.append(url) |
| 279 |
title = self.tag_to_string(atag, False) |
| 280 |
if 'VIDEO' in title.upper(): |
| 281 |
return |
| 282 |
if 'GALLERY' in title.upper(): |
| 283 |
return |
| 284 |
if 'PHOTOS' in title.upper(): |
| 285 |
return |
| 286 |
dtag = adiv.find('div', 'content') |
| 287 |
description = '' |
| 288 |
print('URL ' + url) |
| 289 |
print('TITLE ' + title) |
| 290 |
if dtag is not None: |
| 291 |
stag = dtag.span |
| 292 |
if stag is not None: |
| 293 |
if ''.join(stag['class']) != 'timestamp': |
| 294 |
description = self.tag_to_string(stag, False) |
| 295 |
else: |
| 296 |
description = self.tag_to_string(dtag, False) |
| 297 |
print('DESCRIPTION: ' + description) |
| 298 |
if key not in articles: |
| 299 |
articles[key] = [] |
| 300 |
articles[key].append(dict(title=title, url=url, date='', description=description, author='', content='')) |
| 301 |
|
| 302 |
def parse_web_index(key, keyurl): |
| 303 |
print('Section: ' + key + ': ' + self.url_prefix + keyurl) |
| 304 |
try: |
| 305 |
soup = self.index_to_soup(self.url_prefix + keyurl) |
| 306 |
except Exception: |
| 307 |
print('Section: ' + key + ' NOT FOUND') |
| 308 |
return |
| 309 |
ans.append(key) |
| 310 |
mainsoup = soup.find('div', 'bodywrapper') |
| 311 |
footer = mainsoup.find(attrs={'id': 'footerfeature'}) |
| 312 |
if footer is not None: |
| 313 |
footer.extract() |
| 314 |
for wdiv in mainsoup.findAll(attrs={'class': ['genericfeature']}): |
| 315 |
wdiv.extract() |
| 316 |
for wdiv in mainsoup.findAll(attrs={'class': ['headline', 'featurecontent']}): |
| 317 |
handle_article(wdiv, key) |
| 318 |
|
| 319 |
for k, url in self.postmedia_index_pages: |
| 320 |
parse_web_index(k, url) |
| 321 |
ans = [(key, articles[key]) for key in ans if key in articles] |
| 322 |
return ans |