| 1 |
#!/usr/bin/env python |
| 2 |
# -*- coding: utf-8 -*- |
| 3 |
|
| 4 |
__license__ = 'GPL v3' |
| 5 |
|
| 6 |
''' |
| 7 |
www.canada.com |
| 8 |
''' |
| 9 |
|
| 10 |
import re |
| 11 |
|
| 12 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 13 |
|
| 14 |
|
| 15 |
class CanWestPaper(BasicNewsRecipe): |
| 16 |
# un-comment the following four lines for the Victoria Times Colonist |
| 17 |
# # title = u'Victoria Times Colonist' |
| 18 |
# # url_prefix = 'http://www.timescolonist.com' |
| 19 |
# # description = u'News from Victoria, BC' |
| 20 |
# # fp_tag = 'CAN_TC' |
| 21 |
|
| 22 |
# un-comment the following four lines for the Vancouver Province |
| 23 |
# # title = u'Vancouver Province' |
| 24 |
# # url_prefix = 'http://www.theprovince.com' |
| 25 |
# # description = u'News from Vancouver, BC' |
| 26 |
# # fp_tag = 'CAN_VP' |
| 27 |
|
| 28 |
# un-comment the following four lines for the Vancouver Sun |
| 29 |
# # title = u'Vancouver Sun' |
| 30 |
# # url_prefix = 'http://www.vancouversun.com' |
| 31 |
# # description = u'News from Vancouver, BC' |
| 32 |
# # fp_tag = 'CAN_VS' |
| 33 |
|
| 34 |
# un-comment the following four lines for the Edmonton Journal |
| 35 |
# # title = u'Edmonton Journal' |
| 36 |
# # url_prefix = 'http://www.edmontonjournal.com' |
| 37 |
# # description = u'News from Edmonton, AB' |
| 38 |
# # fp_tag = 'CAN_EJ' |
| 39 |
|
| 40 |
# un-comment the following four lines for the Calgary Herald |
| 41 |
# # title = u'Calgary Herald' |
| 42 |
# # url_prefix = 'http://www.calgaryherald.com' |
| 43 |
# # description = u'News from Calgary, AB' |
| 44 |
# # fp_tag = 'CAN_CH' |
| 45 |
|
| 46 |
# un-comment the following four lines for the Regina Leader-Post |
| 47 |
# # title = u'Regina Leader-Post' |
| 48 |
# # url_prefix = 'http://www.leaderpost.com' |
| 49 |
# # description = u'News from Regina, SK' |
| 50 |
# # fp_tag = '' |
| 51 |
|
| 52 |
# un-comment the following four lines for the Saskatoon Star-Phoenix |
| 53 |
# # title = u'Saskatoon Star-Phoenix' |
| 54 |
# # url_prefix = 'http://www.thestarphoenix.com' |
| 55 |
# # description = u'News from Saskatoon, SK' |
| 56 |
# # fp_tag = '' |
| 57 |
|
| 58 |
# un-comment the following four lines for the Windsor Star |
| 59 |
title = 'Windsor Star' |
| 60 |
url_prefix = 'http://www.windsorstar.com' |
| 61 |
description = 'News from Windsor, ON' |
| 62 |
fp_tag = 'CAN_' |
| 63 |
|
| 64 |
# un-comment the following four lines for the Ottawa Citizen |
| 65 |
# # title = u'Ottawa Citizen' |
| 66 |
# # url_prefix = 'http://www.ottawacitizen.com' |
| 67 |
# # description = u'News from Ottawa, ON' |
| 68 |
# # fp_tag = 'CAN_OC' |
| 69 |
|
| 70 |
# un-comment the following four lines for the Montreal Gazette |
| 71 |
# # title = u'Montreal Gazette' |
| 72 |
# # url_prefix = 'http://www.montrealgazette.com' |
| 73 |
# # description = u'News from Montreal, QC' |
| 74 |
# # fp_tag = 'CAN_MG' |
| 75 |
|
| 76 |
language = 'en_CA' |
| 77 |
__author__ = 'Nick Redding' |
| 78 |
no_stylesheets = True |
| 79 |
timefmt = ' [%b %d]' |
| 80 |
extra_css = ''' |
| 81 |
.timestamp { font-size:xx-small; display: block; } |
| 82 |
#storyheader { font-size: medium; } |
| 83 |
#storyheader h1 { font-size: x-large; } |
| 84 |
#storyheader h2 { font-size: large; font-style: italic; } |
| 85 |
.byline { font-size:xx-small; } |
| 86 |
#photocaption { font-size: small; font-style: italic } |
| 87 |
#photocredit { font-size: xx-small; }''' |
| 88 |
keep_only_tags = [dict(name='div', attrs={'id': 'storyheader'}), dict(name='div', attrs={'id': 'storycontent'})] |
| 89 |
remove_tags = [ |
| 90 |
{'class': 'comments'}, |
| 91 |
dict(name='div', attrs={'class': 'navbar'}), |
| 92 |
dict(name='div', attrs={'class': 'morelinks'}), |
| 93 |
dict(name='div', attrs={'class': 'viewmore'}), |
| 94 |
dict(name='li', attrs={'class': 'email'}), |
| 95 |
dict(name='div', attrs={'class': 'story_tool_hr'}), |
| 96 |
dict(name='div', attrs={'class': 'clear'}), |
| 97 |
dict(name='div', attrs={'class': 'story_tool'}), |
| 98 |
dict(name='div', attrs={'class': 'copyright'}), |
| 99 |
dict(name='div', attrs={'class': 'rule_grey_solid'}), |
| 100 |
dict(name='li', attrs={'class': 'print'}), |
| 101 |
dict(name='li', attrs={'class': 'share'}), |
| 102 |
dict(name='ul', attrs={'class': 'bullet'}), |
| 103 |
] |
| 104 |
|
| 105 |
def get_cover_url(self): |
| 106 |
from datetime import date, timedelta |
| 107 |
|
| 108 |
if self.fp_tag == '': |
| 109 |
return None |
| 110 |
cover = 'http://webmedia.newseum.org/newseum-multimedia/dfp/jpg' + str(date.today().day) + '/lg/' + self.fp_tag + '.jpg' |
| 111 |
br = BasicNewsRecipe.get_browser(self) |
| 112 |
daysback = 1 |
| 113 |
try: |
| 114 |
br.open(cover) |
| 115 |
except Exception: |
| 116 |
while daysback < 7: |
| 117 |
cover = ( |
| 118 |
'http://webmedia.newseum.org/newseum-multimedia/dfp/jpg' |
| 119 |
+ str((date.today() - timedelta(days=daysback)).day) |
| 120 |
+ '/lg/' |
| 121 |
+ self.fp_tag |
| 122 |
+ '.jpg' |
| 123 |
) |
| 124 |
br = BasicNewsRecipe.get_browser(self) |
| 125 |
try: |
| 126 |
br.open(cover) |
| 127 |
except Exception: |
| 128 |
daysback = daysback + 1 |
| 129 |
continue |
| 130 |
break |
| 131 |
if daysback == 7: |
| 132 |
self.log('\nCover unavailable') |
| 133 |
cover = None |
| 134 |
return cover |
| 135 |
|
| 136 |
def fixChars(self, string): |
| 137 |
# Replace lsquo (\x91) |
| 138 |
fixed = string.replace('\x91', '‘') |
| 139 |
# Replace rsquo (\x92) |
| 140 |
fixed = fixed.replace('\x92', '’') |
| 141 |
# Replace ldquo (\x93) |
| 142 |
fixed = fixed.replace('\x93', '“') |
| 143 |
# Replace rdquo (\x94) |
| 144 |
fixed = fixed.replace('\x94', '”') |
| 145 |
# Replace ndash (\x96) |
| 146 |
fixed = fixed.replace('\x96', '–') |
| 147 |
# Replace mdash (\x97) |
| 148 |
fixed = fixed.replace('\x97', '—') |
| 149 |
fixed = fixed.replace('’', '’') |
| 150 |
return fixed |
| 151 |
|
| 152 |
def massageNCXText(self, description): |
| 153 |
return description |
| 154 |
|
| 155 |
def populate_article_metadata(self, article, soup, first): |
| 156 |
if first: |
| 157 |
picdiv = soup.find('body').find('img') |
| 158 |
if picdiv is not None: |
| 159 |
self.add_toc_thumbnail(article, re.sub(r'links\\link\d+\\', '', picdiv['src'])) |
| 160 |
xtitle = article.text_summary.strip() |
| 161 |
if len(xtitle) == 0: |
| 162 |
desc = soup.find('meta', attrs={'property': 'og:description'}) |
| 163 |
if desc is not None: |
| 164 |
article.summary = article.text_summary = desc['content'] |
| 165 |
|
| 166 |
def strip_anchors(self, soup): |
| 167 |
paras = soup.findAll(True) |
| 168 |
for para in paras: |
| 169 |
aTags = para.findAll('a') |
| 170 |
for a in aTags: |
| 171 |
if a.img is None: |
| 172 |
a.replaceWith(a.renderContents().decode('cp1252', 'replace')) |
| 173 |
return soup |
| 174 |
|
| 175 |
def preprocess_html(self, soup): |
| 176 |
return self.strip_anchors(soup) |
| 177 |
|
| 178 |
def parse_index(self): |
| 179 |
soup = self.index_to_soup(self.url_prefix + '/news/todays-paper/index.html') |
| 180 |
|
| 181 |
articles = {} |
| 182 |
key = 'News' |
| 183 |
ans = ['News'] |
| 184 |
|
| 185 |
# Find each instance of class="sectiontitle", class="featurecontent" |
| 186 |
for divtag in soup.findAll('div', attrs={'class': ['section_title02', 'featurecontent']}): |
| 187 |
# self.log(" div class = %s" % divtag['class']) |
| 188 |
if ''.join(divtag['class']).startswith('section_title'): |
| 189 |
# div contains section title |
| 190 |
if not divtag.h3: |
| 191 |
continue |
| 192 |
key = self.tag_to_string(divtag.h3, False) |
| 193 |
ans.append(key) |
| 194 |
self.log('Section name %s' % key) |
| 195 |
continue |
| 196 |
# div contains article data |
| 197 |
h1tag = divtag.find('h1') |
| 198 |
if not h1tag: |
| 199 |
continue |
| 200 |
atag = h1tag.find('a', href=True) |
| 201 |
if not atag: |
| 202 |
continue |
| 203 |
url = self.url_prefix + '/news/todays-paper/' + atag['href'] |
| 204 |
# self.log("Section %s" % key) |
| 205 |
# self.log("url %s" % url) |
| 206 |
title = self.tag_to_string(atag, False) |
| 207 |
# self.log("title %s" % title) |
| 208 |
pubdate = '' |
| 209 |
description = '' |
| 210 |
ptag = divtag.find('p') |
| 211 |
if ptag: |
| 212 |
description = self.tag_to_string(ptag, False) |
| 213 |
# self.log("description %s" % description) |
| 214 |
author = '' |
| 215 |
autag = divtag.find('h4') |
| 216 |
if autag: |
| 217 |
author = self.tag_to_string(autag, False) |
| 218 |
# self.log("author %s" % author) |
| 219 |
if key not in articles: |
| 220 |
articles[key] = [] |
| 221 |
articles[key].append(dict(title=title, url=url, date=pubdate, description=description, author=author, content='')) |
| 222 |
|
| 223 |
ans = [(keyl, articles[keyl]) for keyl in ans if keyl in articles] |
| 224 |
return ans |