| 1 |
#!/usr/bin/env python |
| 2 |
# -*- coding: utf-8 -*- |
| 3 |
from __future__ import print_function |
| 4 |
|
| 5 |
__license__ = 'GPL v3' |
| 6 |
|
| 7 |
''' |
| 8 |
www.canada.com |
| 9 |
''' |
| 10 |
import re |
| 11 |
|
| 12 |
from calibre.ebooks.BeautifulSoup import Tag |
| 13 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 14 |
|
| 15 |
|
| 16 |
def new_tag(soup, name, attrs=()): |
| 17 |
impl = getattr(soup, 'new_tag', None) |
| 18 |
if impl is not None: |
| 19 |
return impl(name, attrs=dict(attrs)) |
| 20 |
return Tag(soup, name, attrs=attrs or None) |
| 21 |
|
| 22 |
|
| 23 |
class TimesColonist(BasicNewsRecipe): |
| 24 |
# Customization -- remove sections you don't want. |
| 25 |
# If your e-reader is an e-ink Kindle and your output profile is |
| 26 |
# set properly this recipe will not include images because the |
| 27 |
# resulting file is too large. If you have one of these and want |
| 28 |
# images you can set kindle_omit_images = False |
| 29 |
# and remove sections (typically the e-ink Kindles will |
| 30 |
# work with about a dozen of these, but your mileage may vary). |
| 31 |
|
| 32 |
kindle_omit_images = True |
| 33 |
|
| 34 |
section_list = [ |
| 35 |
('', 'Web Front Page'), |
| 36 |
('news/', 'News Headlines'), |
| 37 |
('news/b-c/', 'BC News'), |
| 38 |
('news/national/', 'National News'), |
| 39 |
('news/world/', 'World News'), |
| 40 |
('opinion/', 'Opinion'), |
| 41 |
('opinion/letters/', 'Letters'), |
| 42 |
('business/', 'Business'), |
| 43 |
('business/money/', 'Money'), |
| 44 |
('business/technology/', 'Technology'), |
| 45 |
('business/working/', 'Working'), |
| 46 |
('sports/', 'Sports'), |
| 47 |
('sports/hockey/', 'Hockey'), |
| 48 |
('sports/football/', 'Football'), |
| 49 |
('sports/basketball/', 'Basketball'), |
| 50 |
('sports/golf/', 'Golf'), |
| 51 |
('entertainment/', 'entertainment'), |
| 52 |
('entertainment/go/', 'Go!'), |
| 53 |
('entertainment/music/', 'Music'), |
| 54 |
('entertainment/books/', 'Books'), |
| 55 |
('entertainment/Movies/', 'Movies'), |
| 56 |
('entertainment/television/', 'Television'), |
| 57 |
('life/', 'Life'), |
| 58 |
('life/health/', 'Health'), |
| 59 |
('life/travel/', 'Travel'), |
| 60 |
('life/driving/', 'Driving'), |
| 61 |
('life/homes/', 'Homes'), |
| 62 |
('life/food-drink/', 'Food & Drink'), |
| 63 |
] |
| 64 |
|
| 65 |
title = 'Victoria Times Colonist' |
| 66 |
url_prefix = 'http://www.timescolonist.com' |
| 67 |
description = 'News from Victoria, BC' |
| 68 |
fp_tag = 'CAN_TC' |
| 69 |
|
| 70 |
masthead_url = 'http://www.timescolonist.com/gmg/img/global/logoTimesColonist.png' |
| 71 |
|
| 72 |
url_list = [] |
| 73 |
language = 'en_CA' |
| 74 |
__author__ = 'Nick Redding' |
| 75 |
no_stylesheets = True |
| 76 |
timefmt = ' [%b %d]' |
| 77 |
encoding = 'utf-8' |
| 78 |
extra_css = ''' |
| 79 |
.byline { font-size:xx-small; font-weight: bold;} |
| 80 |
h3 { margin-bottom: 6px; } |
| 81 |
.caption { font-size: xx-small; font-style: italic; font-weight: normal; } |
| 82 |
''' |
| 83 |
keep_only_tags = [dict(name='div', attrs={'class': re.compile(r'main.content')})] |
| 84 |
|
| 85 |
def __init__(self, options, log, progress_reporter): |
| 86 |
self.remove_tags = [ |
| 87 |
{'class': 'comments'}, |
| 88 |
{'id': 'photocredit'}, |
| 89 |
dict(name='div', attrs={'class': re.compile(r'top.controls')}), |
| 90 |
dict(name='div', attrs={'class': re.compile(r'^comments')}), |
| 91 |
dict(name='div', attrs={'class': re.compile(r'social')}), |
| 92 |
dict(name='div', attrs={'class': re.compile(r'tools')}), |
| 93 |
dict(name='div', attrs={'class': re.compile(r'bottom.tools')}), |
| 94 |
dict(name='div', attrs={'class': re.compile(r'window')}), |
| 95 |
dict(name='div', attrs={'class': re.compile(r'related.news.element')}), |
| 96 |
] |
| 97 |
print('PROFILE NAME = ' + options.output_profile.short_name) |
| 98 |
if self.kindle_omit_images and options.output_profile.short_name in ['kindle', 'kindle_dx', 'kindle_pw']: |
| 99 |
self.remove_tags.append(dict(name='div', attrs={'class': re.compile(r'image-container')})) |
| 100 |
BasicNewsRecipe.__init__(self, options, log, progress_reporter) |
| 101 |
|
| 102 |
def get_cover_url(self): |
| 103 |
from datetime import date, timedelta |
| 104 |
|
| 105 |
cover = 'http://webmedia.newseum.org/newseum-multimedia/dfp/jpg' + str(date.today().day) + '/lg/' + self.fp_tag + '.jpg' |
| 106 |
br = BasicNewsRecipe.get_browser(self) |
| 107 |
daysback = 1 |
| 108 |
try: |
| 109 |
br.open(cover) |
| 110 |
except Exception: |
| 111 |
while daysback < 7: |
| 112 |
cover = ( |
| 113 |
'http://webmedia.newseum.org/newseum-multimedia/dfp/jpg' |
| 114 |
+ str((date.today() - timedelta(days=daysback)).day) |
| 115 |
+ '/lg/' |
| 116 |
+ self.fp_tag |
| 117 |
+ '.jpg' |
| 118 |
) |
| 119 |
br = BasicNewsRecipe.get_browser(self) |
| 120 |
try: |
| 121 |
br.open(cover) |
| 122 |
except Exception: |
| 123 |
daysback = daysback + 1 |
| 124 |
continue |
| 125 |
break |
| 126 |
if daysback == 7: |
| 127 |
self.log('\nCover unavailable') |
| 128 |
cover = None |
| 129 |
return cover |
| 130 |
|
| 131 |
def fixChars(self, string): |
| 132 |
# Replace lsquo (\x91) |
| 133 |
fixed = string.replace('\x91', '‘') |
| 134 |
# Replace rsquo (\x92) |
| 135 |
fixed = fixed.replace('\x92', '’') |
| 136 |
# Replace ldquo (\x93) |
| 137 |
fixed = fixed.replace('\x93', '“') |
| 138 |
# Replace rdquo (\x94) |
| 139 |
fixed = fixed.replace('\x94', '”') |
| 140 |
# Replace ndash (\x96) |
| 141 |
fixed = fixed.replace('\x96', '–') |
| 142 |
# Replace mdash (\x97) |
| 143 |
fixed = fixed.replace('\x97', '—') |
| 144 |
fixed = fixed.replace('’', '’') |
| 145 |
return fixed |
| 146 |
|
| 147 |
def massageNCXText(self, description): |
| 148 |
return description |
| 149 |
|
| 150 |
def populate_article_metadata(self, article, soup, first): |
| 151 |
if first: |
| 152 |
picdiv = soup.find('body').find('img') |
| 153 |
if picdiv is not None: |
| 154 |
self.add_toc_thumbnail(article, re.sub(r'links\\link\d+\\', '', picdiv['src'])) |
| 155 |
xtitle = article.text_summary.strip() |
| 156 |
if len(xtitle) == 0: |
| 157 |
desc = soup.find('meta', attrs={'property': 'og:description'}) |
| 158 |
if desc is not None: |
| 159 |
article.summary = article.text_summary = desc['content'] |
| 160 |
|
| 161 |
def strip_anchors(self, soup): |
| 162 |
paras = soup.findAll(True) |
| 163 |
for para in paras: |
| 164 |
aTags = para.findAll('a') |
| 165 |
for a in aTags: |
| 166 |
if a.img is None: |
| 167 |
a.replaceWith(a.renderContents().decode('cp1252', 'replace')) |
| 168 |
return soup |
| 169 |
|
| 170 |
def preprocess_html(self, soup): |
| 171 |
byline = soup.find('p', attrs={'class': re.compile(r'ancillary')}) |
| 172 |
if byline is not None: |
| 173 |
authstr = self.tag_to_string(byline, False) |
| 174 |
authstr = re.sub(r'/ *Times Colonist', '/', authstr, flags=re.IGNORECASE) |
| 175 |
authstr = re.sub(r'BY */', '', authstr, flags=re.IGNORECASE) |
| 176 |
newdiv = new_tag(soup, 'div') |
| 177 |
newdiv.insert(0, authstr) |
| 178 |
newdiv['class'] = 'byline' |
| 179 |
byline.replaceWith(newdiv) |
| 180 |
for caption in soup.findAll('p', attrs={'class': re.compile(r'caption')}): |
| 181 |
capstr = self.tag_to_string(caption, False) |
| 182 |
capstr = re.sub(r'Photograph by.*$', '', capstr, flags=re.IGNORECASE) |
| 183 |
newdiv = new_tag(soup, 'div') |
| 184 |
newdiv.insert(0, capstr) |
| 185 |
newdiv['class'] = 'caption' |
| 186 |
caption.replaceWith(newdiv) |
| 187 |
for ptag in soup.findAll('p'): |
| 188 |
ptext = self.tag_to_string(ptag, use_alt=False, normalize_whitespace=True) |
| 189 |
ptext = re.sub(r'\s+', '', ptext) |
| 190 |
if ptext in {'', ' '}: |
| 191 |
ptag.extract() |
| 192 |
return self.strip_anchors(soup) |
| 193 |
|
| 194 |
raeside = False |
| 195 |
|
| 196 |
def handle_articles(self, htag, article_list, sectitle): |
| 197 |
atag = htag.a |
| 198 |
if atag is not None: |
| 199 |
url = atag['href'] |
| 200 |
url = url.strip() |
| 201 |
# print('Checking >>'+url+'<<\n\r') |
| 202 |
if url.startswith('/'): |
| 203 |
url = self.url_prefix + url |
| 204 |
if url in self.url_list: |
| 205 |
return |
| 206 |
self.url_list.append(url) |
| 207 |
title = self.tag_to_string(atag, False) |
| 208 |
if 'VIDEO' in title.upper(): |
| 209 |
return |
| 210 |
if 'GALLERY' in title.upper(): |
| 211 |
return |
| 212 |
if 'PHOTOS' in title.upper(): |
| 213 |
return |
| 214 |
if 'RAESIDE' in title.upper(): |
| 215 |
if self.raeside: |
| 216 |
return |
| 217 |
self.raeside = True |
| 218 |
dtag = htag.findNext('p') |
| 219 |
description = '' |
| 220 |
if dtag is not None: |
| 221 |
description = self.tag_to_string(dtag, False) |
| 222 |
article_list.append(dict(title=title, url=url, date='', description=description, author='', content='')) |
| 223 |
print(sectitle + title + ': description = ' + description + ' URL=' + url + '\n\r') |
| 224 |
|
| 225 |
def add_section_index(self, ans, securl, sectitle): |
| 226 |
print('Add section url=' + self.url_prefix + '/' + securl + '\n\r') |
| 227 |
try: |
| 228 |
soup = self.index_to_soup(self.url_prefix + '/' + securl) |
| 229 |
except Exception: |
| 230 |
return ans |
| 231 |
mainsoup = soup.find('div', attrs={'class': re.compile(r'main.content')}) |
| 232 |
article_list = [] |
| 233 |
for wdiv in mainsoup.findAll('div', attrs={'id': re.compile(r'featured.story')}): |
| 234 |
for htag in wdiv.findAll('h3'): |
| 235 |
self.handle_articles(htag, article_list, sectitle) |
| 236 |
for ladiv in mainsoup.findAll(attrs={'class': re.compile(r'leading.articles')}): |
| 237 |
for wdiv in mainsoup.findAll('div', attrs={'class': re.compile(r'article.row')}): |
| 238 |
for htag in wdiv.findAll('h2'): |
| 239 |
self.handle_articles(htag, article_list, sectitle) |
| 240 |
ans.append((sectitle, article_list)) |
| 241 |
return ans |
| 242 |
|
| 243 |
def parse_index(self): |
| 244 |
ans = [] |
| 245 |
for url, title in self.section_list: |
| 246 |
ans = self.add_section_index(ans, url, title) |
| 247 |
return ans |