Debian Codegraph @

recipes/vic_times.recipe

raw
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3 from __future__ import print_function
4
5 __license__ = 'GPL v3'
6
7 '''
8 www.canada.com
9 '''
10 import re
11
12 from calibre.ebooks.BeautifulSoup import Tag
13 from calibre.web.feeds.news import BasicNewsRecipe
14
15
16 def new_tag(soup, name, attrs=()):
17 impl = getattr(soup, 'new_tag', None)
18 if impl is not None:
19 return impl(name, attrs=dict(attrs))
20 return Tag(soup, name, attrs=attrs or None)
21
22
23 class TimesColonist(BasicNewsRecipe):
24 # Customization -- remove sections you don't want.
25 # If your e-reader is an e-ink Kindle and your output profile is
26 # set properly this recipe will not include images because the
27 # resulting file is too large. If you have one of these and want
28 # images you can set kindle_omit_images = False
29 # and remove sections (typically the e-ink Kindles will
30 # work with about a dozen of these, but your mileage may vary).
31
32 kindle_omit_images = True
33
34 section_list = [
35 ('', 'Web Front Page'),
36 ('news/', 'News Headlines'),
37 ('news/b-c/', 'BC News'),
38 ('news/national/', 'National News'),
39 ('news/world/', 'World News'),
40 ('opinion/', 'Opinion'),
41 ('opinion/letters/', 'Letters'),
42 ('business/', 'Business'),
43 ('business/money/', 'Money'),
44 ('business/technology/', 'Technology'),
45 ('business/working/', 'Working'),
46 ('sports/', 'Sports'),
47 ('sports/hockey/', 'Hockey'),
48 ('sports/football/', 'Football'),
49 ('sports/basketball/', 'Basketball'),
50 ('sports/golf/', 'Golf'),
51 ('entertainment/', 'entertainment'),
52 ('entertainment/go/', 'Go!'),
53 ('entertainment/music/', 'Music'),
54 ('entertainment/books/', 'Books'),
55 ('entertainment/Movies/', 'Movies'),
56 ('entertainment/television/', 'Television'),
57 ('life/', 'Life'),
58 ('life/health/', 'Health'),
59 ('life/travel/', 'Travel'),
60 ('life/driving/', 'Driving'),
61 ('life/homes/', 'Homes'),
62 ('life/food-drink/', 'Food & Drink'),
63 ]
64
65 title = 'Victoria Times Colonist'
66 url_prefix = 'http://www.timescolonist.com'
67 description = 'News from Victoria, BC'
68 fp_tag = 'CAN_TC'
69
70 masthead_url = 'http://www.timescolonist.com/gmg/img/global/logoTimesColonist.png'
71
72 url_list = []
73 language = 'en_CA'
74 __author__ = 'Nick Redding'
75 no_stylesheets = True
76 timefmt = ' [%b %d]'
77 encoding = 'utf-8'
78 extra_css = '''
79 .byline { font-size:xx-small; font-weight: bold;}
80 h3 { margin-bottom: 6px; }
81 .caption { font-size: xx-small; font-style: italic; font-weight: normal; }
82 '''
83 keep_only_tags = [dict(name='div', attrs={'class': re.compile(r'main.content')})]
84
85 def __init__(self, options, log, progress_reporter):
86 self.remove_tags = [
87 {'class': 'comments'},
88 {'id': 'photocredit'},
89 dict(name='div', attrs={'class': re.compile(r'top.controls')}),
90 dict(name='div', attrs={'class': re.compile(r'^comments')}),
91 dict(name='div', attrs={'class': re.compile(r'social')}),
92 dict(name='div', attrs={'class': re.compile(r'tools')}),
93 dict(name='div', attrs={'class': re.compile(r'bottom.tools')}),
94 dict(name='div', attrs={'class': re.compile(r'window')}),
95 dict(name='div', attrs={'class': re.compile(r'related.news.element')}),
96 ]
97 print('PROFILE NAME = ' + options.output_profile.short_name)
98 if self.kindle_omit_images and options.output_profile.short_name in ['kindle', 'kindle_dx', 'kindle_pw']:
99 self.remove_tags.append(dict(name='div', attrs={'class': re.compile(r'image-container')}))
100 BasicNewsRecipe.__init__(self, options, log, progress_reporter)
101
102 def get_cover_url(self):
103 from datetime import date, timedelta
104
105 cover = 'http://webmedia.newseum.org/newseum-multimedia/dfp/jpg' + str(date.today().day) + '/lg/' + self.fp_tag + '.jpg'
106 br = BasicNewsRecipe.get_browser(self)
107 daysback = 1
108 try:
109 br.open(cover)
110 except Exception:
111 while daysback < 7:
112 cover = (
113 'http://webmedia.newseum.org/newseum-multimedia/dfp/jpg'
114 + str((date.today() - timedelta(days=daysback)).day)
115 + '/lg/'
116 + self.fp_tag
117 + '.jpg'
118 )
119 br = BasicNewsRecipe.get_browser(self)
120 try:
121 br.open(cover)
122 except Exception:
123 daysback = daysback + 1
124 continue
125 break
126 if daysback == 7:
127 self.log('\nCover unavailable')
128 cover = None
129 return cover
130
131 def fixChars(self, string):
132 # Replace lsquo (\x91)
133 fixed = string.replace('\x91', '‘')
134 # Replace rsquo (\x92)
135 fixed = fixed.replace('\x92', '’')
136 # Replace ldquo (\x93)
137 fixed = fixed.replace('\x93', '“')
138 # Replace rdquo (\x94)
139 fixed = fixed.replace('\x94', '”')
140 # Replace ndash (\x96)
141 fixed = fixed.replace('\x96', '–')
142 # Replace mdash (\x97)
143 fixed = fixed.replace('\x97', '—')
144 fixed = fixed.replace('&#x2019;', '’')
145 return fixed
146
147 def massageNCXText(self, description):
148 return description
149
150 def populate_article_metadata(self, article, soup, first):
151 if first:
152 picdiv = soup.find('body').find('img')
153 if picdiv is not None:
154 self.add_toc_thumbnail(article, re.sub(r'links\\link\d+\\', '', picdiv['src']))
155 xtitle = article.text_summary.strip()
156 if len(xtitle) == 0:
157 desc = soup.find('meta', attrs={'property': 'og:description'})
158 if desc is not None:
159 article.summary = article.text_summary = desc['content']
160
161 def strip_anchors(self, soup):
162 paras = soup.findAll(True)
163 for para in paras:
164 aTags = para.findAll('a')
165 for a in aTags:
166 if a.img is None:
167 a.replaceWith(a.renderContents().decode('cp1252', 'replace'))
168 return soup
169
170 def preprocess_html(self, soup):
171 byline = soup.find('p', attrs={'class': re.compile(r'ancillary')})
172 if byline is not None:
173 authstr = self.tag_to_string(byline, False)
174 authstr = re.sub(r'/ *Times Colonist', '/', authstr, flags=re.IGNORECASE)
175 authstr = re.sub(r'BY */', '', authstr, flags=re.IGNORECASE)
176 newdiv = new_tag(soup, 'div')
177 newdiv.insert(0, authstr)
178 newdiv['class'] = 'byline'
179 byline.replaceWith(newdiv)
180 for caption in soup.findAll('p', attrs={'class': re.compile(r'caption')}):
181 capstr = self.tag_to_string(caption, False)
182 capstr = re.sub(r'Photograph by.*$', '', capstr, flags=re.IGNORECASE)
183 newdiv = new_tag(soup, 'div')
184 newdiv.insert(0, capstr)
185 newdiv['class'] = 'caption'
186 caption.replaceWith(newdiv)
187 for ptag in soup.findAll('p'):
188 ptext = self.tag_to_string(ptag, use_alt=False, normalize_whitespace=True)
189 ptext = re.sub(r'\s+', '', ptext)
190 if ptext in {'', '&nbsp;'}:
191 ptag.extract()
192 return self.strip_anchors(soup)
193
194 raeside = False
195
196 def handle_articles(self, htag, article_list, sectitle):
197 atag = htag.a
198 if atag is not None:
199 url = atag['href']
200 url = url.strip()
201 # print('Checking >>'+url+'<<\n\r')
202 if url.startswith('/'):
203 url = self.url_prefix + url
204 if url in self.url_list:
205 return
206 self.url_list.append(url)
207 title = self.tag_to_string(atag, False)
208 if 'VIDEO' in title.upper():
209 return
210 if 'GALLERY' in title.upper():
211 return
212 if 'PHOTOS' in title.upper():
213 return
214 if 'RAESIDE' in title.upper():
215 if self.raeside:
216 return
217 self.raeside = True
218 dtag = htag.findNext('p')
219 description = ''
220 if dtag is not None:
221 description = self.tag_to_string(dtag, False)
222 article_list.append(dict(title=title, url=url, date='', description=description, author='', content=''))
223 print(sectitle + title + ': description = ' + description + ' URL=' + url + '\n\r')
224
225 def add_section_index(self, ans, securl, sectitle):
226 print('Add section url=' + self.url_prefix + '/' + securl + '\n\r')
227 try:
228 soup = self.index_to_soup(self.url_prefix + '/' + securl)
229 except Exception:
230 return ans
231 mainsoup = soup.find('div', attrs={'class': re.compile(r'main.content')})
232 article_list = []
233 for wdiv in mainsoup.findAll('div', attrs={'id': re.compile(r'featured.story')}):
234 for htag in wdiv.findAll('h3'):
235 self.handle_articles(htag, article_list, sectitle)
236 for ladiv in mainsoup.findAll(attrs={'class': re.compile(r'leading.articles')}):
237 for wdiv in mainsoup.findAll('div', attrs={'class': re.compile(r'article.row')}):
238 for htag in wdiv.findAll('h2'):
239 self.handle_articles(htag, article_list, sectitle)
240 ans.append((sectitle, article_list))
241 return ans
242
243 def parse_index(self):
244 ans = []
245 for url, title in self.section_list:
246 ans = self.add_section_index(ans, url, title)
247 return ans