| 1 |
__license__ = 'GPL v3' |
| 2 |
__copyright__ = '2009-2013, Darko Miletic <darko.miletic at gmail.com>' |
| 3 |
''' |
| 4 |
www.thestar.com |
| 5 |
''' |
| 6 |
|
| 7 |
from collections import defaultdict |
| 8 |
|
| 9 |
from calibre.web.feeds.news import BasicNewsRecipe, classes |
| 10 |
|
| 11 |
|
| 12 |
def absolutize(url): |
| 13 |
if url.startswith('/'): |
| 14 |
url = 'https://www.thestar.com' + url |
| 15 |
return url |
| 16 |
|
| 17 |
|
| 18 |
class TheTorontoStar(BasicNewsRecipe): |
| 19 |
title = 'The Toronto Star' |
| 20 |
__author__ = 'Darko Miletic' |
| 21 |
description = "Thestar.com is Canada's largest online news site. Stay current with our sports, business entertainment news and more from the Toronto Star and thestar.com" # noqa: E501 |
| 22 |
oldest_article = 2 |
| 23 |
language = 'en_CA' |
| 24 |
max_articles_per_feed = 100 |
| 25 |
no_stylesheets = True |
| 26 |
use_embedded_content = False |
| 27 |
delay = 2 |
| 28 |
publisher = 'The Toronto Star' |
| 29 |
encoding = 'utf-8' |
| 30 |
|
| 31 |
keep_only_tags = [classes('headline asset-summary metaPrimary tsArticleContainer')] |
| 32 |
remove_tags = [classes('share-container shareIcons articleFeedbackCTA asset-comments'), dict(name='button')] |
| 33 |
|
| 34 |
def parse_section(self, section): |
| 35 |
for article in section.findAll('article'): |
| 36 |
a = article.find('a', attrs={'class': 'tnt-asset-link', 'href': True, 'aria-label': True}) |
| 37 |
if a is not None: |
| 38 |
title = a['aria-label'] |
| 39 |
url = absolutize(a['href']) |
| 40 |
desc = '' |
| 41 |
summ = article.find(attrs={'class': 'tnt-summary'}) |
| 42 |
if summ is not None: |
| 43 |
desc = self.tag_to_string(summ) |
| 44 |
section = '' |
| 45 |
sec = article.find(attrs={'class': 'label-flag-section'}) |
| 46 |
if sec is not None: |
| 47 |
section = self.tag_to_string(sec).strip().lower().capitalize() |
| 48 |
if section == 'Gta': |
| 49 |
section = 'GTA' |
| 50 |
self.log(section + ':', title) |
| 51 |
yield section, {'title': title, 'url': url, 'description': desc} |
| 52 |
|
| 53 |
def parse_index(self): |
| 54 |
soup = self.index_to_soup('https://www.thestar.com/') |
| 55 |
ans = defaultdict(list) |
| 56 |
for section in soup.findAll('section', attrs={'class': 'block'}): |
| 57 |
for sec, article in self.parse_section(section): |
| 58 |
ans[sec].append(article) |
| 59 |
return list(ans.items()) |