| 1 |
#!/usr/bin/env python |
| 2 |
# vim:fileencoding=utf-8 |
| 3 |
# License: GPLv3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net> |
| 4 |
|
| 5 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 6 |
|
| 7 |
|
| 8 |
def absolutize(href): |
| 9 |
if href.startswith('/'): |
| 10 |
href = 'https://www.theglobeandmail.com' + href |
| 11 |
return href |
| 12 |
|
| 13 |
|
| 14 |
def classes(classes): |
| 15 |
q = frozenset(classes.split(' ')) |
| 16 |
return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)}) |
| 17 |
|
| 18 |
|
| 19 |
class GlobeMail(BasicNewsRecipe): |
| 20 |
title = 'The Globe and Mail' |
| 21 |
__author__ = 'Kovid Goyal' |
| 22 |
encoding = 'utf-8' |
| 23 |
publisher = 'Globe & Mail' |
| 24 |
language = 'en_CA' |
| 25 |
ignore_duplicate_articles = {'title', 'url'} |
| 26 |
no_stylesheets = True |
| 27 |
remove_attributes = ['style'] |
| 28 |
delay = 1 |
| 29 |
|
| 30 |
keep_only_tags = [classes('article-title c-article-meta l-article l-article-title')] |
| 31 |
remove_tags = [ |
| 32 |
classes('c-ad pb-f-commercial-dfp-ads pb-f-article-actions pb-f-article-meta article-toolbar'), |
| 33 |
dict(name='source'), |
| 34 |
] |
| 35 |
|
| 36 |
def parse_index(self): |
| 37 |
ans = [] |
| 38 |
for section in 'canada opinion politics sports life arts world real-estate'.split(): |
| 39 |
if self.test and len(ans) >= self.test[0]: |
| 40 |
break |
| 41 |
soup = self.index_to_soup('https://www.theglobeandmail.com/{}/'.format(section)) |
| 42 |
self.log('Processing section:', section) |
| 43 |
articles = list(self.parse_gm_section(soup)) |
| 44 |
if articles: |
| 45 |
ans.append((section.capitalize(), articles)) |
| 46 |
return ans |
| 47 |
|
| 48 |
def parse_gm_section(self, soup): |
| 49 |
for a in soup.findAll('a', href=True, attrs={'data-lt-pos': lambda x: x and x.endswith('featured')}): |
| 50 |
headline = a.find('div', 'c-card__hed-text') |
| 51 |
title = self.tag_to_string(headline) |
| 52 |
url = absolutize(a['href']) |
| 53 |
self.log(' ', title, 'at', url) |
| 54 |
yield {'title': title, 'url': url} |