| 1 |
import re |
| 2 |
|
| 3 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 4 |
|
| 5 |
|
| 6 |
class GrantLand(BasicNewsRecipe): |
| 7 |
title = 'Grantland' |
| 8 |
description = 'Writings on Sports & Pop Culture' |
| 9 |
language = 'en' |
| 10 |
__author__ = 'barty on mobileread.com forum' |
| 11 |
max_articles_per_feed = 100 |
| 12 |
no_stylesheets = True |
| 13 |
# auto_cleanup is too aggressive sometimes and we end up with blank |
| 14 |
# articles |
| 15 |
auto_cleanup = False |
| 16 |
timefmt = ' [%a, %d %b %Y]' |
| 17 |
oldest_article = 90 |
| 18 |
|
| 19 |
cover_url = 'http://cdn0.sbnation.com/imported_assets/740965/blog_grantland_grid_3.jpg' |
| 20 |
masthead_url = 'http://a1.espncdn.com/prod/assets/grantland/grantland-logo.jpg' |
| 21 |
|
| 22 |
INDEX = 'http://www.grantland.com' |
| 23 |
CATEGORIES = [ |
| 24 |
# comment out second line if you don't want older articles |
| 25 |
# (user friendly name, url suffix, max number of articles to load) |
| 26 |
('Today in Grantland', '', 20), |
| 27 |
('In Case You Missed It', 'incaseyoumissedit', 35), |
| 28 |
] |
| 29 |
|
| 30 |
remove_tags = [ |
| 31 |
{'name': ['style', 'aside', 'nav', 'footer', 'script']}, |
| 32 |
{'name': 'h1', 'text': 'Grantland'}, |
| 33 |
{'id': ['header', 'col-right']}, |
| 34 |
{'class': ['connect_widget']}, |
| 35 |
{'name': 'section', 'class': re.compile(r'\b(ad|module)\b')}, |
| 36 |
] |
| 37 |
|
| 38 |
preprocess_regexps = [ |
| 39 |
# remove blog banners |
| 40 |
(re.compile(r'<a href="/blog/(?:(?!</a>).)+</a>', re.DOTALL | re.IGNORECASE), lambda m: ''), |
| 41 |
] |
| 42 |
|
| 43 |
def parse_index(self): |
| 44 |
feeds = [] |
| 45 |
seen_urls = set() |
| 46 |
|
| 47 |
for category in self.CATEGORIES: |
| 48 |
cat_name, tag, max_articles = category |
| 49 |
self.log('Reading category:', cat_name) |
| 50 |
articles = [] |
| 51 |
|
| 52 |
page = '%s/%s' % (self.INDEX, tag) |
| 53 |
soup = self.index_to_soup(page) |
| 54 |
|
| 55 |
main = soup.find('div', id='col-main') |
| 56 |
if main is None: |
| 57 |
main = soup |
| 58 |
|
| 59 |
for tag in main.findAll('a', href=re.compile(r'(story|post)/_/id/\d+')): |
| 60 |
url = tag['href'] |
| 61 |
if url in seen_urls: |
| 62 |
continue |
| 63 |
title = tag.string |
| 64 |
# blank title probably means <a href=".."><img /></a>. skip |
| 65 |
if not title: |
| 66 |
continue |
| 67 |
self.log('\tFound article:', title) |
| 68 |
self.log('\t', url) |
| 69 |
articles.append({'title': title, 'url': url}) |
| 70 |
seen_urls.add(url) |
| 71 |
|
| 72 |
if len(articles) >= max_articles: |
| 73 |
break |
| 74 |
|
| 75 |
if articles: |
| 76 |
feeds.append((cat_name, articles)) |
| 77 |
|
| 78 |
return feeds |