Debian Codegraph @

recipes/grantland.recipe

raw
1 import re
2
3 from calibre.web.feeds.news import BasicNewsRecipe
4
5
6 class GrantLand(BasicNewsRecipe):
7 title = 'Grantland'
8 description = 'Writings on Sports & Pop Culture'
9 language = 'en'
10 __author__ = 'barty on mobileread.com forum'
11 max_articles_per_feed = 100
12 no_stylesheets = True
13 # auto_cleanup is too aggressive sometimes and we end up with blank
14 # articles
15 auto_cleanup = False
16 timefmt = ' [%a, %d %b %Y]'
17 oldest_article = 90
18
19 cover_url = 'http://cdn0.sbnation.com/imported_assets/740965/blog_grantland_grid_3.jpg'
20 masthead_url = 'http://a1.espncdn.com/prod/assets/grantland/grantland-logo.jpg'
21
22 INDEX = 'http://www.grantland.com'
23 CATEGORIES = [
24 # comment out second line if you don't want older articles
25 # (user friendly name, url suffix, max number of articles to load)
26 ('Today in Grantland', '', 20),
27 ('In Case You Missed It', 'incaseyoumissedit', 35),
28 ]
29
30 remove_tags = [
31 {'name': ['style', 'aside', 'nav', 'footer', 'script']},
32 {'name': 'h1', 'text': 'Grantland'},
33 {'id': ['header', 'col-right']},
34 {'class': ['connect_widget']},
35 {'name': 'section', 'class': re.compile(r'\b(ad|module)\b')},
36 ]
37
38 preprocess_regexps = [
39 # remove blog banners
40 (re.compile(r'<a href="/blog/(?:(?!</a>).)+</a>', re.DOTALL | re.IGNORECASE), lambda m: ''),
41 ]
42
43 def parse_index(self):
44 feeds = []
45 seen_urls = set()
46
47 for category in self.CATEGORIES:
48 cat_name, tag, max_articles = category
49 self.log('Reading category:', cat_name)
50 articles = []
51
52 page = '%s/%s' % (self.INDEX, tag)
53 soup = self.index_to_soup(page)
54
55 main = soup.find('div', id='col-main')
56 if main is None:
57 main = soup
58
59 for tag in main.findAll('a', href=re.compile(r'(story|post)/_/id/\d+')):
60 url = tag['href']
61 if url in seen_urls:
62 continue
63 title = tag.string
64 # blank title probably means <a href=".."><img /></a>. skip
65 if not title:
66 continue
67 self.log('\tFound article:', title)
68 self.log('\t', url)
69 articles.append({'title': title, 'url': url})
70 seen_urls.add(url)
71
72 if len(articles) >= max_articles:
73 break
74
75 if articles:
76 feeds.append((cat_name, articles))
77
78 return feeds