Debian Codegraph @

recipes/inc.recipe

raw
1 #!/usr/bin/env python
2
3 __license__ = 'GPL v3'
4 __copyright__ = '2014, Nikolas Mangold-Takao <nmangold at gmail.com>'
5 __version__ = '0.10'
6
7 from calibre.web.feeds.recipes import BasicNewsRecipe
8
9
10 class IncMagazineRecipe(BasicNewsRecipe):
11 language = 'en'
12 version = 1
13
14 __author__ = 'Nikolas Mangold-Takao <nmangold at gmail.com>'
15
16 title = 'Inc Magazine'
17 publisher = 'Mansueto Ventures LLC'
18 category = 'News, Business'
19 description = 'Free account required to browse website. Handbook of the American Entrepeneur'
20
21 # signup is free, without account browsing is limited and recipe won't work
22 needs_subscription = True
23
24 use_embedded_content = False
25 remove_empty_feeds = True
26
27 no_stylesheets = True
28 remove_javascript = True
29
30 INDEX = 'http://www.inc.com/magazine'
31
32 remove_tags_before = dict(name='div', attrs={'id': 'outercontainer'})
33 remove_tags = [dict(name='div', attrs={'id': 'morearticles'})]
34 remove_tags_after = dict(name='div', attrs={'id': 'article-share-footer'})
35
36 extra_css = '''
37 div.byline {font-size: x-small; color: #696969; margin-top: 0.4em;}
38 '''
39
40 def get_browser(self):
41 def has_login_name(form):
42 try:
43 form.find_control(name='email')
44 except Exception:
45 return False
46 else:
47 return True
48
49 br = BasicNewsRecipe.get_browser(self)
50 if self.username is not None and self.password is not None:
51 br.open('http://www.inc.com/login')
52 br.select_form(predicate=has_login_name)
53 br['email'] = self.username
54 br['password'] = self.password
55 br.submit()
56 return br
57
58 def parse_index(self):
59 soup = self.index_to_soup(self.INDEX)
60
61 feeds = []
62 articles = []
63
64 # We need for sure 'toc-top-l-column' and 'toc-bot-l-column'
65 # I am not sure about 'toc-top-r-column and 'toc-bot-r-column', seems
66 # to be mostly ads
67 for toc in soup.findAll('div', id=lambda x: x and x.endswith('-l-column')):
68 # h1-h2-h3
69 for article in toc.findAll('div', {'class': lambda x: x and x.startswith('toc-h')}):
70 link = article.find('a')
71 if link is None:
72 continue
73 article_title = self.tag_to_string(link)
74 article_link = link.get('href')
75 articles.append({'title': article_title, 'url': article_link})
76 feeds.append(('Articles', articles)) # TODO figure out sections
77 return feeds
78
79 def preprocess_html(self, soup):
80 for alink in soup.findAll('a'):
81 if alink.string is not None:
82 tstr = alink.string
83 alink.replaceWith(tstr)
84 return soup