Debian Codegraph @

recipes/chronicle_higher_ed.recipe

raw
1 import re
2 from collections import OrderedDict
3
4 from calibre.web.feeds.recipes import BasicNewsRecipe
5
6
7 class Chronicle(BasicNewsRecipe):
8 title = 'The Chronicle of Higher Education'
9 __author__ = 'Rick Shang'
10
11 description = 'Weekly news and job-information source for college and university faculty members, administrators, and students.'
12 language = 'en'
13 category = 'news'
14 encoding = 'UTF-8'
15 keep_only_tags = [
16 dict(name='div', attrs={'class': ['article', 'blog-mod']}),
17 ]
18 remove_tags = [
19 dict(name='div', attrs={'class': ['related module1', 'maintitle', 'entry-utility', 'object-meta']}),
20 dict(name='div', attrs={'id': ['section-nav', 'icon-row', 'enlarge-popup', 'confirm-popup']}),
21 dict(name='a', attrs={'class': 'show-enlarge enlarge'}),
22 ]
23 no_javascript = True
24 no_stylesheets = True
25
26 needs_subscription = True
27
28 def get_browser(self):
29 br = BasicNewsRecipe.get_browser(self)
30 if self.username is not None and self.password is not None:
31 br.open('http://chronicle.com/myaccount/login')
32 br.select_form(nr=1)
33 br['username'] = self.username
34 br['password'] = self.password
35 br.submit()
36 return br
37
38 def parse_index(self):
39 # Go to the issue
40 soup0 = self.index_to_soup('http://chronicle.com/section/Archives/39/')
41 issue = soup0.find('ul', attrs={'class': 'feature-promo-list'}).li
42 issueurl = 'http://chronicle.com' + issue.a['href']
43
44 # Find date
45 dates = self.tag_to_string(issue.a).split(': ')[-1]
46 self.timefmt = ' [%s]' % dates
47
48 # Find cover
49 cover = soup0.find('div', attrs={'class': 'side-content'}).find(attrs={'src': re.compile(r'photos/biz/Current')})
50 if cover is not None:
51 if 'chronicle.com' in cover['src']:
52 self.cover_url = cover['src']
53 else:
54 self.cover_url = 'http://chronicle.com' + cover['src']
55 # Go to the main body
56 soup = self.index_to_soup(issueurl)
57 div = soup.find('div', attrs={'id': 'article-body'})
58
59 feeds = OrderedDict()
60 section_title = ''
61 for post in div.findAll('li'):
62 articles = []
63 a = post.find('a', href=True)
64 if a is not None:
65 title = self.tag_to_string(a)
66 url = 'http://chronicle.com' + a['href'].strip()
67 sectiontitle = post.findPrevious('h3')
68 if sectiontitle is None:
69 sectiontitle = post.findPrevious('h4')
70 section_title = self.tag_to_string(sectiontitle)
71 desc = self.tag_to_string(post.find('p'))
72 articles.append({'title': title, 'url': url, 'description': desc, 'date': ''})
73
74 if articles:
75 if section_title not in feeds:
76 feeds[section_title] = []
77 feeds[section_title] += articles
78 ans = list(feeds.items())
79 return ans
80
81 def preprocess_html(self, soup):
82 # process all the images
83 for div in soup.findAll('div', attrs={'class': 'tableauPlaceholder'}):
84 noscripts = div.find('noscript').a
85 div.replaceWith(noscripts)
86 for div0 in soup.findAll('div', text='Powered by Tableau'):
87 div0.extract()
88 return soup