Debian Codegraph @

recipes/first_things.recipe

raw
1 #!/usr/bin/env python
2 from __future__ import unicode_literals
3
4 __license__ = 'GPL v3'
5 __copyright__ = '2017, John Hutson <jfhutson at gmail.com>'
6 '''
7 firstthings.com
8 '''
9 import html5lib
10 from lxml import html
11
12 from calibre.web.feeds.news import BasicNewsRecipe
13
14
15 class FirstThings(BasicNewsRecipe):
16 title = 'First Things'
17 __author__ = 'John Hutson'
18 description = "America's Most Influential Journal of Religion and Public Life"
19 INDEX = 'https://www.firstthings.com/current-edition'
20 language = 'en'
21 encoding = 'utf-8'
22
23 no_stylesheets = True
24
25 keep_only_tags = [
26 dict(name='h1'),
27 dict(
28 attrs={
29 'itemprop': [
30 'author',
31 ]
32 }
33 ),
34 dict(attrs={'itemprop': 'articleBody'}),
35 ]
36
37 extra_css = '''
38 .small-caps { font-variant: small-caps }
39 .drop-cap { float: left; font-size: 75px; line-height: 60px; padding-top: 4px; padding-right: 8px; padding-left: 3px;}
40 '''
41
42 def preprocess_raw_html(self, raw, url):
43 return html.tostring(html5lib.parse(raw, treebuilder='lxml', namespaceHTMLElements=False), method='html', encoding='unicode')
44
45 def parse_index(self):
46 soup = self.index_to_soup(self.INDEX)
47 cover = soup.find('a', 'cover-link')
48 if cover is not None:
49 img = cover
50 if img:
51 self.cover_url = img['href']
52 current_section, current_articles = 'Cover Story', []
53 feeds = []
54 for div in soup.findAll(['h3', 'h4', 'a']):
55 if div.name == 'h3':
56 if current_articles:
57 feeds.append((current_section, current_articles))
58 current_articles = []
59 current_section = self.tag_to_string(div)
60 self.log('\nFound section:', current_section)
61 elif div.name == 'h4':
62 a = div.findChild('a')
63 title = self.tag_to_string(a)
64 url = a['href']
65 desc = ''
66 if url.startswith('/'):
67 url = 'https://www.firstthings.com/' + url
68 elif div.name == 'a' and div.rel == 'author':
69 desc = self.tag_to_string(div)
70 current_articles.append({'title': title, 'url': url, 'description': desc})
71
72 if current_articles:
73 feeds.append((current_section, current_articles))
74 return feeds