| 1 |
#!/usr/bin/env python |
| 2 |
from __future__ import unicode_literals |
| 3 |
|
| 4 |
__license__ = 'GPL v3' |
| 5 |
__copyright__ = '2017, John Hutson <jfhutson at gmail.com>' |
| 6 |
''' |
| 7 |
firstthings.com |
| 8 |
''' |
| 9 |
import html5lib |
| 10 |
from lxml import html |
| 11 |
|
| 12 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 13 |
|
| 14 |
|
| 15 |
class FirstThings(BasicNewsRecipe): |
| 16 |
title = 'First Things' |
| 17 |
__author__ = 'John Hutson' |
| 18 |
description = "America's Most Influential Journal of Religion and Public Life" |
| 19 |
INDEX = 'https://www.firstthings.com/current-edition' |
| 20 |
language = 'en' |
| 21 |
encoding = 'utf-8' |
| 22 |
|
| 23 |
no_stylesheets = True |
| 24 |
|
| 25 |
keep_only_tags = [ |
| 26 |
dict(name='h1'), |
| 27 |
dict( |
| 28 |
attrs={ |
| 29 |
'itemprop': [ |
| 30 |
'author', |
| 31 |
] |
| 32 |
} |
| 33 |
), |
| 34 |
dict(attrs={'itemprop': 'articleBody'}), |
| 35 |
] |
| 36 |
|
| 37 |
extra_css = ''' |
| 38 |
.small-caps { font-variant: small-caps } |
| 39 |
.drop-cap { float: left; font-size: 75px; line-height: 60px; padding-top: 4px; padding-right: 8px; padding-left: 3px;} |
| 40 |
''' |
| 41 |
|
| 42 |
def preprocess_raw_html(self, raw, url): |
| 43 |
return html.tostring(html5lib.parse(raw, treebuilder='lxml', namespaceHTMLElements=False), method='html', encoding='unicode') |
| 44 |
|
| 45 |
def parse_index(self): |
| 46 |
soup = self.index_to_soup(self.INDEX) |
| 47 |
cover = soup.find('a', 'cover-link') |
| 48 |
if cover is not None: |
| 49 |
img = cover |
| 50 |
if img: |
| 51 |
self.cover_url = img['href'] |
| 52 |
current_section, current_articles = 'Cover Story', [] |
| 53 |
feeds = [] |
| 54 |
for div in soup.findAll(['h3', 'h4', 'a']): |
| 55 |
if div.name == 'h3': |
| 56 |
if current_articles: |
| 57 |
feeds.append((current_section, current_articles)) |
| 58 |
current_articles = [] |
| 59 |
current_section = self.tag_to_string(div) |
| 60 |
self.log('\nFound section:', current_section) |
| 61 |
elif div.name == 'h4': |
| 62 |
a = div.findChild('a') |
| 63 |
title = self.tag_to_string(a) |
| 64 |
url = a['href'] |
| 65 |
desc = '' |
| 66 |
if url.startswith('/'): |
| 67 |
url = 'https://www.firstthings.com/' + url |
| 68 |
elif div.name == 'a' and div.rel == 'author': |
| 69 |
desc = self.tag_to_string(div) |
| 70 |
current_articles.append({'title': title, 'url': url, 'description': desc}) |
| 71 |
|
| 72 |
if current_articles: |
| 73 |
feeds.append((current_section, current_articles)) |
| 74 |
return feeds |