| 1 |
#!/usr/bin/env python |
| 2 |
import re |
| 3 |
from collections import defaultdict |
| 4 |
|
| 5 |
from calibre.web.feeds.news import BasicNewsRecipe, classes |
| 6 |
|
| 7 |
BASE = 'https://www.nature.com' |
| 8 |
|
| 9 |
|
| 10 |
def absurl(url): |
| 11 |
if url.startswith('/'): |
| 12 |
url = BASE + url |
| 13 |
elif url.startswith('http://'): |
| 14 |
url = 'https' + url[4:] |
| 15 |
return url |
| 16 |
|
| 17 |
|
| 18 |
def check_words(words): |
| 19 |
return lambda x: x and frozenset(words.split()).intersection(x.split()) |
| 20 |
|
| 21 |
|
| 22 |
def has_all_of(words): |
| 23 |
return lambda x: x and frozenset(words.split()).issubset(x.split()) |
| 24 |
|
| 25 |
|
| 26 |
class Nature(BasicNewsRecipe): |
| 27 |
title = 'Nature' |
| 28 |
__author__ = 'Jose Ortiz' |
| 29 |
description = ( |
| 30 |
'Nature is a weekly international multidisciplinary scientific journal' |
| 31 |
' publishing peer-reviewed research in all fields of science and' |
| 32 |
' technology on the basis of its originality, importance,' |
| 33 |
' interdisciplinary interest, timeliness, accessibility, elegance and' |
| 34 |
' surprising conclusions. Nature also provides rapid, authoritative,' |
| 35 |
' insightful and arresting news and interpretation of topical and coming' |
| 36 |
' trends affecting science, scientists and the wider public.' |
| 37 |
) |
| 38 |
language = 'en' |
| 39 |
encoding = 'UTF-8' |
| 40 |
no_javascript = True |
| 41 |
no_stylesheets = True |
| 42 |
|
| 43 |
keep_only_tags = [dict(name='article')] |
| 44 |
|
| 45 |
remove_tags = [ |
| 46 |
classes('u-hide-print hide-print c-latest-content__item c-context-bar c-pdf-button__container u-js-hide'), |
| 47 |
dict(name='img', attrs={'class': ['visually-hidden']}), |
| 48 |
] |
| 49 |
|
| 50 |
def parse_index(self): |
| 51 |
soup = self.index_to_soup(BASE + '/nature/current-issue') |
| 52 |
self.cover_url = soup.find('img', attrs={'data-test': check_words('issue-cover-image')})['src'] |
| 53 |
try: |
| 54 |
self.cover_url = re.sub(r'\bw\d+\b', 'w1000', self.cover_url) # enlarge cover size resolution |
| 55 |
except Exception: |
| 56 |
''' |
| 57 |
failed, img src might have changed, use default width 200 |
| 58 |
''' |
| 59 |
pass |
| 60 |
|
| 61 |
section_tags = soup.find_all('section', attrs={'data-container-type': 'issue-section-list'}) |
| 62 |
|
| 63 |
sections = defaultdict(list) |
| 64 |
ordered_sec_titles = [] |
| 65 |
index = [] |
| 66 |
|
| 67 |
for sec in section_tags: |
| 68 |
sec_title = self.tag_to_string(sec.find('h2')) |
| 69 |
ordered_sec_titles.append(sec_title) |
| 70 |
for article in sec.findAll('article'): |
| 71 |
try: |
| 72 |
url = absurl(article.find('a', {'itemprop': check_words('url')})['href']) |
| 73 |
except TypeError: |
| 74 |
continue |
| 75 |
title = self.tag_to_string(article.find('h3', {'itemprop': has_all_of('name headline')})) |
| 76 |
date = ' [' + self.tag_to_string(article.find('time', {'itemprop': check_words('datePublished')})) + ']' |
| 77 |
author = self.tag_to_string(article.find('li', {'itemprop': check_words('creator')})) |
| 78 |
description = self.tag_to_string(article.find(attrs={'data-test': check_words('article.type')})) + ' • ' |
| 79 |
description += self.tag_to_string(article.find('div', attrs={'itemprop': check_words('description')})) |
| 80 |
sections[sec_title].append({'title': title, 'url': url, 'description': description, 'date': date, 'author': author}) |
| 81 |
|
| 82 |
for k in ordered_sec_titles: |
| 83 |
index.append((k, sections[k])) |
| 84 |
return index |
| 85 |
|
| 86 |
def preprocess_html(self, soup): |
| 87 |
for img in soup.findAll('img', {'data-src': True}): |
| 88 |
if img['data-src'].startswith('//'): |
| 89 |
img['src'] = 'https:' + img['data-src'] |
| 90 |
else: |
| 91 |
img['src'] = img['data-src'] |
| 92 |
for div in soup.findAll('div', {'data-component': check_words('article-container')})[1:]: |
| 93 |
div.extract() |
| 94 |
return soup |