Debian Codegraph @

recipes/nature.recipe

raw
1 #!/usr/bin/env python
2 import re
3 from collections import defaultdict
4
5 from calibre.web.feeds.news import BasicNewsRecipe, classes
6
7 BASE = 'https://www.nature.com'
8
9
10 def absurl(url):
11 if url.startswith('/'):
12 url = BASE + url
13 elif url.startswith('http://'):
14 url = 'https' + url[4:]
15 return url
16
17
18 def check_words(words):
19 return lambda x: x and frozenset(words.split()).intersection(x.split())
20
21
22 def has_all_of(words):
23 return lambda x: x and frozenset(words.split()).issubset(x.split())
24
25
26 class Nature(BasicNewsRecipe):
27 title = 'Nature'
28 __author__ = 'Jose Ortiz'
29 description = (
30 'Nature is a weekly international multidisciplinary scientific journal'
31 ' publishing peer-reviewed research in all fields of science and'
32 ' technology on the basis of its originality, importance,'
33 ' interdisciplinary interest, timeliness, accessibility, elegance and'
34 ' surprising conclusions. Nature also provides rapid, authoritative,'
35 ' insightful and arresting news and interpretation of topical and coming'
36 ' trends affecting science, scientists and the wider public.'
37 )
38 language = 'en'
39 encoding = 'UTF-8'
40 no_javascript = True
41 no_stylesheets = True
42
43 keep_only_tags = [dict(name='article')]
44
45 remove_tags = [
46 classes('u-hide-print hide-print c-latest-content__item c-context-bar c-pdf-button__container u-js-hide'),
47 dict(name='img', attrs={'class': ['visually-hidden']}),
48 ]
49
50 def parse_index(self):
51 soup = self.index_to_soup(BASE + '/nature/current-issue')
52 self.cover_url = soup.find('img', attrs={'data-test': check_words('issue-cover-image')})['src']
53 try:
54 self.cover_url = re.sub(r'\bw\d+\b', 'w1000', self.cover_url) # enlarge cover size resolution
55 except Exception:
56 '''
57 failed, img src might have changed, use default width 200
58 '''
59 pass
60
61 section_tags = soup.find_all('section', attrs={'data-container-type': 'issue-section-list'})
62
63 sections = defaultdict(list)
64 ordered_sec_titles = []
65 index = []
66
67 for sec in section_tags:
68 sec_title = self.tag_to_string(sec.find('h2'))
69 ordered_sec_titles.append(sec_title)
70 for article in sec.findAll('article'):
71 try:
72 url = absurl(article.find('a', {'itemprop': check_words('url')})['href'])
73 except TypeError:
74 continue
75 title = self.tag_to_string(article.find('h3', {'itemprop': has_all_of('name headline')}))
76 date = ' [' + self.tag_to_string(article.find('time', {'itemprop': check_words('datePublished')})) + ']'
77 author = self.tag_to_string(article.find('li', {'itemprop': check_words('creator')}))
78 description = self.tag_to_string(article.find(attrs={'data-test': check_words('article.type')})) + ' • '
79 description += self.tag_to_string(article.find('div', attrs={'itemprop': check_words('description')}))
80 sections[sec_title].append({'title': title, 'url': url, 'description': description, 'date': date, 'author': author})
81
82 for k in ordered_sec_titles:
83 index.append((k, sections[k]))
84 return index
85
86 def preprocess_html(self, soup):
87 for img in soup.findAll('img', {'data-src': True}):
88 if img['data-src'].startswith('//'):
89 img['src'] = 'https:' + img['data-src']
90 else:
91 img['src'] = img['data-src']
92 for div in soup.findAll('div', {'data-component': check_words('article-container')})[1:]:
93 div.extract()
94 return soup