Debian Codegraph @

recipes/science_journal.recipe

raw
1 #!/usr/bin/env python
2 from calibre.web.feeds.news import BasicNewsRecipe, classes, prefixed_classes
3
4
5 def absurl(url):
6 if url.startswith('/'):
7 url = 'https://www.science.org' + url
8 return url
9
10
11 class science(BasicNewsRecipe):
12 title = 'Science Journal'
13 __author__ = 'unkn0wn'
14 description = (
15 'Science continues to publish the very best in research across the sciences, with articles that consistently rank among the most cited in the world.'
16 )
17 encoding = 'utf-8'
18 no_javascript = True
19 no_stylesheets = True
20 remove_attributes = ['style', 'height', 'width']
21 masthead_url = 'https://www.science.org/pb-assets/images/styleguide/logo-1672180580750.svg'
22 language = 'en'
23 simultaneous_downloads = 1
24 browser_type = 'webengine'
25
26 extra_css = '''
27 .news-article__figure__caption, .calibre-nuked-tag-figcaption, .card-related {font-size:small;}
28 .core-self-citation, .meta-panel__left-content, .news-article__hero__top-meta {font-size:small;}
29 .contributors, .news-article__hero__bottom-meta, #bibliography, #elettersSection {font-size:small;}
30 img {display:block; margin:0 auto;}
31 .core-lede {font-style:italic; color:#202020;}
32 '''
33
34 ignore_duplicate_articles = {'url'}
35
36 keep_only_tags = [
37 classes('meta-panel__left-content news-article__hero__info news-article__hero__figure bodySection'),
38 dict(name='h1', attrs={'property': 'name'}),
39 dict(name='div', **classes('core-lede contributors core-self-citation')),
40 dict(attrs={'data-core-wrapper': 'content'}),
41 ]
42
43 remove_tags = [classes('pb-ad news-article__hero__scroller news-article__version-of-story')]
44
45 recipe_specific_options = {
46 'issue': {
47 'short': 'Enter the Issue Number you want to download\n(Vol/Issue format)',
48 'long': 'For example, 385/6710',
49 'default': 'current',
50 },
51 'res': {
52 'short': 'For hi-res images, select a resolution from the\nfollowing options: 800, 1000, 1200 or 1500',
53 'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use 400 or 300.',
54 'default': '600',
55 },
56 }
57
58 def preprocess_html(self, soup):
59 for img in soup.findAll('img', attrs={'src': True}):
60 if img['src'].endswith('.jpg'):
61 res = '/cdn-cgi/image/width=600'
62 w = self.recipe_specific_options.get('res')
63 if w and isinstance(w, str):
64 res = '/cdn-cgi/image/width=' + w
65 img['src'] = absurl(res + img['src'])
66 for div in soup.findAll('div', attrs={'role': 'paragraph'}):
67 div.name = 'p'
68 return soup
69
70 def postprocess_html(self, soup, first_fetch):
71 bd = soup.find('body')
72 if bd:
73 bd.attrs = {}
74 return soup
75
76 def parse_index(self):
77 issue_url = 'https://www.science.org/toc/science/current'
78 d = self.recipe_specific_options.get('issue')
79 if d and isinstance(d, str):
80 issue_url = 'https://www.science.org/toc/science/' + d
81
82 soup = self.index_to_soup(issue_url)
83 tme = soup.find(**classes('journal-issue__vol'))
84 if tme:
85 self.timefmt = ' [%s]' % self.tag_to_string(tme).strip().replace('|', ' | ')
86 det = soup.find(attrs={'id': 'journal-issue-details'})
87 if det:
88 self.description = self.tag_to_string(det).strip()
89 cov = soup.find(**classes('cover-image__image'))
90 if cov:
91 self.cover_url = absurl('/cdn-cgi/image/width=800' + cov.img['src'])
92
93 feeds = []
94
95 for sec in soup.findAll('section', **prefixed_classes('toc__section')):
96 name = sec.find(**classes('sidebar-article-title--decorated'))
97 section = self.tag_to_string(name).strip()
98 self.log(section)
99
100 articles = []
101
102 for card in sec.findAll(**classes('card-header')):
103 ti = card.find(**classes('article-title'))
104 url = absurl(ti.a['href'])
105 title = self.tag_to_string(ti).strip()
106 desc = ''
107 meta = card.find(**classes('card-meta'))
108 if meta:
109 desc = self.tag_to_string(meta).strip()
110 self.log(' ', title, '\n\t', desc, '\n\t\t', url)
111 articles.append({'title': title, 'description': desc, 'url': url})
112 feeds.append((section, articles))
113 return feeds