Debian Codegraph @

recipes/scientific_american.recipe

raw
1 #!/usr/bin/env python
2 __license__ = 'GPL v3'
3
4 import json
5 from datetime import datetime
6 from urllib.parse import urljoin
7
8 from calibre.web.feeds.news import BasicNewsRecipe, prefixed_classes
9
10
11 class ScientificAmerican(BasicNewsRecipe):
12 title = 'Scientific American'
13 description = 'Popular Science. Monthly magazine. Should be downloaded around the middle of each month.'
14 category = 'science'
15 __author__ = 'Kovid Goyal'
16 no_stylesheets = True
17 language = 'en'
18 publisher = 'Nature Publishing Group'
19 remove_empty_feeds = True
20 remove_javascript = True
21 timefmt = ' [%B %Y]'
22 remove_attributes = ['height', 'width']
23 masthead_url = 'https://static.scientificamerican.com/sciam/assets/Image/newsletter/salogo.png'
24 extra_css = '''
25 [class^="default_dek-"], [class^="short_dek-"] { font-style:italic; color:#202020; }
26 [class^="article_authors-"] {font-size:small; color:#202020; }
27 [class^="lead_image_"], .calibre-nuked-tag-figcaption { font-size:small; text-align:center; }
28 [class^="bio-"] { font-size:small; color:#404040; }
29 em, blockquote { color:#202020; }
30 img {display:block; margin:0 auto; }
31 '''
32
33 needs_subscription = 'optional'
34
35 keep_only_tags = [
36 prefixed_classes('default_hed- default_dek- short_hed- short_dek- article_authors- lead_image_ article__body- bio-'),
37 ]
38 remove_tags = [dict(name=['button', 'svg', 'iframe', 'source'])]
39
40 def preprocess_html(self, soup):
41 for a in soup.findAll('a', href=lambda h: h and 'getsciam' in h):
42 p = a.find_parent('p')
43 if not p:
44 continue
45 prev_h2 = p.find_previous_sibling('h2')
46 if prev_h2 and 'supporting science journalism' in prev_h2.get_text().lower():
47 for tag in [p.find_previous_sibling('hr'), prev_h2, p.find_next_sibling('hr'), p]:
48 if tag:
49 tag.decompose()
50 for h2 in soup.findAll(['h2', 'h3']):
51 h2.name = 'h4'
52 for fig in soup.findAll('figcaption'):
53 for p in fig.findAll('p'):
54 p.name = 'div'
55 res = '?w=600'
56 w = self.recipe_specific_options.get('res')
57 if w and isinstance(w, str):
58 res = '?w=' + w
59 for img in soup.findAll('img', src=True):
60 img['src'] = img['src'].split('?')[0] + res
61 return soup
62
63 def get_browser(self, *args):
64 br = BasicNewsRecipe.get_browser(self)
65 if self.username and self.password:
66 br.open('https://www.scientificamerican.com/account/login/')
67 br.select_form(predicate=lambda f: f.attrs.get('id') == 'login')
68 br['emailAddress'] = self.username
69 br['password'] = self.password
70 br.submit()
71 return br
72
73 recipe_specific_options = {
74 'issue_url': {
75 'short': 'The issue URL ',
76 'long': (
77 'For example, https://www.scientificamerican.com/issue/sa/2024/07-01/'
78 '\nYou can also download special-editions, physics, health, mind magazines by pasting the URL here.'
79 ),
80 },
81 'res': {
82 'short': 'For hi-res images, select a resolution from the\nfollowing options: 800, 1000, 1200 or 1500',
83 'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use 400 or 300.',
84 'default': '600',
85 },
86 }
87
88 def parse_index(self):
89 # Get the cover, date and issue URL
90 d = self.recipe_specific_options.get('issue_url')
91 if d and isinstance(d, str):
92 issue = d
93 else:
94 fp_soup = self.index_to_soup('https://www.scientificamerican.com')
95 curr_issue_link = fp_soup.find(**prefixed_classes('latest_issue_links-'))
96 if not curr_issue_link:
97 self.abort_recipe_processing('Unable to find issue link')
98 issue = 'https://www.scientificamerican.com' + curr_issue_link.a['href']
99
100 soup = self.index_to_soup(issue)
101 script = soup.find('script', id='__DATA__')
102 if not script:
103 self.abort_recipe_processing('Unable to find script')
104
105 JSON = script.contents[0].split('JSON.parse(`')[1].replace('\\\\', '\\')
106 data = json.JSONDecoder().raw_decode(JSON)[0]
107 issue_info = data.get('initialData', {}).get('issueData', {})
108 if not issue_info:
109 self.abort_recipe_processing('Unable to find issue info')
110
111 self.cover_url = issue_info['image_url'] + '?w=800'
112
113 edition_date = datetime.strptime(issue_info['issue_date'], '%Y-%m-%d')
114 self.timefmt = f' [{edition_date:%B %Y}]'
115
116 feeds = {}
117 for section in issue_info.get('article_previews', {}):
118 for article in issue_info.get('article_previews', {}).get(section, []):
119 self.log('\t', article['title'])
120 feed_name = article['column']
121 if feed_name not in feeds:
122 feeds[feed_name] = []
123 feeds[feed_name].append({
124 'title': article['title'],
125 'url': urljoin(
126 'https://www.scientificamerican.com/article/',
127 article['slug'],
128 ),
129 'description': article['summary'],
130 'page_number': int(article['page_number']),
131 })
132 # articles are already sorted by page number within a feed, but feeds are not in order
133 sorted_feeds = dict(sorted(feeds.items(), key=lambda x: x[1][0]['page_number']))
134 return sorted_feeds.items()