| 1 |
#!/usr/bin/env python |
| 2 |
__license__ = 'GPL v3' |
| 3 |
|
| 4 |
import json |
| 5 |
from datetime import datetime |
| 6 |
from urllib.parse import urljoin |
| 7 |
|
| 8 |
from calibre.web.feeds.news import BasicNewsRecipe, prefixed_classes |
| 9 |
|
| 10 |
|
| 11 |
class ScientificAmerican(BasicNewsRecipe): |
| 12 |
title = 'Scientific American' |
| 13 |
description = 'Popular Science. Monthly magazine. Should be downloaded around the middle of each month.' |
| 14 |
category = 'science' |
| 15 |
__author__ = 'Kovid Goyal' |
| 16 |
no_stylesheets = True |
| 17 |
language = 'en' |
| 18 |
publisher = 'Nature Publishing Group' |
| 19 |
remove_empty_feeds = True |
| 20 |
remove_javascript = True |
| 21 |
timefmt = ' [%B %Y]' |
| 22 |
remove_attributes = ['height', 'width'] |
| 23 |
masthead_url = 'https://static.scientificamerican.com/sciam/assets/Image/newsletter/salogo.png' |
| 24 |
extra_css = ''' |
| 25 |
[class^="default_dek-"], [class^="short_dek-"] { font-style:italic; color:#202020; } |
| 26 |
[class^="article_authors-"] {font-size:small; color:#202020; } |
| 27 |
[class^="lead_image_"], .calibre-nuked-tag-figcaption { font-size:small; text-align:center; } |
| 28 |
[class^="bio-"] { font-size:small; color:#404040; } |
| 29 |
em, blockquote { color:#202020; } |
| 30 |
img {display:block; margin:0 auto; } |
| 31 |
''' |
| 32 |
|
| 33 |
needs_subscription = 'optional' |
| 34 |
|
| 35 |
keep_only_tags = [ |
| 36 |
prefixed_classes('default_hed- default_dek- short_hed- short_dek- article_authors- lead_image_ article__body- bio-'), |
| 37 |
] |
| 38 |
remove_tags = [dict(name=['button', 'svg', 'iframe', 'source'])] |
| 39 |
|
| 40 |
def preprocess_html(self, soup): |
| 41 |
for a in soup.findAll('a', href=lambda h: h and 'getsciam' in h): |
| 42 |
p = a.find_parent('p') |
| 43 |
if not p: |
| 44 |
continue |
| 45 |
prev_h2 = p.find_previous_sibling('h2') |
| 46 |
if prev_h2 and 'supporting science journalism' in prev_h2.get_text().lower(): |
| 47 |
for tag in [p.find_previous_sibling('hr'), prev_h2, p.find_next_sibling('hr'), p]: |
| 48 |
if tag: |
| 49 |
tag.decompose() |
| 50 |
for h2 in soup.findAll(['h2', 'h3']): |
| 51 |
h2.name = 'h4' |
| 52 |
for fig in soup.findAll('figcaption'): |
| 53 |
for p in fig.findAll('p'): |
| 54 |
p.name = 'div' |
| 55 |
res = '?w=600' |
| 56 |
w = self.recipe_specific_options.get('res') |
| 57 |
if w and isinstance(w, str): |
| 58 |
res = '?w=' + w |
| 59 |
for img in soup.findAll('img', src=True): |
| 60 |
img['src'] = img['src'].split('?')[0] + res |
| 61 |
return soup |
| 62 |
|
| 63 |
def get_browser(self, *args): |
| 64 |
br = BasicNewsRecipe.get_browser(self) |
| 65 |
if self.username and self.password: |
| 66 |
br.open('https://www.scientificamerican.com/account/login/') |
| 67 |
br.select_form(predicate=lambda f: f.attrs.get('id') == 'login') |
| 68 |
br['emailAddress'] = self.username |
| 69 |
br['password'] = self.password |
| 70 |
br.submit() |
| 71 |
return br |
| 72 |
|
| 73 |
recipe_specific_options = { |
| 74 |
'issue_url': { |
| 75 |
'short': 'The issue URL ', |
| 76 |
'long': ( |
| 77 |
'For example, https://www.scientificamerican.com/issue/sa/2024/07-01/' |
| 78 |
'\nYou can also download special-editions, physics, health, mind magazines by pasting the URL here.' |
| 79 |
), |
| 80 |
}, |
| 81 |
'res': { |
| 82 |
'short': 'For hi-res images, select a resolution from the\nfollowing options: 800, 1000, 1200 or 1500', |
| 83 |
'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use 400 or 300.', |
| 84 |
'default': '600', |
| 85 |
}, |
| 86 |
} |
| 87 |
|
| 88 |
def parse_index(self): |
| 89 |
# Get the cover, date and issue URL |
| 90 |
d = self.recipe_specific_options.get('issue_url') |
| 91 |
if d and isinstance(d, str): |
| 92 |
issue = d |
| 93 |
else: |
| 94 |
fp_soup = self.index_to_soup('https://www.scientificamerican.com') |
| 95 |
curr_issue_link = fp_soup.find(**prefixed_classes('latest_issue_links-')) |
| 96 |
if not curr_issue_link: |
| 97 |
self.abort_recipe_processing('Unable to find issue link') |
| 98 |
issue = 'https://www.scientificamerican.com' + curr_issue_link.a['href'] |
| 99 |
|
| 100 |
soup = self.index_to_soup(issue) |
| 101 |
script = soup.find('script', id='__DATA__') |
| 102 |
if not script: |
| 103 |
self.abort_recipe_processing('Unable to find script') |
| 104 |
|
| 105 |
JSON = script.contents[0].split('JSON.parse(`')[1].replace('\\\\', '\\') |
| 106 |
data = json.JSONDecoder().raw_decode(JSON)[0] |
| 107 |
issue_info = data.get('initialData', {}).get('issueData', {}) |
| 108 |
if not issue_info: |
| 109 |
self.abort_recipe_processing('Unable to find issue info') |
| 110 |
|
| 111 |
self.cover_url = issue_info['image_url'] + '?w=800' |
| 112 |
|
| 113 |
edition_date = datetime.strptime(issue_info['issue_date'], '%Y-%m-%d') |
| 114 |
self.timefmt = f' [{edition_date:%B %Y}]' |
| 115 |
|
| 116 |
feeds = {} |
| 117 |
for section in issue_info.get('article_previews', {}): |
| 118 |
for article in issue_info.get('article_previews', {}).get(section, []): |
| 119 |
self.log('\t', article['title']) |
| 120 |
feed_name = article['column'] |
| 121 |
if feed_name not in feeds: |
| 122 |
feeds[feed_name] = [] |
| 123 |
feeds[feed_name].append({ |
| 124 |
'title': article['title'], |
| 125 |
'url': urljoin( |
| 126 |
'https://www.scientificamerican.com/article/', |
| 127 |
article['slug'], |
| 128 |
), |
| 129 |
'description': article['summary'], |
| 130 |
'page_number': int(article['page_number']), |
| 131 |
}) |
| 132 |
# articles are already sorted by page number within a feed, but feeds are not in order |
| 133 |
sorted_feeds = dict(sorted(feeds.items(), key=lambda x: x[1][0]['page_number'])) |
| 134 |
return sorted_feeds.items() |