Debian Codegraph @

recipes/new_scientist.recipe

raw
1 ##
2 # Title: New Scientist RSS recipe
3 # Contact: AprilHare, Darko Miletic <darko.miletic at gmail.com>
4 ##
5 # License: GNU General Public License v3 - http://www.gnu.org/copyleft/gpl.html
6 # Copyright: 2008-2016, AprilHare, Darko Miletic <darko.miletic at gmail.com>
7 ##
8 # Written: 2008
9 # Last Edited: Jan 2016
10 ##
11
12 """
13 01-19-2012: Added GrayScale Image conversion and Duplicant article removals
14 12-31-2015: Major rewrite due to massive changes in site structure
15 01-27-2016: Added support for series index and minor cleanup
16 """
17
18 __license__ = 'GNU General Public License v3 - http://www.gnu.org/copyleft/gpl.html'
19 __copyright__ = '2008-2016, AprilHare, Darko Miletic <darko.miletic at gmail.com>'
20 __version__ = 'v0.6.1'
21 __date__ = '2016-01-27'
22 __author__ = 'Darko Miletic'
23
24 '''
25 newscientist.com
26 '''
27
28 from calibre.web.feeds.news import BasicNewsRecipe
29
30
31 def classes(classes):
32 q = frozenset(classes.split(' '))
33 return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)})
34
35
36 class NewScientist(BasicNewsRecipe):
37 title = 'New Scientist - Online News w. subscription'
38 description = 'Science news and science articles from New Scientist, based on feeds.'
39 language = 'en'
40 publisher = 'Reed Business Information Ltd.'
41 category = 'science news, science articles, science jobs, drugs, cancer, depression, computer software'
42 oldest_article = 15
43 max_articles_per_feed = 100
44 no_stylesheets = True
45 use_embedded_content = False
46 encoding = 'utf-8'
47 needs_subscription = 'optional'
48 remove_empty_feeds = True
49 ignore_duplicate_articles = {'url'}
50 resolve_internal_links = True
51 remove_attributes = ['style', 'height', 'width']
52 masthead_url = 'https://cdn.shopify.com/s/files/1/0266/6843/3505/files/logo.svg?v=1629189295'
53
54 conversion_options = {'comment': description, 'tags': category, 'publisher': publisher, 'language': language}
55
56 extra_css = '''
57 img {display:block; margin:0 auto;}
58 .ArticleHeader__Category { font-size:small; color:#404040; }
59 .ArticleHeader__Author, .ArticleHeader__DateTimeWrapper { font-size:small; }
60 .ArticleHeader__Copy { font-style:italic; color:#202020; }
61 .ArticleImage { font-size:small; text-align:center; }
62 .ArticleImageCaption__Credit { font-size:smaller; }
63 '''
64
65 keep_only_tags = [classes('ArticleHeader ArticleContent')]
66
67 remove_tags = [dict(name=['svg', 'button']), classes('ArticleHeader__SocialWrapper AdvertWrapper ReadMoreWithImage ArticleTopics')]
68
69 def preprocess_html(self, soup):
70 time = soup.find(**classes('ArticleHeader__DateTimeWrapper'))
71 if time:
72 time.name = 'div'
73 for img in soup.findAll('img', attrs={'data-src': True}):
74 img['src'] = img['data-src'].split('?')[0] + '?width=700'
75 for figc in soup.findAll('figcaption'):
76 for p in figc.findAll('p'):
77 p.name = 'div'
78 return soup
79
80 def get_article_url(self, article):
81 ans = BasicNewsRecipe.get_article_url(self, article)
82 return ans.partition('?')[0]
83
84 def print_version(self, url):
85 if '/video/' in url:
86 return None
87 return url
88
89 def get_browser(self):
90 br = BasicNewsRecipe.get_browser(self)
91 if self.username is not None and self.password is not None:
92
93 def is_login_form(form):
94 return 'action' in form.attrs and form.attrs['action'] == '/login/'
95
96 br.open('https://www.newscientist.com/login/')
97 br.select_form(predicate=is_login_form)
98 br['email'] = self.username
99 br['password'] = self.password
100 res = br.submit().read()
101 if b'>Your account<' not in res:
102 raise ValueError('Failed to log in to New Scientist, check your username and password')
103 return br
104
105 feeds = [
106 ('News', 'https://www.newscientist.com/section/news/feed/'),
107 ('Features', 'https://www.newscientist.com/section/features/feed/'),
108 ('Physics', 'https://www.newscientist.com/subject/physics/feed/'),
109 ('Technology', 'https://www.newscientist.com/subject/technology/feed/'),
110 ('Space', 'https://www.newscientist.com/subject/space/feed/'),
111 ('Life', 'https://www.newscientist.com/subject/life/feed/'),
112 ('Earth', 'https://www.newscientist.com/subject/earth/feed/'),
113 ('Health', 'https://www.newscientist.com/subject/health/feed/'),
114 ('Humans', 'https://www.newscientist.com/subject/humans/feed/'),
115 ]
116
117 def get_cover_url(self):
118 soup = self.index_to_soup('https://www.newscientist.com/issues/current/')
119 div = soup.find('div', attrs={'class': 'ThisWeeksMagazineHero__CoverInfo'})
120 return div.find(**classes('ThisWeeksMagazineHero__ImageLink')).img['src']