Debian Codegraph @

recipes/new_scientist_mag.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 """
4 newscientist.com
5 """
6
7 from calibre.web.feeds.news import BasicNewsRecipe, classes
8
9
10 class NewScientist(BasicNewsRecipe):
11 title = 'New Scientist Magazine'
12 __author__ = 'unkn0wn'
13 description = (
14 'New Scientist is the world’s most popular weekly science and technology publication. '
15 'We cover international news from a scientific standpoint, and ask the big-picture questions '
16 'about life, the universe and what it means to be human. If someone in the world has a good idea, '
17 'you will read about it in New Scientist.'
18 )
19 language = 'en'
20 publisher = 'Reed Business Information Ltd.'
21 category = 'science news, science articles, science jobs, drugs, cancer, depression, computer software'
22 no_stylesheets = True
23 use_embedded_content = False
24 encoding = 'utf-8'
25 needs_subscription = 'optional'
26 remove_empty_feeds = True
27 ignore_duplicate_articles = {'url'}
28 resolve_internal_links = True
29 remove_attributes = ['style', 'height', 'width']
30 masthead_url = 'https://cdn.shopify.com/s/files/1/0266/6843/3505/files/logo.svg?v=1629189295'
31
32 conversion_options = {'comment': description, 'tags': category, 'publisher': publisher, 'language': language}
33
34 def get_browser(self):
35 br = BasicNewsRecipe.get_browser(self)
36 if self.username is not None and self.password is not None:
37
38 def is_login_form(form):
39 return 'action' in form.attrs and form.attrs['action'] == '/login/'
40
41 br.open('https://www.newscientist.com/login/')
42 br.select_form(predicate=is_login_form)
43 br['email'] = self.username
44 br['password'] = self.password
45 res = br.submit().read()
46 if b'>Your account<' not in res:
47 raise ValueError('Failed to log in to New Scientist, check your username and password')
48 return br
49
50 # def print_version(self, url):
51 # return 'https://webcache.googleusercontent.com/search?q=cache:' + url.split('?')[0]
52
53 extra_css = '''
54 img {display:block; margin:0 auto;}
55 .article__category { font-size:small; color:#404040; }
56 .article__byline, .article__date { font-size:small; }
57 .article__excerpt { font-style:italic; color:#202020; }
58 figcaption { font-size:small; text-align:center; }
59 .wp-image__caption--credit { font-size:smaller; }
60 '''
61
62 keep_only_tags = [classes('article__category article__title article__excerpt article__byline article__date article__content')]
63
64 remove_tags = [
65 dict(name=['svg', 'button']),
66 classes('advert-wrapper read-more__standard wp-block-newscientist-newsletter-promotion js-content-prompt-opportunity'),
67 ]
68
69 recipe_specific_options = {'issue': {'short': 'Enter the Issue Number you want to download ', 'long': 'For example, 3498'}}
70
71 def parse_index(self):
72 issue_url = 'https://www.newscientist.com/issues/current/'
73 d = self.recipe_specific_options.get('issue')
74 if d and isinstance(d, str):
75 issue_url = 'https://www.newscientist.com/issue/' + d
76
77 soup = self.index_to_soup(issue_url)
78 issue_date = soup.find(**classes('issue__date'))
79 issue_number = soup.find(**classes('issue__number'))
80 cover = soup.find(**classes('issue__cover'))
81 if issue_date is None or issue_number is None or cover is None or cover.img is None:
82 raise ValueError('Failed to find issue metadata on New Scientist')
83 issue = ' - '.join((self.tag_to_string(issue_date).strip(), self.tag_to_string(issue_number).strip()))
84 self.log('Downloading issue:', issue)
85 self.timefmt = ' [' + issue + ']'
86 self.cover_url = cover.img['src']
87
88 feeds = []
89 for cont in soup.findAll(**classes('issue__table-of-contents-section')):
90 sec = self.tag_to_string(cont.find('h3')).strip()
91 self.log(sec)
92 articles = []
93 for a in cont.findAll('a', **classes('content-item')):
94 url = a.get('href')
95 title_tag = a.find(**classes('content-item__title'))
96 if not url or title_tag is None:
97 continue
98 if not url.startswith('http'):
99 url = 'https://www.newscientist.com' + url
100 title = self.tag_to_string(title_tag).strip()
101 desc = []
102 for x in (
103 a.find(**classes('content-item__subject')),
104 a.find(**classes('content-item__tag')),
105 a.find(**classes('content-item__excerpt')),
106 ):
107 text = self.tag_to_string(x).strip()
108 if text:
109 desc.append(text)
110 desc = ' | '.join(desc)
111 self.log('\t', title, '\n\t', desc, '\n\t\t', url)
112 articles.append({'title': title, 'description': desc, 'url': url})
113 if articles:
114 feeds.append((sec, articles))
115 return feeds
116
117 def preprocess_html(self, soup):
118 time = soup.find('time', **classes('article__pubdate'))
119 if time:
120 time.name = 'div'
121 for img in soup.findAll('img', attrs={'data-src': True}):
122 img['src'] = img['data-src'].split('?')[0] + '?width=700'
123 for figc in soup.findAll('figcaption'):
124 for p in figc.findAll('p'):
125 p.name = 'div'
126 return soup