Debian Codegraph @

recipes/mit_technology_review.recipe

raw
1 #!/usr/bin/env python
2 from __future__ import unicode_literals
3
4 __license__ = 'GPL v3'
5 __copyright__ = '2015 Michael Marotta <mikefm at gmail.net>'
6 # Written April 2015
7 # Last edited 07/2024
8 '''
9 technologyreview.com
10 '''
11 import json
12 import re
13 from collections import OrderedDict
14
15 from calibre.web.feeds.news import BasicNewsRecipe, prefixed_classes
16
17
18 def absurl(x):
19 if x.startswith('//'):
20 x = 'http:' + x
21 elif not x.startswith('http'):
22 x = 'http://www.technologyreview.com' + x
23 return x
24
25
26 def classes(classes):
27 q = frozenset(classes.split(' '))
28 return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)})
29
30
31 class MitTechnologyReview(BasicNewsRecipe):
32 title = 'MIT Technology Review Magazine'
33 __author__ = 'Michael Marotta, revised by unkn0wn'
34 description = (
35 'Bi-monthly magazine version of MIT Technology Review.'
36 ' This is different than the recipe named simply "Technology Review"'
37 ' which downloads the rss feed with daily articles from the website.'
38 )
39 language = 'en'
40 encoding = 'utf-8'
41 tags = 'news, technology, science'
42 no_stylesheets = True
43 remove_empty_feeds = True
44 remove_attributes = ['height', 'width', 'style', 'padding', 'padding-top']
45 delay = 1
46 masthead_url = 'https://wp-preprod.technologyreview.com/wp-content/uploads/2021/08/Screen-Shot-2021-08-20-at-11.11.12-AM-e1629473232355.png'
47 extra_css = '''
48 #pub-d{font-size:small;}
49 #cre-d{font-size:xx-small; text-align:center; color:gray;}
50 #cap-d{font-size:small; text-align:center;}
51 blockquote{text-align:center; color:#404040;}
52 em { color:#202020;}
53 '''
54 keep_only_tags = [prefixed_classes('contentHeader contentArticleHeader contentBody')]
55 remove_tags = [
56 dict(name='aside'),
57 dict(name='svg'),
58 prefixed_classes('image__placeholder sliderAd__wrapper eyebrow__wrap-- screen-reader-text'),
59 ]
60
61 recipe_specific_options = {
62 'issue_url': {
63 'short': 'The issue URL ',
64 'long': 'For example, https://www.technologyreview.com/magazines/the-education-issue/',
65 'default': 'http://www.technologyreview.com/magazine/',
66 }
67 }
68
69 def parse_index(self):
70 # for past editions, change the issue link below
71 issue = 'http://www.technologyreview.com/magazine/'
72 d = self.recipe_specific_options.get('issue_url')
73 if d and isinstance(d, str):
74 issue = d
75 soup = self.index_to_soup(issue)
76 if script := soup.find('script', id='preload'):
77 raw = script.contents[0]
78 m = re.search(r'\"children\":\[{\"name\":\"magazine-hero\"', raw)
79 spl = re.split(r'(?=\{)', raw[m.start() :], 1)[1]
80 data = json.JSONDecoder().raw_decode(spl)[0]
81 self.cover_url = data['children'][0]['config']['src'] + '?fit=572,786'
82 self.timefmt = ' [' + data['config']['issueDate'] + ']'
83 self.description = data['config']['description']
84 self.title = 'MIT TR: ' + data['config']['title']
85
86 # parse articles
87 feeds = OrderedDict()
88
89 classNamePrefixes = ['magazineHero__letter--', 'teaserItem__title', 'teaserItem--aside__title']
90 for div in soup.findAll(attrs={'class': lambda x: any(x.startswith(prefix) for prefix in classNamePrefixes) if x else False}):
91 articles = []
92 a = div.find('a', href=True)
93 title = self.tag_to_string(a).strip()
94 href = absurl(a['href'])
95 desc = ''
96 section_title = 'Letter from the editor'
97 d = div.findParent(attrs={'class': lambda z: z and z.startswith(('teaserItem__wrapper', 'teaserItem--aside__wrapper'))})
98 if d:
99 excerpt = d.find(attrs={'class': lambda x: x and x.startswith(('teaserItem__excerpt', 'teaserItem--aside__excerpt'))})
100 if excerpt:
101 desc = self.tag_to_string(excerpt).strip()
102
103 sec = d.find(attrs={'class': lambda x: x and x.startswith('teaserItem__eyebrowText')})
104 if sec:
105 section_title = self.tag_to_string(sec).replace('Categorized in ', '').strip()
106
107 if not href or not title:
108 continue
109
110 self.log(section_title)
111 self.log('\t', title)
112 self.log('\t', desc)
113 self.log('\t\t', href)
114
115 articles.append({'title': title, 'url': href, 'description': desc})
116 if articles:
117 if section_title not in feeds:
118 feeds[section_title] = []
119 feeds[section_title] += articles
120 ans = list(feeds.items())
121 return ans
122
123 def preprocess_html(self, soup):
124 for bq in soup.findAll('blockquote'):
125 for strong in bq.findAll('strong'):
126 strong.name = 'div'
127 for date in soup.findAll(attrs={'class': lambda x: x and x.startswith(('contentArticleHeader__publishDate', 'byline__wrapper'))}):
128 date['id'] = 'pub-d'
129 for li in date.findAll(('li', 'ul')):
130 li.name = 'span'
131 for cap in soup.findAll('figcaption'):
132 cap['id'] = 'cap-d'
133 for credit in soup.findAll(attrs={'class': lambda x: x and x.startswith(('image__credit', 'image-credit'))}):
134 credit['id'] = 'cre-d'
135 for img in soup.findAll(srcset=True):
136 img['src'] = absurl(img['srcset'].split()[0])
137 del img['srcset']
138 for img in soup.findAll('img', attrs={'src': True}):
139 img['src'] = img['src'].split('?')[0] + '?w=800'
140 return soup