| 1 |
#!/usr/bin/env python |
| 2 |
from __future__ import unicode_literals |
| 3 |
|
| 4 |
__license__ = 'GPL v3' |
| 5 |
__copyright__ = '2015 Michael Marotta <mikefm at gmail.net>' |
| 6 |
# Written April 2015 |
| 7 |
# Last edited 07/2024 |
| 8 |
''' |
| 9 |
technologyreview.com |
| 10 |
''' |
| 11 |
import json |
| 12 |
import re |
| 13 |
from collections import OrderedDict |
| 14 |
|
| 15 |
from calibre.web.feeds.news import BasicNewsRecipe, prefixed_classes |
| 16 |
|
| 17 |
|
| 18 |
def absurl(x): |
| 19 |
if x.startswith('//'): |
| 20 |
x = 'http:' + x |
| 21 |
elif not x.startswith('http'): |
| 22 |
x = 'http://www.technologyreview.com' + x |
| 23 |
return x |
| 24 |
|
| 25 |
|
| 26 |
def classes(classes): |
| 27 |
q = frozenset(classes.split(' ')) |
| 28 |
return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)}) |
| 29 |
|
| 30 |
|
| 31 |
class MitTechnologyReview(BasicNewsRecipe): |
| 32 |
title = 'MIT Technology Review Magazine' |
| 33 |
__author__ = 'Michael Marotta, revised by unkn0wn' |
| 34 |
description = ( |
| 35 |
'Bi-monthly magazine version of MIT Technology Review.' |
| 36 |
' This is different than the recipe named simply "Technology Review"' |
| 37 |
' which downloads the rss feed with daily articles from the website.' |
| 38 |
) |
| 39 |
language = 'en' |
| 40 |
encoding = 'utf-8' |
| 41 |
tags = 'news, technology, science' |
| 42 |
no_stylesheets = True |
| 43 |
remove_empty_feeds = True |
| 44 |
remove_attributes = ['height', 'width', 'style', 'padding', 'padding-top'] |
| 45 |
delay = 1 |
| 46 |
masthead_url = 'https://wp-preprod.technologyreview.com/wp-content/uploads/2021/08/Screen-Shot-2021-08-20-at-11.11.12-AM-e1629473232355.png' |
| 47 |
extra_css = ''' |
| 48 |
#pub-d{font-size:small;} |
| 49 |
#cre-d{font-size:xx-small; text-align:center; color:gray;} |
| 50 |
#cap-d{font-size:small; text-align:center;} |
| 51 |
blockquote{text-align:center; color:#404040;} |
| 52 |
em { color:#202020;} |
| 53 |
''' |
| 54 |
keep_only_tags = [prefixed_classes('contentHeader contentArticleHeader contentBody')] |
| 55 |
remove_tags = [ |
| 56 |
dict(name='aside'), |
| 57 |
dict(name='svg'), |
| 58 |
prefixed_classes('image__placeholder sliderAd__wrapper eyebrow__wrap-- screen-reader-text'), |
| 59 |
] |
| 60 |
|
| 61 |
recipe_specific_options = { |
| 62 |
'issue_url': { |
| 63 |
'short': 'The issue URL ', |
| 64 |
'long': 'For example, https://www.technologyreview.com/magazines/the-education-issue/', |
| 65 |
'default': 'http://www.technologyreview.com/magazine/', |
| 66 |
} |
| 67 |
} |
| 68 |
|
| 69 |
def parse_index(self): |
| 70 |
# for past editions, change the issue link below |
| 71 |
issue = 'http://www.technologyreview.com/magazine/' |
| 72 |
d = self.recipe_specific_options.get('issue_url') |
| 73 |
if d and isinstance(d, str): |
| 74 |
issue = d |
| 75 |
soup = self.index_to_soup(issue) |
| 76 |
if script := soup.find('script', id='preload'): |
| 77 |
raw = script.contents[0] |
| 78 |
m = re.search(r'\"children\":\[{\"name\":\"magazine-hero\"', raw) |
| 79 |
spl = re.split(r'(?=\{)', raw[m.start() :], 1)[1] |
| 80 |
data = json.JSONDecoder().raw_decode(spl)[0] |
| 81 |
self.cover_url = data['children'][0]['config']['src'] + '?fit=572,786' |
| 82 |
self.timefmt = ' [' + data['config']['issueDate'] + ']' |
| 83 |
self.description = data['config']['description'] |
| 84 |
self.title = 'MIT TR: ' + data['config']['title'] |
| 85 |
|
| 86 |
# parse articles |
| 87 |
feeds = OrderedDict() |
| 88 |
|
| 89 |
classNamePrefixes = ['magazineHero__letter--', 'teaserItem__title', 'teaserItem--aside__title'] |
| 90 |
for div in soup.findAll(attrs={'class': lambda x: any(x.startswith(prefix) for prefix in classNamePrefixes) if x else False}): |
| 91 |
articles = [] |
| 92 |
a = div.find('a', href=True) |
| 93 |
title = self.tag_to_string(a).strip() |
| 94 |
href = absurl(a['href']) |
| 95 |
desc = '' |
| 96 |
section_title = 'Letter from the editor' |
| 97 |
d = div.findParent(attrs={'class': lambda z: z and z.startswith(('teaserItem__wrapper', 'teaserItem--aside__wrapper'))}) |
| 98 |
if d: |
| 99 |
excerpt = d.find(attrs={'class': lambda x: x and x.startswith(('teaserItem__excerpt', 'teaserItem--aside__excerpt'))}) |
| 100 |
if excerpt: |
| 101 |
desc = self.tag_to_string(excerpt).strip() |
| 102 |
|
| 103 |
sec = d.find(attrs={'class': lambda x: x and x.startswith('teaserItem__eyebrowText')}) |
| 104 |
if sec: |
| 105 |
section_title = self.tag_to_string(sec).replace('Categorized in ', '').strip() |
| 106 |
|
| 107 |
if not href or not title: |
| 108 |
continue |
| 109 |
|
| 110 |
self.log(section_title) |
| 111 |
self.log('\t', title) |
| 112 |
self.log('\t', desc) |
| 113 |
self.log('\t\t', href) |
| 114 |
|
| 115 |
articles.append({'title': title, 'url': href, 'description': desc}) |
| 116 |
if articles: |
| 117 |
if section_title not in feeds: |
| 118 |
feeds[section_title] = [] |
| 119 |
feeds[section_title] += articles |
| 120 |
ans = list(feeds.items()) |
| 121 |
return ans |
| 122 |
|
| 123 |
def preprocess_html(self, soup): |
| 124 |
for bq in soup.findAll('blockquote'): |
| 125 |
for strong in bq.findAll('strong'): |
| 126 |
strong.name = 'div' |
| 127 |
for date in soup.findAll(attrs={'class': lambda x: x and x.startswith(('contentArticleHeader__publishDate', 'byline__wrapper'))}): |
| 128 |
date['id'] = 'pub-d' |
| 129 |
for li in date.findAll(('li', 'ul')): |
| 130 |
li.name = 'span' |
| 131 |
for cap in soup.findAll('figcaption'): |
| 132 |
cap['id'] = 'cap-d' |
| 133 |
for credit in soup.findAll(attrs={'class': lambda x: x and x.startswith(('image__credit', 'image-credit'))}): |
| 134 |
credit['id'] = 'cre-d' |
| 135 |
for img in soup.findAll(srcset=True): |
| 136 |
img['src'] = absurl(img['srcset'].split()[0]) |
| 137 |
del img['srcset'] |
| 138 |
for img in soup.findAll('img', attrs={'src': True}): |
| 139 |
img['src'] = img['src'].split('?')[0] + '?w=800' |
| 140 |
return soup |