Debian Codegraph @

recipes/new_yorker.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 # License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
4 from __future__ import absolute_import, division, print_function, unicode_literals
5
6 from collections import defaultdict
7
8 from calibre import browser
9 from calibre.web.feeds.news import BasicNewsRecipe, classes, prefixed_classes
10
11 web_version = False
12
13
14 def absurl(x):
15 if x.startswith('/') and not x.startswith('//'):
16 x = 'https://www.newyorker.com' + x
17 return x
18
19
20 class NewYorker(BasicNewsRecipe):
21 if web_version:
22 title = 'The New Yorker (All Articles)'
23 description = 'All articles from newyorker.com including daily content, reviews, and magazine pieces'
24 else:
25 title = 'The New Yorker Magazine'
26 description = "Articles of the week's New Yorker magazine"
27 language = 'en_US'
28 __author__ = 'Kovid Goyal'
29 no_stylesheets = True
30 timefmt = ' [%b %d]'
31 encoding = 'utf-8'
32 extra_css = '''
33 img { display:block; margin:0 auto; max-width: 100%; }
34 .rubric__name, .byline, time, .caption { font-size:small; }
35 .byline, .rubric__name { font-size:smaller; font-weight: bold;}
36 h3 { margin-bottom: 6px; }
37 .caption, .contributors { font-size: smaller; font-style: italic; font-weight: normal; }
38 .content-header__accreditation { font-style:italic; }
39 '''
40
41 keep_only_tags = [
42 classes('article__content-header article__body contributors'),
43 ]
44 remove_tags = [
45 classes('social-icons'),
46 prefixed_classes('ConsentBannerWrapper- ResponsiveCartoonCTA- ContentHeaderContributorImage- ContributorBioAvatar-'),
47 dict(childtypes='iframe'),
48 dict(name='svg'),
49 ]
50 remove_attributes = ['style']
51
52 def preprocess_html(self, soup):
53 w = '/w_320' # use '/w_640' for highres
54 for img in soup.findAll('img'):
55 if img.has_attr('srcset'):
56 for x in img['srcset'].split():
57 if w in x:
58 img['src'] = x
59 elif img.find_previous_sibling('source', attrs={'srcset': True}):
60 srcset = img.find_previous_sibling('source', attrs={'srcset': True})
61 for x in srcset['srcset'].split():
62 if w in x:
63 img['src'] = x
64 elif '/w_560' in x:
65 img['src'] = x
66 for src in soup.findAll('source'):
67 src.decompose()
68 for noscript in soup.findAll('noscript'):
69 noscript.name = 'div'
70 return soup
71
72 # def preprocess_image(self, img_data, image_url):
73 # from PIL import Image
74 # from calibre import fit_image
75 # from io import BytesIO
76 # img = Image.open(BytesIO(img_data)).convert('RGB')
77 # scaled, nwidth, nheight = fit_image(img.width, img.height, 1024, 1024)
78 # if scaled:
79 # img = img.resize((nwidth, nheight))
80 # buf = BytesIO()
81 # img.save(buf, format='JPEG')
82 # return buf.getvalue()
83
84 recipe_specific_options = {'date': {'short': 'The date of the edition to download (YYYY/MM/DD format)', 'long': 'For example, 2024/07/08'}}
85
86 # The New Yorker changes the content it delivers based on cookies, so the
87 # following ensures that we send no cookies
88 def get_browser(self, *args, **kwargs):
89 return self
90
91 def clone_browser(self, *args, **kwargs):
92 return self.get_browser()
93
94 def open_novisit(self, *args, **kwargs):
95 br = browser()
96 return br.open_novisit(*args, **kwargs)
97
98 open = open_novisit
99
100 if web_version:
101 use_embedded_content = False
102 feeds = [
103 ('The New Yorker', 'https://www.newyorker.com/feed/everything'),
104 ]
105 else:
106
107 def parse_index(self):
108 issue_url = 'https://www.newyorker.com/magazine?intcid=magazine'
109 d = self.recipe_specific_options.get('date')
110 if d and isinstance(d, str):
111 issue_url = 'https://www.newyorker.com/magazine/' + d
112 soup = self.index_to_soup(issue_url)
113 cover_img = soup.find('picture', attrs={'class': lambda x: x and 'asset-embed__responsive-asset' in x})
114 if cover_img is not None:
115 self.cover_url = cover_img.img['src']
116 self.log('Found cover:', self.cover_url)
117 try:
118 # the src original resolution w_280 was too low, replace w_280 with w_560
119 cover_url_width_index = self.cover_url.find('w_')
120 old_width = self.cover_url[cover_url_width_index : cover_url_width_index + 5]
121 self.cover_url = self.cover_url.replace(old_width, 'w_640')
122 except Exception:
123 self.log('Failed enlarging cover img, using the original one')
124
125 feeds_dict = defaultdict(list)
126 for section in soup.findAll('section', attrs={'class': lambda x: x and 'SummaryRiverSection-' in x}):
127 for h2 in section.findAll(attrs={'class': lambda x: x and 'SectionTitleHed-' in x}):
128 secname = self.tag_to_string(h2)
129 self.log(secname)
130 for a in section.findAll('a', href=True, attrs={'class': lambda x: x and 'summary-item__hed-link' in x}):
131 # Skip non-article stub tiles: summary-item--externallink
132 # (Cartoon Caption Contest, links to interactive contest)
133 # and summary-item--gallery (Cartoon Slide Show, links to
134 # gallery widget). Neither has an article body to extract.
135 wrapper = a.find_parent(attrs={'class': lambda x: x and 'SummaryItemWrapper-' in (' '.join(x) if isinstance(x, list) else x)})
136 if wrapper:
137 wcls = wrapper.get('class') or []
138 if isinstance(wcls, list):
139 wcls = ' '.join(wcls)
140 if 'summary-item--externallink' in wcls or 'summary-item--gallery' in wcls:
141 continue
142 section = secname
143 url = absurl(a['href'])
144 title = self.tag_to_string(a)
145 desc = ''
146 summ = a.find_next_sibling(attrs={'class': lambda x: x and 'summary-item__dek' in x})
147 if summ:
148 desc = self.tag_to_string(summ)
149 byl = a.find_next_sibling(attrs={'class': lambda x: x and 'summary-item__byline-' in x})
150 if byl:
151 desc = self.tag_to_string(byl) + ' | ' + desc
152 rub = a.find_previous_sibling(attrs={'class': lambda x: x and 'summary-item__rubric' in x})
153 if rub:
154 desc = self.tag_to_string(rub) + ' | ' + desc
155 self.log('\t', title, '\n\t', desc, '\n\t\t', url)
156 feeds_dict[section].append({'title': title, 'url': url, 'description': desc})
157 return feeds_dict.items()