| 1 |
#!/usr/bin/env python |
| 2 |
# vim:fileencoding=utf-8 |
| 3 |
# License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net> |
| 4 |
from __future__ import absolute_import, division, print_function, unicode_literals |
| 5 |
|
| 6 |
from collections import defaultdict |
| 7 |
|
| 8 |
from calibre import browser |
| 9 |
from calibre.web.feeds.news import BasicNewsRecipe, classes, prefixed_classes |
| 10 |
|
| 11 |
web_version = False |
| 12 |
|
| 13 |
|
| 14 |
def absurl(x): |
| 15 |
if x.startswith('/') and not x.startswith('//'): |
| 16 |
x = 'https://www.newyorker.com' + x |
| 17 |
return x |
| 18 |
|
| 19 |
|
| 20 |
class NewYorker(BasicNewsRecipe): |
| 21 |
if web_version: |
| 22 |
title = 'The New Yorker (All Articles)' |
| 23 |
description = 'All articles from newyorker.com including daily content, reviews, and magazine pieces' |
| 24 |
else: |
| 25 |
title = 'The New Yorker Magazine' |
| 26 |
description = "Articles of the week's New Yorker magazine" |
| 27 |
language = 'en_US' |
| 28 |
__author__ = 'Kovid Goyal' |
| 29 |
no_stylesheets = True |
| 30 |
timefmt = ' [%b %d]' |
| 31 |
encoding = 'utf-8' |
| 32 |
extra_css = ''' |
| 33 |
img { display:block; margin:0 auto; max-width: 100%; } |
| 34 |
.rubric__name, .byline, time, .caption { font-size:small; } |
| 35 |
.byline, .rubric__name { font-size:smaller; font-weight: bold;} |
| 36 |
h3 { margin-bottom: 6px; } |
| 37 |
.caption, .contributors { font-size: smaller; font-style: italic; font-weight: normal; } |
| 38 |
.content-header__accreditation { font-style:italic; } |
| 39 |
''' |
| 40 |
|
| 41 |
keep_only_tags = [ |
| 42 |
classes('article__content-header article__body contributors'), |
| 43 |
] |
| 44 |
remove_tags = [ |
| 45 |
classes('social-icons'), |
| 46 |
prefixed_classes('ConsentBannerWrapper- ResponsiveCartoonCTA- ContentHeaderContributorImage- ContributorBioAvatar-'), |
| 47 |
dict(childtypes='iframe'), |
| 48 |
dict(name='svg'), |
| 49 |
] |
| 50 |
remove_attributes = ['style'] |
| 51 |
|
| 52 |
def preprocess_html(self, soup): |
| 53 |
w = '/w_320' # use '/w_640' for highres |
| 54 |
for img in soup.findAll('img'): |
| 55 |
if img.has_attr('srcset'): |
| 56 |
for x in img['srcset'].split(): |
| 57 |
if w in x: |
| 58 |
img['src'] = x |
| 59 |
elif img.find_previous_sibling('source', attrs={'srcset': True}): |
| 60 |
srcset = img.find_previous_sibling('source', attrs={'srcset': True}) |
| 61 |
for x in srcset['srcset'].split(): |
| 62 |
if w in x: |
| 63 |
img['src'] = x |
| 64 |
elif '/w_560' in x: |
| 65 |
img['src'] = x |
| 66 |
for src in soup.findAll('source'): |
| 67 |
src.decompose() |
| 68 |
for noscript in soup.findAll('noscript'): |
| 69 |
noscript.name = 'div' |
| 70 |
return soup |
| 71 |
|
| 72 |
# def preprocess_image(self, img_data, image_url): |
| 73 |
# from PIL import Image |
| 74 |
# from calibre import fit_image |
| 75 |
# from io import BytesIO |
| 76 |
# img = Image.open(BytesIO(img_data)).convert('RGB') |
| 77 |
# scaled, nwidth, nheight = fit_image(img.width, img.height, 1024, 1024) |
| 78 |
# if scaled: |
| 79 |
# img = img.resize((nwidth, nheight)) |
| 80 |
# buf = BytesIO() |
| 81 |
# img.save(buf, format='JPEG') |
| 82 |
# return buf.getvalue() |
| 83 |
|
| 84 |
recipe_specific_options = {'date': {'short': 'The date of the edition to download (YYYY/MM/DD format)', 'long': 'For example, 2024/07/08'}} |
| 85 |
|
| 86 |
# The New Yorker changes the content it delivers based on cookies, so the |
| 87 |
# following ensures that we send no cookies |
| 88 |
def get_browser(self, *args, **kwargs): |
| 89 |
return self |
| 90 |
|
| 91 |
def clone_browser(self, *args, **kwargs): |
| 92 |
return self.get_browser() |
| 93 |
|
| 94 |
def open_novisit(self, *args, **kwargs): |
| 95 |
br = browser() |
| 96 |
return br.open_novisit(*args, **kwargs) |
| 97 |
|
| 98 |
open = open_novisit |
| 99 |
|
| 100 |
if web_version: |
| 101 |
use_embedded_content = False |
| 102 |
feeds = [ |
| 103 |
('The New Yorker', 'https://www.newyorker.com/feed/everything'), |
| 104 |
] |
| 105 |
else: |
| 106 |
|
| 107 |
def parse_index(self): |
| 108 |
issue_url = 'https://www.newyorker.com/magazine?intcid=magazine' |
| 109 |
d = self.recipe_specific_options.get('date') |
| 110 |
if d and isinstance(d, str): |
| 111 |
issue_url = 'https://www.newyorker.com/magazine/' + d |
| 112 |
soup = self.index_to_soup(issue_url) |
| 113 |
cover_img = soup.find('picture', attrs={'class': lambda x: x and 'asset-embed__responsive-asset' in x}) |
| 114 |
if cover_img is not None: |
| 115 |
self.cover_url = cover_img.img['src'] |
| 116 |
self.log('Found cover:', self.cover_url) |
| 117 |
try: |
| 118 |
# the src original resolution w_280 was too low, replace w_280 with w_560 |
| 119 |
cover_url_width_index = self.cover_url.find('w_') |
| 120 |
old_width = self.cover_url[cover_url_width_index : cover_url_width_index + 5] |
| 121 |
self.cover_url = self.cover_url.replace(old_width, 'w_640') |
| 122 |
except Exception: |
| 123 |
self.log('Failed enlarging cover img, using the original one') |
| 124 |
|
| 125 |
feeds_dict = defaultdict(list) |
| 126 |
for section in soup.findAll('section', attrs={'class': lambda x: x and 'SummaryRiverSection-' in x}): |
| 127 |
for h2 in section.findAll(attrs={'class': lambda x: x and 'SectionTitleHed-' in x}): |
| 128 |
secname = self.tag_to_string(h2) |
| 129 |
self.log(secname) |
| 130 |
for a in section.findAll('a', href=True, attrs={'class': lambda x: x and 'summary-item__hed-link' in x}): |
| 131 |
# Skip non-article stub tiles: summary-item--externallink |
| 132 |
# (Cartoon Caption Contest, links to interactive contest) |
| 133 |
# and summary-item--gallery (Cartoon Slide Show, links to |
| 134 |
# gallery widget). Neither has an article body to extract. |
| 135 |
wrapper = a.find_parent(attrs={'class': lambda x: x and 'SummaryItemWrapper-' in (' '.join(x) if isinstance(x, list) else x)}) |
| 136 |
if wrapper: |
| 137 |
wcls = wrapper.get('class') or [] |
| 138 |
if isinstance(wcls, list): |
| 139 |
wcls = ' '.join(wcls) |
| 140 |
if 'summary-item--externallink' in wcls or 'summary-item--gallery' in wcls: |
| 141 |
continue |
| 142 |
section = secname |
| 143 |
url = absurl(a['href']) |
| 144 |
title = self.tag_to_string(a) |
| 145 |
desc = '' |
| 146 |
summ = a.find_next_sibling(attrs={'class': lambda x: x and 'summary-item__dek' in x}) |
| 147 |
if summ: |
| 148 |
desc = self.tag_to_string(summ) |
| 149 |
byl = a.find_next_sibling(attrs={'class': lambda x: x and 'summary-item__byline-' in x}) |
| 150 |
if byl: |
| 151 |
desc = self.tag_to_string(byl) + ' | ' + desc |
| 152 |
rub = a.find_previous_sibling(attrs={'class': lambda x: x and 'summary-item__rubric' in x}) |
| 153 |
if rub: |
| 154 |
desc = self.tag_to_string(rub) + ' | ' + desc |
| 155 |
self.log('\t', title, '\n\t', desc, '\n\t\t', url) |
| 156 |
feeds_dict[section].append({'title': title, 'url': url, 'description': desc}) |
| 157 |
return feeds_dict.items() |