| 1 |
# vim:fileencoding=utf-8 |
| 2 |
""" |
| 3 |
Calibre recipe for FAZ.NET (Frankfurter Allgemeine Zeitung). |
| 4 |
|
| 5 |
The site's markup changes fairly often, so this recipe carries a fair amount |
| 6 |
of bespoke cleanup for a handful of recurring content types: |
| 7 |
- plain articles |
| 8 |
- "storytelling" articles whose body is embedded as JSON in a |
| 9 |
``__NUXT_DATA__`` script tag instead of plain HTML |
| 10 |
- Tickaroo-based live blogs ("tik4-*" classes) |
| 11 |
- "Bilderstrecke" image galleries, also delivered via ``__NUXT_DATA__`` |
| 12 |
""" |
| 13 |
|
| 14 |
from __future__ import absolute_import, division, print_function, unicode_literals |
| 15 |
|
| 16 |
import json |
| 17 |
import re |
| 18 |
|
| 19 |
from bs4 import BeautifulSoup, NavigableString |
| 20 |
|
| 21 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 22 |
|
| 23 |
# --------------------------------------------------------------------------- |
| 24 |
# Shared helpers |
| 25 |
# --------------------------------------------------------------------------- |
| 26 |
|
| 27 |
# FAZ serves a "960w" entry in most srcset lists for the standard article- |
| 28 |
# width image. When present, the entry just before it is a higher-density |
| 29 |
# (retina) variant of the same image, which is what we prefer. |
| 30 |
SRCSET_MARKERS = ('960w', '960w,') |
| 31 |
|
| 32 |
|
| 33 |
def pick_srcset_source(srcset, fallback=None): |
| 34 |
"""Pick the best URL out of a ``srcset`` attribute string. |
| 35 |
|
| 36 |
Returns the source right before the "960w" marker if one is found, |
| 37 |
otherwise ``fallback`` if given, otherwise the first source listed. |
| 38 |
""" |
| 39 |
sources = srcset.split() |
| 40 |
for i, source in enumerate(sources): |
| 41 |
if source in SRCSET_MARKERS: |
| 42 |
return sources[i - 1] |
| 43 |
return fallback if fallback is not None else sources[0] |
| 44 |
|
| 45 |
|
| 46 |
def find_image_source(img_tag): |
| 47 |
"""Collapse an <img srcset=...> tag down to a single concrete src.""" |
| 48 |
img_tag['src'] = pick_srcset_source(img_tag['srcset'], img_tag.get('src')) |
| 49 |
del img_tag['srcset'] |
| 50 |
|
| 51 |
|
| 52 |
def format_tickaroo_liveblog(recipe, soup): |
| 53 |
"""Reformat a Tickaroo live-blog ("tik4-*") article into plain flow HTML.""" |
| 54 |
|
| 55 |
IMG_SELECTOR = 'img.tik4-media-image__img[srcset]' |
| 56 |
RICH_TEXT_DIV_SELECTOR = 'div.tik4-content-block.tik4-content-block--rich-text.tik4-content-block--position-2' |
| 57 |
AUTHOR_WRAPPER_SELECTOR = '.tik4-author__wrapper' |
| 58 |
AUTHOR_NAME_SELECTOR = '.tik4-author__name__t' |
| 59 |
AUTHOR_THUMB_SELECTOR = '.tik4-author__thumb' |
| 60 |
MEDIA_BODY_SELECTOR = '.tik4-media-body' |
| 61 |
MEDIA_BODY_TITLE_CLASS = 'tik4-media-body__title' |
| 62 |
MEDIA_BODY_TITLE_SELECTOR = '.' + MEDIA_BODY_TITLE_CLASS |
| 63 |
MEDIA_BODY_PUNCTUATION_SELECTOR = '.tik4-media-body__punctuation' |
| 64 |
MEDIA_BODY_CREDIT_SELECTOR = '.tik4-media-body__credit' |
| 65 |
EXTERNAL_LINK_SELECTOR = 'a.tik4-we-simple' |
| 66 |
EXTERNAL_LINK_TITLE_SELECTOR = '.tik4-we-simple__title' |
| 67 |
EXTERNAL_LINK_DESC_SELECTOR = '.tik4-we-simple__description' |
| 68 |
EXTERNAL_TEASER_IMG_SELECTOR = '[data-external-selector="teaser-object parent-image-label"]' |
| 69 |
|
| 70 |
# 1. Resolve images that only carry a srcset. |
| 71 |
for img_tag in soup.select(IMG_SELECTOR): |
| 72 |
find_image_source(img_tag) |
| 73 |
|
| 74 |
# 2. Add a line break before each rich-text news block. |
| 75 |
for div_tag in soup.select(RICH_TEXT_DIV_SELECTOR): |
| 76 |
div_tag.insert_before(soup.new_tag('br')) |
| 77 |
|
| 78 |
# 3. Add a line break before each timestamp. |
| 79 |
for time_tag in soup.find_all('time'): |
| 80 |
time_tag.insert_before(soup.new_tag('br')) |
| 81 |
|
| 82 |
# 4. Move the author name next to the author thumbnail. |
| 83 |
for author_wrapper in soup.select(AUTHOR_WRAPPER_SELECTOR): |
| 84 |
author_name_tag = author_wrapper.select_one(AUTHOR_NAME_SELECTOR) |
| 85 |
author_thumb_tag = author_wrapper.select_one(AUTHOR_THUMB_SELECTOR) |
| 86 |
if not author_name_tag: |
| 87 |
continue |
| 88 |
|
| 89 |
extracted_name = author_name_tag.extract() |
| 90 |
extracted_name['class'] = MEDIA_BODY_TITLE_CLASS |
| 91 |
|
| 92 |
if author_thumb_tag: |
| 93 |
img = author_thumb_tag.find('img') |
| 94 |
if img and img.has_attr('srcset'): |
| 95 |
del img['srcset'] |
| 96 |
author_thumb_tag.insert_after(extracted_name) |
| 97 |
|
| 98 |
# 5. Fold punctuation and credit into the media title. |
| 99 |
for media_body in soup.select(MEDIA_BODY_SELECTOR): |
| 100 |
title = media_body.select_one(MEDIA_BODY_TITLE_SELECTOR) |
| 101 |
if not title: |
| 102 |
continue |
| 103 |
punctuation = media_body.select_one(MEDIA_BODY_PUNCTUATION_SELECTOR) |
| 104 |
credit = media_body.select_one(MEDIA_BODY_CREDIT_SELECTOR) |
| 105 |
if punctuation: |
| 106 |
title.append(punctuation.extract()) |
| 107 |
if credit: |
| 108 |
title.append(credit.extract()) |
| 109 |
|
| 110 |
# 6. Unwrap "external link" teasers (tik4-we-simple) into plain text/links. |
| 111 |
for link_tag in soup.select(EXTERNAL_LINK_SELECTOR): |
| 112 |
href = link_tag.get('href') |
| 113 |
if not href: |
| 114 |
continue |
| 115 |
|
| 116 |
teaser_soup = recipe.index_to_soup(href) |
| 117 |
if not teaser_soup: |
| 118 |
continue |
| 119 |
|
| 120 |
picture = teaser_soup.find('picture') or teaser_soup.select_one(EXTERNAL_TEASER_IMG_SELECTOR) |
| 121 |
if picture: |
| 122 |
link_tag.insert_before(soup.new_tag('br')) |
| 123 |
if picture.img: |
| 124 |
link_tag.insert_before(picture.img.extract()) |
| 125 |
link_tag.insert_after(soup.new_tag('br')) |
| 126 |
|
| 127 |
title_element = link_tag.select_one(EXTERNAL_LINK_TITLE_SELECTOR) |
| 128 |
if title_element: |
| 129 |
title_element.insert_before(soup.new_tag('br')) |
| 130 |
title_element.insert_after(soup.new_tag('br')) |
| 131 |
|
| 132 |
description_element = link_tag.select_one(EXTERNAL_LINK_DESC_SELECTOR) |
| 133 |
if description_element: |
| 134 |
description_element.name = 'a' |
| 135 |
description_element['href'] = href |
| 136 |
description_element.wrap(soup.new_tag('div')) |
| 137 |
|
| 138 |
link_tag.unwrap() |
| 139 |
|
| 140 |
|
| 141 |
def bilderstrecke(soup, script_tag): |
| 142 |
"""Render a "Bilderstrecke" image gallery from its embedded JSON blob. |
| 143 |
|
| 144 |
The gallery data lives inside a ``__NUXT_DATA__`` script tag as a flat |
| 145 |
JSON array; string values are referenced elsewhere in the array by index |
| 146 |
rather than inlined, which is why the lookups below index back into |
| 147 |
``image_data``. |
| 148 |
""" |
| 149 |
try: |
| 150 |
image_data = json.loads(script_tag.contents[0]) |
| 151 |
except ValueError, IndexError: |
| 152 |
return |
| 153 |
if not isinstance(image_data, list) or not any(isinstance(item, dict) and 'caption' in item for item in image_data): |
| 154 |
return |
| 155 |
|
| 156 |
header_teaser = soup.find_all(attrs={'data-external-selector': 'header-teaser'}) |
| 157 |
header_teaser_element = None |
| 158 |
if header_teaser: |
| 159 |
if len(header_teaser) > 1: |
| 160 |
header_teaser[0].extract() |
| 161 |
header_teaser_element = header_teaser[0] |
| 162 |
|
| 163 |
image_gallery = soup.new_tag('div') |
| 164 |
oldfigure = None |
| 165 |
for item in image_data: |
| 166 |
if not (isinstance(item, dict) and 'caption' in item and 'derivatives' in item): |
| 167 |
continue |
| 168 |
|
| 169 |
caption = soup.new_tag('p') |
| 170 |
caption.append(image_data[int(item['caption'])]) |
| 171 |
caption['class'] = 'body-elements__image-figcaption' |
| 172 |
if 'source' in item: |
| 173 |
credit = soup.new_tag('span') |
| 174 |
credit.append(image_data[int(item['source'])]) |
| 175 |
credit['class'] = 'body-elements__image-figcaption--source' |
| 176 |
caption.append(credit) |
| 177 |
|
| 178 |
figure = soup.new_tag('figure') |
| 179 |
img = soup.new_tag('img') |
| 180 |
img['src'] = image_data[int(image_data[int(item['derivatives'])]['original_aspect_ratio'])] |
| 181 |
figure.append(img) |
| 182 |
figure.append(caption) |
| 183 |
if img['src'] != oldfigure: |
| 184 |
image_gallery.append(figure) |
| 185 |
oldfigure = img['src'] |
| 186 |
|
| 187 |
if header_teaser_element: |
| 188 |
header_teaser_element.insert_after(image_gallery) |
| 189 |
else: |
| 190 |
soup.append(image_gallery) |
| 191 |
|
| 192 |
for tag in soup.find_all(class_='header-teaser__image--default'): |
| 193 |
tag.extract() |
| 194 |
|
| 195 |
|
| 196 |
def story(soup, tag): |
| 197 |
"""Move the first caption next to the first image in a story article.""" |
| 198 |
first_image = soup.find('img', attrs={'loading': 'lazy'}) |
| 199 |
first_caption = soup.find('figcaption', attrs={'class': 'caption'}) |
| 200 |
if first_image and first_caption: |
| 201 |
first_image.insert_after(first_caption.extract()) |
| 202 |
|
| 203 |
|
| 204 |
class FazNet(BasicNewsRecipe): |
| 205 |
# Version 11.1m |
| 206 |
# Update 2026 |
| 207 |
# original by Armin Geller |
| 208 |
# overhaul to deal with changes in the faz.net websites |
| 209 |
|
| 210 |
title = 'FAZ.NET' |
| 211 |
__author__ = 'Unknown' |
| 212 |
description = 'Frankfurter Allgemeine Zeitung' |
| 213 |
publisher = 'Frankfurter Allgemeine Zeitung GmbH' |
| 214 |
category = 'news, politics, Germany' |
| 215 |
cover_url = 'https://upload.wikimedia.org/wikipedia/commons/7/72/Frankfurter_Allgemeine_logo.svg' |
| 216 |
encoding = 'utf-8' |
| 217 |
language = 'de' |
| 218 |
ignore_duplicate_articles = {'title', 'url'} |
| 219 |
max_articles_per_feed = 30 |
| 220 |
no_stylesheets = True |
| 221 |
remove_javascript = True |
| 222 |
scale_news_images = (10, 100) |
| 223 |
delay = 1 |
| 224 |
|
| 225 |
extra_css = ''' |
| 226 |
.scrolly-title {font-size: 1.5em; font-weight:bold; text-align:left;} |
| 227 |
.quote,.tik4-quote__text,.tik4-we-simple__title, .section-heading { |
| 228 |
font-size: 1.3em; font-weight:bold; text-align:center;color:grey;margin-bottom:1em;margin-top:1em;} |
| 229 |
.intro-text,.author, .tik4-quote__cite--source {font-size: 0.7em; font-weight:bold; text-align:center; display:block; |
| 230 |
margin-bottom: 0.95em; color:grey;} |
| 231 |
.intro-label,.header-label__content {font-size: 0.7em; font-weight:bold; text-align:left; display:block; |
| 232 |
margin-bottom: 0.95em; color:grey;} |
| 233 |
h2 {font-size:1.4em;text-align:left;} |
| 234 |
.dict-txt,.caption,.body-elements__image-figcaption,.header-teaser__image-details,.meta2, |
| 235 |
.tik4-media-body__title,.scrolly-text {line-height:0.8; display:block; margin-top:0.05em;margin-bottom:1em; font-size: 0.85em; text-align:left;} |
| 236 |
.caption__copyright,.meta2-italic,.body-elements__image-figcaption--source,.header-teaser__image-details--source, |
| 237 |
.dict-img-src,.tik4-media-body__credit, .tik4-we-simple__meta {font-size: 0.65em; font-style:italic; text-align:left;margin-left:0.4em;} |
| 238 |
.header-detail--bold {font-size:0.6em; font-weight:bold; margin-bottom:0.75em;text-align:left;} |
| 239 |
.header-detail {font-size:0.8em} |
| 240 |
time {font-size:0.6em; font-weight: normal; margin-bottom:0.75em; text-align:left; display:block;} |
| 241 |
.copy--intro,.header-teaser,.scrolly-intro {font-size:1em; font-style:italic; font-weight:bold;margin-bottom:1em;} |
| 242 |
.dict-hl {font-size:1em; font-style:italic; font-weight:bold} |
| 243 |
.tik4-media-image,.dict-box {margin-bottom:1em;margin-top:1em;} |
| 244 |
''' |
| 245 |
|
| 246 |
keep_only_tags = [ |
| 247 |
dict(name='article', attrs={'class': ['article', 'storytelling']}), |
| 248 |
dict(name='body'), |
| 249 |
dict(name='div', attrs={'class': ['imageGallery', 'image_only']}), |
| 250 |
dict(name='div', attrs={'class': 'tik4-live__container'}), |
| 251 |
dict(name='script', attrs={'id': '__NUXT_DATA__'}), |
| 252 |
dict(name='figure', attrs={'class': 'image'}), |
| 253 |
] |
| 254 |
|
| 255 |
remove_tags = [ |
| 256 |
dict( |
| 257 |
name='div', |
| 258 |
attrs={ |
| 259 |
'class': [ |
| 260 |
'tik4-sharing', |
| 261 |
'tik4-load-more-bottom', |
| 262 |
'tik4-by', |
| 263 |
'header-detail__image', |
| 264 |
'mm-adbox', |
| 265 |
'upper-toolbar', |
| 266 |
'tik4-chapters__label', |
| 267 |
'tik4-we-toggle', |
| 268 |
] |
| 269 |
}, |
| 270 |
), |
| 271 |
dict(name='svg'), |
| 272 |
dict(name='div', attrs={'data-module': 'teaser'}), |
| 273 |
dict( |
| 274 |
attrs={ |
| 275 |
'data-external-selector': [ |
| 276 |
'upper-toolbar', |
| 277 |
'article-footer', |
| 278 |
'related-articles', |
| 279 |
'placeholder', |
| 280 |
'media-label', |
| 281 |
] |
| 282 |
} |
| 283 |
), |
| 284 |
dict(name='img', attrs={'class': 'sm:hidden'}), |
| 285 |
] |
| 286 |
|
| 287 |
remove_attributes = ['onclick', 'initialdata', 'sizes'] |
| 288 |
|
| 289 |
# To fetch a single article instead of the full set of RSS feeds (handy |
| 290 |
# while debugging a markup change), set TEST_ARTICLE_URL below. |
| 291 |
TEST_ARTICLE_URL = None |
| 292 |
# "https://www.faz.net/aktuell/besser-leben/stil/bilderstrecke-mode-accessoires-fuer-maenner-16997909.html" |
| 293 |
|
| 294 |
if TEST_ARTICLE_URL: |
| 295 |
|
| 296 |
def parse_index(self): |
| 297 |
return [('Articles', [{'title': 'Test article', 'url': self.TEST_ARTICLE_URL}])] |
| 298 |
|
| 299 |
else: |
| 300 |
feeds = [ |
| 301 |
('FAZ.NET Aktuell', 'https://www.faz.net/rss/aktuell/'), |
| 302 |
('Politik', 'https://www.faz.net/rss/aktuell/politik/'), |
| 303 |
('Wirtschaft', 'https://www.faz.net/rss/aktuell/wirtschaft/'), |
| 304 |
('Feuilleton', 'https://www.faz.net/rss/aktuell/feuilleton/'), |
| 305 |
('Sport', 'https://www.faz.net/rss/aktuell/sport/'), |
| 306 |
('Lebensstil', 'https://www.faz.net/rss/aktuell/besser-leben/stil/'), |
| 307 |
('Gesellschaft', 'https://www.faz.net/rss/aktuell/gesellschaft/'), |
| 308 |
('Finanzen', 'https://www.faz.net/rss/aktuell/finanzen/'), |
| 309 |
('Technik & Motor', 'https://www.faz.net/rss/aktuell/technik-motor/'), |
| 310 |
('Wissen', 'https://www.faz.net/rss/aktuell/wissen/'), |
| 311 |
('Reise', 'https://www.faz.net/rss/aktuell/reise/'), |
| 312 |
('Karriere & Hochschule', 'https://www.faz.net/rss/aktuell/karriere-hochschule/'), |
| 313 |
('Rhein-Main', 'https://www.faz.net/rss/aktuell/rhein-main/'), |
| 314 |
] |
| 315 |
|
| 316 |
def _unpack_storytelling_article(self, soup): |
| 317 |
"""Render "storytelling" articles whose body is embedded as JSON. |
| 318 |
|
| 319 |
These articles ship an empty <article class="storytelling"> shell; |
| 320 |
the actual content lives inside a __NUXT_DATA__ script tag and has |
| 321 |
to be parsed out and re-inserted as HTML. |
| 322 |
""" |
| 323 |
article_tag = soup.find(name='article', attrs={'class': 'storytelling'}) |
| 324 |
if not (article_tag and not article_tag.get_text().strip()): |
| 325 |
return |
| 326 |
|
| 327 |
nuxt_script_tag = soup.find('script', attrs={'id': '__NUXT_DATA__', 'type': 'application/json'}) |
| 328 |
if not nuxt_script_tag: |
| 329 |
return |
| 330 |
|
| 331 |
try: |
| 332 |
json_data = json.loads(str(nuxt_script_tag.contents[0])) |
| 333 |
for item in json_data: |
| 334 |
if not (isinstance(item, dict) and 'body' in item): |
| 335 |
continue |
| 336 |
body = json_data[int(item['body'])] |
| 337 |
if not body: |
| 338 |
continue |
| 339 |
new_soup = BeautifulSoup(body, 'html.parser') |
| 340 |
soup.append(new_soup) |
| 341 |
storytelling_tag = soup.find(attrs={'data-content': 'storytelling'}) |
| 342 |
if storytelling_tag: |
| 343 |
storytelling_tag.unwrap() |
| 344 |
break |
| 345 |
except Exception: |
| 346 |
pass |
| 347 |
|
| 348 |
def preprocess_html(self, soup): |
| 349 |
self._unpack_storytelling_article(soup) |
| 350 |
|
| 351 |
# Format hovering images: resolve lazy-loaded src, and drop duplicate |
| 352 |
# captions that immediately repeat the same text. |
| 353 |
for figure_tag in soup.find_all('figure', class_='image'): |
| 354 |
img_tag = figure_tag.find('img', attrs={'data-src': True}) |
| 355 |
if img_tag: |
| 356 |
img_tag['src'] = img_tag['data-src'] |
| 357 |
del img_tag['data-src'] |
| 358 |
caption_tag = figure_tag.find('figcaption') |
| 359 |
if caption_tag: |
| 360 |
next_caption = caption_tag.find_next('figcaption') |
| 361 |
if next_caption and caption_tag.get_text(strip=True) == next_caption.get_text(strip=True): |
| 362 |
next_caption.extract() |
| 363 |
|
| 364 |
# Move the headline in front of the lead image, if it isn't already. |
| 365 |
lead_image = soup.find('figure', attrs={'class': 'image'}) |
| 366 |
h1 = soup.find('h1') |
| 367 |
if lead_image and h1 and h1.parent and not h1.parent.find(lead_image): |
| 368 |
lead_image.insert_before(h1.parent.extract()) |
| 369 |
|
| 370 |
# Normalize all figure captions to <span class="caption">. |
| 371 |
for tag in soup.find_all('figcaption'): |
| 372 |
tag['class'] = 'caption' |
| 373 |
tag.name = 'span' |
| 374 |
|
| 375 |
# Format the "header-detail" byline/date block into a plain span. |
| 376 |
header_detail_selectors = ['header-detail', 'header-detail-opinion header-detail'] |
| 377 |
header_detail_tag = soup.find(attrs={'data-external-selector': header_detail_selectors}) |
| 378 |
if header_detail_tag: |
| 379 |
span = soup.new_tag('span') |
| 380 |
span['class'] = 'header-detail' |
| 381 |
span.string = header_detail_tag.get_text(separator=' ', strip=True).replace(' ,', ',') |
| 382 |
if header_detail_tag.img and header_detail_tag.img.parent and header_detail_tag != header_detail_tag.img.parent: |
| 383 |
header_detail_tag.insert_before(header_detail_tag.img.parent.extract()) |
| 384 |
header_detail_tag.replace_with(span) |
| 385 |
|
| 386 |
# Format "story"-type articles (moves the first caption next to the |
| 387 |
# first image). |
| 388 |
if soup.find(class_='storyContainer'): |
| 389 |
story(soup, None) |
| 390 |
|
| 391 |
# Extract images and text from "Bilderstrecke" image galleries: these |
| 392 |
# show up in the body as a lone "<n> Bilder" paragraph, with the |
| 393 |
# actual gallery data embedded in a __NUXT_DATA__ script tag. |
| 394 |
for par in soup.find_all('p'): |
| 395 |
if len(par.contents) != 1: |
| 396 |
continue |
| 397 |
content = str(par.contents[0]).strip() |
| 398 |
if re.search(r'^[1-9]\d* Bilder$', content): |
| 399 |
par.extract() |
| 400 |
nuxt_script_tag = soup.find('script', attrs={'id': '__NUXT_DATA__', 'type': 'application/json'}) |
| 401 |
if nuxt_script_tag: |
| 402 |
bilderstrecke(soup, nuxt_script_tag) |
| 403 |
break |
| 404 |
|
| 405 |
# Unwrap buttons, dropping "Weitere anzeigen" (show more) entirely. |
| 406 |
for tag in soup.find_all('button'): |
| 407 |
if 'Weitere anzeigen' in tag.get_text(): |
| 408 |
tag.extract() |
| 409 |
else: |
| 410 |
tag.unwrap() |
| 411 |
|
| 412 |
# Rename the section-label span to a class we style explicitly. |
| 413 |
label_tag = soup.find('span', class_='sm:o2-medium') |
| 414 |
if label_tag: |
| 415 |
label_tag['class'] = 'header-label__content' |
| 416 |
|
| 417 |
# Drop the screen-reader-only colon inside the section label. |
| 418 |
header_label_tag = soup.find(class_='header-label__content') |
| 419 |
if header_label_tag: |
| 420 |
colon_tag = header_label_tag.find(class_='sr-only') |
| 421 |
if colon_tag: |
| 422 |
colon_tag.extract() |
| 423 |
|
| 424 |
# Clean up <h2> headers: turn nested <div>s into <span>s. |
| 425 |
h2_tag = soup.find('h2') |
| 426 |
if h2_tag: |
| 427 |
for div_tag in h2_tag.find_all('div'): |
| 428 |
div_tag.name = 'span' |
| 429 |
|
| 430 |
# Format the "header-teaser" lead paragraph. |
| 431 |
header_teaser_tag = soup.find(attrs={'data-external-selector': 'header-teaser'}) |
| 432 |
if header_teaser_tag: |
| 433 |
header_teaser_tag['class'] = 'header-teaser' |
| 434 |
|
| 435 |
# Skip articles behind the paywall. |
| 436 |
if soup.find(id='faz-paywall'): |
| 437 |
self.abort_article() |
| 438 |
|
| 439 |
# Remove the F.A.Z. newsletter ad. |
| 440 |
for tag in soup.find_all(attrs={'class': 'body-elements__paragraph'}): |
| 441 |
if tag.contents and isinstance(tag.contents[0], NavigableString): |
| 442 |
if 'F.A.Z.-Newsletter' in str(tag.contents[0]): |
| 443 |
tag.extract() |
| 444 |
|
| 445 |
# Remove the "prefer us on Google" nudge (paragraph and link forms). |
| 446 |
for tag in soup.find_all('p'): |
| 447 |
if 'in Ihren Suchergebnissen sehen' in tag.get_text(): |
| 448 |
tag.extract() |
| 449 |
for tag in soup.find_all('a'): |
| 450 |
if 'bei Google bevorzugen' in tag.get_text(): |
| 451 |
tag.extract() |
| 452 |
|
| 453 |
# Reformat Tickaroo live blogs. |
| 454 |
if soup.find(attrs={'class': 'tik4-live__container'}): |
| 455 |
format_tickaroo_liveblog(self, soup) |
| 456 |
|
| 457 |
# Resolve any remaining srcset-only images. |
| 458 |
for img_tag in soup.find_all('img', srcset=True): |
| 459 |
find_image_source(img_tag) |
| 460 |
|
| 461 |
# Resolve <picture><source srcset=...> images. |
| 462 |
for picture_tag in soup.find_all('picture'): |
| 463 |
source_tag = picture_tag.find('source', srcset=True) |
| 464 |
img_tag = picture_tag.find('img') if source_tag else None |
| 465 |
if img_tag: |
| 466 |
img_tag['src'] = pick_srcset_source(source_tag['srcset']) |
| 467 |
|
| 468 |
# Strip all image attributes except src/alt/title; drop images with |
| 469 |
# no resolvable src entirely. |
| 470 |
attributes_to_keep = {'src', 'alt', 'title'} |
| 471 |
for img_tag in soup.find_all('img'): |
| 472 |
if not img_tag.has_attr('src'): |
| 473 |
img_tag.decompose() |
| 474 |
continue |
| 475 |
for attr in list(img_tag.attrs): |
| 476 |
if attr not in attributes_to_keep: |
| 477 |
del img_tag[attr] |
| 478 |
|
| 479 |
return soup |
| 480 |
|
| 481 |
def postprocess_html(self, soup, first_fetch): |
| 482 |
# Remove the style attribute from certain image captions. |
| 483 |
for tag in soup.select('div.meta2'): |
| 484 |
if tag.has_attr('style'): |
| 485 |
del tag['style'] |
| 486 |
if tag.span: |
| 487 |
tag.span.unwrap() |
| 488 |
|
| 489 |
# Unwrap any remaining <span> inside a caption. |
| 490 |
for tag in soup.find_all(attrs={'class': 'caption'}): |
| 491 |
span = tag.find('span') |
| 492 |
if span: |
| 493 |
span.unwrap() |
| 494 |
|
| 495 |
# Add a trailing period between a figure caption and its credit, |
| 496 |
# where the caption doesn't already end in punctuation. |
| 497 |
caption_classes = [ |
| 498 |
'body-elements__image-figcaption', |
| 499 |
'header-teaser__image-details', |
| 500 |
'caption', |
| 501 |
'meta2', |
| 502 |
] |
| 503 |
for tag in soup.find_all(attrs={'class': caption_classes}): |
| 504 |
if tag.string is None and tag.contents and tag.contents[0].string: |
| 505 |
tag = tag.contents[0] |
| 506 |
if tag.string: |
| 507 |
text = str(tag.string).strip() |
| 508 |
if text and text[-1] not in '.?!:': |
| 509 |
tag.string.replace_with(text + '. ') |
| 510 |
|
| 511 |
# Add a line break between an image and a following header-teaser. |
| 512 |
for tag in soup.find_all(class_='header-teaser'): |
| 513 |
tag.insert_before(soup.new_tag('br')) |
| 514 |
|
| 515 |
# Wrap every image in its own <div>, stripped of layout attributes. |
| 516 |
for img_tag in soup.find_all('img'): |
| 517 |
for attrib in ('height', 'width', 'border', 'align', 'style'): |
| 518 |
if img_tag.has_attr(attrib): |
| 519 |
del img_tag[attrib] |
| 520 |
parent = img_tag.parent |
| 521 |
index = parent.contents.index(img_tag) |
| 522 |
img_tag.extract() |
| 523 |
div_tag = soup.new_tag('div') |
| 524 |
parent.insert(index, div_tag) |
| 525 |
div_tag.append(img_tag) |
| 526 |
|
| 527 |
return soup |