Debian Codegraph @

recipes/faz_net.recipe

raw
1 # vim:fileencoding=utf-8
2 """
3 Calibre recipe for FAZ.NET (Frankfurter Allgemeine Zeitung).
4
5 The site's markup changes fairly often, so this recipe carries a fair amount
6 of bespoke cleanup for a handful of recurring content types:
7 - plain articles
8 - "storytelling" articles whose body is embedded as JSON in a
9 ``__NUXT_DATA__`` script tag instead of plain HTML
10 - Tickaroo-based live blogs ("tik4-*" classes)
11 - "Bilderstrecke" image galleries, also delivered via ``__NUXT_DATA__``
12 """
13
14 from __future__ import absolute_import, division, print_function, unicode_literals
15
16 import json
17 import re
18
19 from bs4 import BeautifulSoup, NavigableString
20
21 from calibre.web.feeds.news import BasicNewsRecipe
22
23 # ---------------------------------------------------------------------------
24 # Shared helpers
25 # ---------------------------------------------------------------------------
26
27 # FAZ serves a "960w" entry in most srcset lists for the standard article-
28 # width image. When present, the entry just before it is a higher-density
29 # (retina) variant of the same image, which is what we prefer.
30 SRCSET_MARKERS = ('960w', '960w,')
31
32
33 def pick_srcset_source(srcset, fallback=None):
34 """Pick the best URL out of a ``srcset`` attribute string.
35
36 Returns the source right before the "960w" marker if one is found,
37 otherwise ``fallback`` if given, otherwise the first source listed.
38 """
39 sources = srcset.split()
40 for i, source in enumerate(sources):
41 if source in SRCSET_MARKERS:
42 return sources[i - 1]
43 return fallback if fallback is not None else sources[0]
44
45
46 def find_image_source(img_tag):
47 """Collapse an <img srcset=...> tag down to a single concrete src."""
48 img_tag['src'] = pick_srcset_source(img_tag['srcset'], img_tag.get('src'))
49 del img_tag['srcset']
50
51
52 def format_tickaroo_liveblog(recipe, soup):
53 """Reformat a Tickaroo live-blog ("tik4-*") article into plain flow HTML."""
54
55 IMG_SELECTOR = 'img.tik4-media-image__img[srcset]'
56 RICH_TEXT_DIV_SELECTOR = 'div.tik4-content-block.tik4-content-block--rich-text.tik4-content-block--position-2'
57 AUTHOR_WRAPPER_SELECTOR = '.tik4-author__wrapper'
58 AUTHOR_NAME_SELECTOR = '.tik4-author__name__t'
59 AUTHOR_THUMB_SELECTOR = '.tik4-author__thumb'
60 MEDIA_BODY_SELECTOR = '.tik4-media-body'
61 MEDIA_BODY_TITLE_CLASS = 'tik4-media-body__title'
62 MEDIA_BODY_TITLE_SELECTOR = '.' + MEDIA_BODY_TITLE_CLASS
63 MEDIA_BODY_PUNCTUATION_SELECTOR = '.tik4-media-body__punctuation'
64 MEDIA_BODY_CREDIT_SELECTOR = '.tik4-media-body__credit'
65 EXTERNAL_LINK_SELECTOR = 'a.tik4-we-simple'
66 EXTERNAL_LINK_TITLE_SELECTOR = '.tik4-we-simple__title'
67 EXTERNAL_LINK_DESC_SELECTOR = '.tik4-we-simple__description'
68 EXTERNAL_TEASER_IMG_SELECTOR = '[data-external-selector="teaser-object parent-image-label"]'
69
70 # 1. Resolve images that only carry a srcset.
71 for img_tag in soup.select(IMG_SELECTOR):
72 find_image_source(img_tag)
73
74 # 2. Add a line break before each rich-text news block.
75 for div_tag in soup.select(RICH_TEXT_DIV_SELECTOR):
76 div_tag.insert_before(soup.new_tag('br'))
77
78 # 3. Add a line break before each timestamp.
79 for time_tag in soup.find_all('time'):
80 time_tag.insert_before(soup.new_tag('br'))
81
82 # 4. Move the author name next to the author thumbnail.
83 for author_wrapper in soup.select(AUTHOR_WRAPPER_SELECTOR):
84 author_name_tag = author_wrapper.select_one(AUTHOR_NAME_SELECTOR)
85 author_thumb_tag = author_wrapper.select_one(AUTHOR_THUMB_SELECTOR)
86 if not author_name_tag:
87 continue
88
89 extracted_name = author_name_tag.extract()
90 extracted_name['class'] = MEDIA_BODY_TITLE_CLASS
91
92 if author_thumb_tag:
93 img = author_thumb_tag.find('img')
94 if img and img.has_attr('srcset'):
95 del img['srcset']
96 author_thumb_tag.insert_after(extracted_name)
97
98 # 5. Fold punctuation and credit into the media title.
99 for media_body in soup.select(MEDIA_BODY_SELECTOR):
100 title = media_body.select_one(MEDIA_BODY_TITLE_SELECTOR)
101 if not title:
102 continue
103 punctuation = media_body.select_one(MEDIA_BODY_PUNCTUATION_SELECTOR)
104 credit = media_body.select_one(MEDIA_BODY_CREDIT_SELECTOR)
105 if punctuation:
106 title.append(punctuation.extract())
107 if credit:
108 title.append(credit.extract())
109
110 # 6. Unwrap "external link" teasers (tik4-we-simple) into plain text/links.
111 for link_tag in soup.select(EXTERNAL_LINK_SELECTOR):
112 href = link_tag.get('href')
113 if not href:
114 continue
115
116 teaser_soup = recipe.index_to_soup(href)
117 if not teaser_soup:
118 continue
119
120 picture = teaser_soup.find('picture') or teaser_soup.select_one(EXTERNAL_TEASER_IMG_SELECTOR)
121 if picture:
122 link_tag.insert_before(soup.new_tag('br'))
123 if picture.img:
124 link_tag.insert_before(picture.img.extract())
125 link_tag.insert_after(soup.new_tag('br'))
126
127 title_element = link_tag.select_one(EXTERNAL_LINK_TITLE_SELECTOR)
128 if title_element:
129 title_element.insert_before(soup.new_tag('br'))
130 title_element.insert_after(soup.new_tag('br'))
131
132 description_element = link_tag.select_one(EXTERNAL_LINK_DESC_SELECTOR)
133 if description_element:
134 description_element.name = 'a'
135 description_element['href'] = href
136 description_element.wrap(soup.new_tag('div'))
137
138 link_tag.unwrap()
139
140
141 def bilderstrecke(soup, script_tag):
142 """Render a "Bilderstrecke" image gallery from its embedded JSON blob.
143
144 The gallery data lives inside a ``__NUXT_DATA__`` script tag as a flat
145 JSON array; string values are referenced elsewhere in the array by index
146 rather than inlined, which is why the lookups below index back into
147 ``image_data``.
148 """
149 try:
150 image_data = json.loads(script_tag.contents[0])
151 except ValueError, IndexError:
152 return
153 if not isinstance(image_data, list) or not any(isinstance(item, dict) and 'caption' in item for item in image_data):
154 return
155
156 header_teaser = soup.find_all(attrs={'data-external-selector': 'header-teaser'})
157 header_teaser_element = None
158 if header_teaser:
159 if len(header_teaser) > 1:
160 header_teaser[0].extract()
161 header_teaser_element = header_teaser[0]
162
163 image_gallery = soup.new_tag('div')
164 oldfigure = None
165 for item in image_data:
166 if not (isinstance(item, dict) and 'caption' in item and 'derivatives' in item):
167 continue
168
169 caption = soup.new_tag('p')
170 caption.append(image_data[int(item['caption'])])
171 caption['class'] = 'body-elements__image-figcaption'
172 if 'source' in item:
173 credit = soup.new_tag('span')
174 credit.append(image_data[int(item['source'])])
175 credit['class'] = 'body-elements__image-figcaption--source'
176 caption.append(credit)
177
178 figure = soup.new_tag('figure')
179 img = soup.new_tag('img')
180 img['src'] = image_data[int(image_data[int(item['derivatives'])]['original_aspect_ratio'])]
181 figure.append(img)
182 figure.append(caption)
183 if img['src'] != oldfigure:
184 image_gallery.append(figure)
185 oldfigure = img['src']
186
187 if header_teaser_element:
188 header_teaser_element.insert_after(image_gallery)
189 else:
190 soup.append(image_gallery)
191
192 for tag in soup.find_all(class_='header-teaser__image--default'):
193 tag.extract()
194
195
196 def story(soup, tag):
197 """Move the first caption next to the first image in a story article."""
198 first_image = soup.find('img', attrs={'loading': 'lazy'})
199 first_caption = soup.find('figcaption', attrs={'class': 'caption'})
200 if first_image and first_caption:
201 first_image.insert_after(first_caption.extract())
202
203
204 class FazNet(BasicNewsRecipe):
205 # Version 11.1m
206 # Update 2026
207 # original by Armin Geller
208 # overhaul to deal with changes in the faz.net websites
209
210 title = 'FAZ.NET'
211 __author__ = 'Unknown'
212 description = 'Frankfurter Allgemeine Zeitung'
213 publisher = 'Frankfurter Allgemeine Zeitung GmbH'
214 category = 'news, politics, Germany'
215 cover_url = 'https://upload.wikimedia.org/wikipedia/commons/7/72/Frankfurter_Allgemeine_logo.svg'
216 encoding = 'utf-8'
217 language = 'de'
218 ignore_duplicate_articles = {'title', 'url'}
219 max_articles_per_feed = 30
220 no_stylesheets = True
221 remove_javascript = True
222 scale_news_images = (10, 100)
223 delay = 1
224
225 extra_css = '''
226 .scrolly-title {font-size: 1.5em; font-weight:bold; text-align:left;}
227 .quote,.tik4-quote__text,.tik4-we-simple__title, .section-heading {
228 font-size: 1.3em; font-weight:bold; text-align:center;color:grey;margin-bottom:1em;margin-top:1em;}
229 .intro-text,.author, .tik4-quote__cite--source {font-size: 0.7em; font-weight:bold; text-align:center; display:block;
230 margin-bottom: 0.95em; color:grey;}
231 .intro-label,.header-label__content {font-size: 0.7em; font-weight:bold; text-align:left; display:block;
232 margin-bottom: 0.95em; color:grey;}
233 h2 {font-size:1.4em;text-align:left;}
234 .dict-txt,.caption,.body-elements__image-figcaption,.header-teaser__image-details,.meta2,
235 .tik4-media-body__title,.scrolly-text {line-height:0.8; display:block; margin-top:0.05em;margin-bottom:1em; font-size: 0.85em; text-align:left;}
236 .caption__copyright,.meta2-italic,.body-elements__image-figcaption--source,.header-teaser__image-details--source,
237 .dict-img-src,.tik4-media-body__credit, .tik4-we-simple__meta {font-size: 0.65em; font-style:italic; text-align:left;margin-left:0.4em;}
238 .header-detail--bold {font-size:0.6em; font-weight:bold; margin-bottom:0.75em;text-align:left;}
239 .header-detail {font-size:0.8em}
240 time {font-size:0.6em; font-weight: normal; margin-bottom:0.75em; text-align:left; display:block;}
241 .copy--intro,.header-teaser,.scrolly-intro {font-size:1em; font-style:italic; font-weight:bold;margin-bottom:1em;}
242 .dict-hl {font-size:1em; font-style:italic; font-weight:bold}
243 .tik4-media-image,.dict-box {margin-bottom:1em;margin-top:1em;}
244 '''
245
246 keep_only_tags = [
247 dict(name='article', attrs={'class': ['article', 'storytelling']}),
248 dict(name='body'),
249 dict(name='div', attrs={'class': ['imageGallery', 'image_only']}),
250 dict(name='div', attrs={'class': 'tik4-live__container'}),
251 dict(name='script', attrs={'id': '__NUXT_DATA__'}),
252 dict(name='figure', attrs={'class': 'image'}),
253 ]
254
255 remove_tags = [
256 dict(
257 name='div',
258 attrs={
259 'class': [
260 'tik4-sharing',
261 'tik4-load-more-bottom',
262 'tik4-by',
263 'header-detail__image',
264 'mm-adbox',
265 'upper-toolbar',
266 'tik4-chapters__label',
267 'tik4-we-toggle',
268 ]
269 },
270 ),
271 dict(name='svg'),
272 dict(name='div', attrs={'data-module': 'teaser'}),
273 dict(
274 attrs={
275 'data-external-selector': [
276 'upper-toolbar',
277 'article-footer',
278 'related-articles',
279 'placeholder',
280 'media-label',
281 ]
282 }
283 ),
284 dict(name='img', attrs={'class': 'sm:hidden'}),
285 ]
286
287 remove_attributes = ['onclick', 'initialdata', 'sizes']
288
289 # To fetch a single article instead of the full set of RSS feeds (handy
290 # while debugging a markup change), set TEST_ARTICLE_URL below.
291 TEST_ARTICLE_URL = None
292 # "https://www.faz.net/aktuell/besser-leben/stil/bilderstrecke-mode-accessoires-fuer-maenner-16997909.html"
293
294 if TEST_ARTICLE_URL:
295
296 def parse_index(self):
297 return [('Articles', [{'title': 'Test article', 'url': self.TEST_ARTICLE_URL}])]
298
299 else:
300 feeds = [
301 ('FAZ.NET Aktuell', 'https://www.faz.net/rss/aktuell/'),
302 ('Politik', 'https://www.faz.net/rss/aktuell/politik/'),
303 ('Wirtschaft', 'https://www.faz.net/rss/aktuell/wirtschaft/'),
304 ('Feuilleton', 'https://www.faz.net/rss/aktuell/feuilleton/'),
305 ('Sport', 'https://www.faz.net/rss/aktuell/sport/'),
306 ('Lebensstil', 'https://www.faz.net/rss/aktuell/besser-leben/stil/'),
307 ('Gesellschaft', 'https://www.faz.net/rss/aktuell/gesellschaft/'),
308 ('Finanzen', 'https://www.faz.net/rss/aktuell/finanzen/'),
309 ('Technik & Motor', 'https://www.faz.net/rss/aktuell/technik-motor/'),
310 ('Wissen', 'https://www.faz.net/rss/aktuell/wissen/'),
311 ('Reise', 'https://www.faz.net/rss/aktuell/reise/'),
312 ('Karriere & Hochschule', 'https://www.faz.net/rss/aktuell/karriere-hochschule/'),
313 ('Rhein-Main', 'https://www.faz.net/rss/aktuell/rhein-main/'),
314 ]
315
316 def _unpack_storytelling_article(self, soup):
317 """Render "storytelling" articles whose body is embedded as JSON.
318
319 These articles ship an empty <article class="storytelling"> shell;
320 the actual content lives inside a __NUXT_DATA__ script tag and has
321 to be parsed out and re-inserted as HTML.
322 """
323 article_tag = soup.find(name='article', attrs={'class': 'storytelling'})
324 if not (article_tag and not article_tag.get_text().strip()):
325 return
326
327 nuxt_script_tag = soup.find('script', attrs={'id': '__NUXT_DATA__', 'type': 'application/json'})
328 if not nuxt_script_tag:
329 return
330
331 try:
332 json_data = json.loads(str(nuxt_script_tag.contents[0]))
333 for item in json_data:
334 if not (isinstance(item, dict) and 'body' in item):
335 continue
336 body = json_data[int(item['body'])]
337 if not body:
338 continue
339 new_soup = BeautifulSoup(body, 'html.parser')
340 soup.append(new_soup)
341 storytelling_tag = soup.find(attrs={'data-content': 'storytelling'})
342 if storytelling_tag:
343 storytelling_tag.unwrap()
344 break
345 except Exception:
346 pass
347
348 def preprocess_html(self, soup):
349 self._unpack_storytelling_article(soup)
350
351 # Format hovering images: resolve lazy-loaded src, and drop duplicate
352 # captions that immediately repeat the same text.
353 for figure_tag in soup.find_all('figure', class_='image'):
354 img_tag = figure_tag.find('img', attrs={'data-src': True})
355 if img_tag:
356 img_tag['src'] = img_tag['data-src']
357 del img_tag['data-src']
358 caption_tag = figure_tag.find('figcaption')
359 if caption_tag:
360 next_caption = caption_tag.find_next('figcaption')
361 if next_caption and caption_tag.get_text(strip=True) == next_caption.get_text(strip=True):
362 next_caption.extract()
363
364 # Move the headline in front of the lead image, if it isn't already.
365 lead_image = soup.find('figure', attrs={'class': 'image'})
366 h1 = soup.find('h1')
367 if lead_image and h1 and h1.parent and not h1.parent.find(lead_image):
368 lead_image.insert_before(h1.parent.extract())
369
370 # Normalize all figure captions to <span class="caption">.
371 for tag in soup.find_all('figcaption'):
372 tag['class'] = 'caption'
373 tag.name = 'span'
374
375 # Format the "header-detail" byline/date block into a plain span.
376 header_detail_selectors = ['header-detail', 'header-detail-opinion header-detail']
377 header_detail_tag = soup.find(attrs={'data-external-selector': header_detail_selectors})
378 if header_detail_tag:
379 span = soup.new_tag('span')
380 span['class'] = 'header-detail'
381 span.string = header_detail_tag.get_text(separator=' ', strip=True).replace(' ,', ',')
382 if header_detail_tag.img and header_detail_tag.img.parent and header_detail_tag != header_detail_tag.img.parent:
383 header_detail_tag.insert_before(header_detail_tag.img.parent.extract())
384 header_detail_tag.replace_with(span)
385
386 # Format "story"-type articles (moves the first caption next to the
387 # first image).
388 if soup.find(class_='storyContainer'):
389 story(soup, None)
390
391 # Extract images and text from "Bilderstrecke" image galleries: these
392 # show up in the body as a lone "<n> Bilder" paragraph, with the
393 # actual gallery data embedded in a __NUXT_DATA__ script tag.
394 for par in soup.find_all('p'):
395 if len(par.contents) != 1:
396 continue
397 content = str(par.contents[0]).strip()
398 if re.search(r'^[1-9]\d* Bilder$', content):
399 par.extract()
400 nuxt_script_tag = soup.find('script', attrs={'id': '__NUXT_DATA__', 'type': 'application/json'})
401 if nuxt_script_tag:
402 bilderstrecke(soup, nuxt_script_tag)
403 break
404
405 # Unwrap buttons, dropping "Weitere anzeigen" (show more) entirely.
406 for tag in soup.find_all('button'):
407 if 'Weitere anzeigen' in tag.get_text():
408 tag.extract()
409 else:
410 tag.unwrap()
411
412 # Rename the section-label span to a class we style explicitly.
413 label_tag = soup.find('span', class_='sm:o2-medium')
414 if label_tag:
415 label_tag['class'] = 'header-label__content'
416
417 # Drop the screen-reader-only colon inside the section label.
418 header_label_tag = soup.find(class_='header-label__content')
419 if header_label_tag:
420 colon_tag = header_label_tag.find(class_='sr-only')
421 if colon_tag:
422 colon_tag.extract()
423
424 # Clean up <h2> headers: turn nested <div>s into <span>s.
425 h2_tag = soup.find('h2')
426 if h2_tag:
427 for div_tag in h2_tag.find_all('div'):
428 div_tag.name = 'span'
429
430 # Format the "header-teaser" lead paragraph.
431 header_teaser_tag = soup.find(attrs={'data-external-selector': 'header-teaser'})
432 if header_teaser_tag:
433 header_teaser_tag['class'] = 'header-teaser'
434
435 # Skip articles behind the paywall.
436 if soup.find(id='faz-paywall'):
437 self.abort_article()
438
439 # Remove the F.A.Z. newsletter ad.
440 for tag in soup.find_all(attrs={'class': 'body-elements__paragraph'}):
441 if tag.contents and isinstance(tag.contents[0], NavigableString):
442 if 'F.A.Z.-Newsletter' in str(tag.contents[0]):
443 tag.extract()
444
445 # Remove the "prefer us on Google" nudge (paragraph and link forms).
446 for tag in soup.find_all('p'):
447 if 'in Ihren Suchergebnissen sehen' in tag.get_text():
448 tag.extract()
449 for tag in soup.find_all('a'):
450 if 'bei Google bevorzugen' in tag.get_text():
451 tag.extract()
452
453 # Reformat Tickaroo live blogs.
454 if soup.find(attrs={'class': 'tik4-live__container'}):
455 format_tickaroo_liveblog(self, soup)
456
457 # Resolve any remaining srcset-only images.
458 for img_tag in soup.find_all('img', srcset=True):
459 find_image_source(img_tag)
460
461 # Resolve <picture><source srcset=...> images.
462 for picture_tag in soup.find_all('picture'):
463 source_tag = picture_tag.find('source', srcset=True)
464 img_tag = picture_tag.find('img') if source_tag else None
465 if img_tag:
466 img_tag['src'] = pick_srcset_source(source_tag['srcset'])
467
468 # Strip all image attributes except src/alt/title; drop images with
469 # no resolvable src entirely.
470 attributes_to_keep = {'src', 'alt', 'title'}
471 for img_tag in soup.find_all('img'):
472 if not img_tag.has_attr('src'):
473 img_tag.decompose()
474 continue
475 for attr in list(img_tag.attrs):
476 if attr not in attributes_to_keep:
477 del img_tag[attr]
478
479 return soup
480
481 def postprocess_html(self, soup, first_fetch):
482 # Remove the style attribute from certain image captions.
483 for tag in soup.select('div.meta2'):
484 if tag.has_attr('style'):
485 del tag['style']
486 if tag.span:
487 tag.span.unwrap()
488
489 # Unwrap any remaining <span> inside a caption.
490 for tag in soup.find_all(attrs={'class': 'caption'}):
491 span = tag.find('span')
492 if span:
493 span.unwrap()
494
495 # Add a trailing period between a figure caption and its credit,
496 # where the caption doesn't already end in punctuation.
497 caption_classes = [
498 'body-elements__image-figcaption',
499 'header-teaser__image-details',
500 'caption',
501 'meta2',
502 ]
503 for tag in soup.find_all(attrs={'class': caption_classes}):
504 if tag.string is None and tag.contents and tag.contents[0].string:
505 tag = tag.contents[0]
506 if tag.string:
507 text = str(tag.string).strip()
508 if text and text[-1] not in '.?!:':
509 tag.string.replace_with(text + '. ')
510
511 # Add a line break between an image and a following header-teaser.
512 for tag in soup.find_all(class_='header-teaser'):
513 tag.insert_before(soup.new_tag('br'))
514
515 # Wrap every image in its own <div>, stripped of layout attributes.
516 for img_tag in soup.find_all('img'):
517 for attrib in ('height', 'width', 'border', 'align', 'style'):
518 if img_tag.has_attr(attrib):
519 del img_tag[attrib]
520 parent = img_tag.parent
521 index = parent.contents.index(img_tag)
522 img_tag.extract()
523 div_tag = soup.new_tag('div')
524 parent.insert(index, div_tag)
525 div_tag.append(img_tag)
526
527 return soup