| 1 |
#!/usr/bin/env python |
| 2 |
# vim:fileencoding=utf-8 |
| 3 |
# License: GPLv3 Copyright: 2016, Aimylios <aimylios at gmx.de> |
| 4 |
|
| 5 |
from __future__ import absolute_import, division, print_function, unicode_literals |
| 6 |
|
| 7 |
''' |
| 8 |
handelsblatt.com |
| 9 |
''' |
| 10 |
|
| 11 |
import re |
| 12 |
|
| 13 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 14 |
|
| 15 |
|
| 16 |
class Handelsblatt(BasicNewsRecipe): |
| 17 |
title = 'Handelsblatt' |
| 18 |
__author__ = 'Aimylios' # based on the work of malfi and Hegi |
| 19 |
description = 'RSS-Feeds von Handelsblatt.com' |
| 20 |
publisher = 'Verlagsgruppe Handelsblatt GmbH' |
| 21 |
publication_type = 'newspaper' |
| 22 |
needs_subscription = 'optional' |
| 23 |
language = 'de' |
| 24 |
|
| 25 |
oldest_article = 2 |
| 26 |
max_articles_per_feed = 15 |
| 27 |
simultaneous_downloads = 10 |
| 28 |
no_stylesheets = True |
| 29 |
remove_javascript = True |
| 30 |
remove_empty_feeds = True |
| 31 |
ignore_duplicate_articles = {'title', 'url'} |
| 32 |
|
| 33 |
conversion_options = {'smarten_punctuation': True, 'publisher': publisher} |
| 34 |
|
| 35 |
# uncomment this to reduce file size |
| 36 |
# compress_news_images = True |
| 37 |
# compress_news_images_max_size = 16 |
| 38 |
|
| 39 |
login_url = 'https://id.handelsblatt.com/login?service=https://www.handelsblatt.com' |
| 40 |
cover_source = 'https://www.ikiosk.de/shop/epaper/handelsblatt.html' |
| 41 |
masthead_url = 'https://www.handelsblatt.com/images/logo_handelsblatt/11002806/8-formatOriginal.png' |
| 42 |
|
| 43 |
feeds = [ |
| 44 |
('Top-Themen', 'http://www.handelsblatt.com/contentexport/feed/top-themen'), |
| 45 |
('Politik', 'http://www.handelsblatt.com/contentexport/feed/politik'), |
| 46 |
('Unternehmen', 'http://www.handelsblatt.com/contentexport/feed/unternehmen'), |
| 47 |
('Finanzen', 'http://www.handelsblatt.com/contentexport/feed/finanzen'), |
| 48 |
('Technologie', 'http://www.handelsblatt.com/contentexport/feed/technologie'), |
| 49 |
('Panorama', 'http://www.handelsblatt.com/contentexport/feed/panorama'), |
| 50 |
] |
| 51 |
|
| 52 |
keep_only_tags = [dict(name='div', attrs={'class': ['vhb-article-container']})] |
| 53 |
|
| 54 |
remove_tags = [ |
| 55 |
dict(name='span', attrs={'class': ['vhb-colon', 'vhb-label-premium']}), |
| 56 |
dict( |
| 57 |
name='aside', |
| 58 |
attrs={'class': ['vhb-article-element vhb-left', 'vhb-article-element vhb-left vhb-teasergallery', 'vhb-article-element vhb-left vhb-shorttexts']}, |
| 59 |
), |
| 60 |
dict(name='aside', attrs={'class': re.compile(r'vhb-club-events')}), |
| 61 |
dict(name='article', attrs={'class': ['vhb-imagegallery vhb-teaser', 'vhb-teaser vhb-type-video']}), |
| 62 |
dict(name='small', attrs={'class': ['vhb-credit']}), |
| 63 |
dict( |
| 64 |
name='div', |
| 65 |
attrs={ |
| 66 |
'class': [ |
| 67 |
'dg_health', |
| 68 |
'fb-post', |
| 69 |
'header-bar', |
| 70 |
'lb_consent--placeholder', |
| 71 |
'lb-item embed', |
| 72 |
'lb-post-actions', |
| 73 |
'mod--displaynone', |
| 74 |
'white_content', |
| 75 |
'opinary-widget-wrapper', |
| 76 |
'vhb-article__content-element--shorttextgallery', |
| 77 |
'vhb-hollow-area vhb-hollow-area--col-1', |
| 78 |
] |
| 79 |
}, |
| 80 |
), |
| 81 |
dict(name='div', attrs={'class': re.compile(r'stepstone')}), |
| 82 |
dict(name='div', attrs={'class': re.compile(r'vhb-imagegallery')}), |
| 83 |
dict(name='div', attrs={'id': ['highcharts_infografik']}), |
| 84 |
dict(name='div', attrs={'id': re.compile(r'dax-sentiment')}), |
| 85 |
dict(name=['div', 'section'], attrs={'class': re.compile(r'slider')}), |
| 86 |
dict(name='a', attrs={'class': ['twitter-follow-button']}), |
| 87 |
dict(name='img', attrs={'class': ['highlight-icon', 'lb-author__avatar', 'pin-icon']}), |
| 88 |
dict(name='img', attrs={'alt': re.compile(r'Handelsblatt Morning Briefing')}), |
| 89 |
dict(name=['blockquote', 'button', 'link']), |
| 90 |
] |
| 91 |
|
| 92 |
preprocess_regexps = [ |
| 93 |
# insert ". " after the location at the beginning of an article |
| 94 |
(re.compile(r'(<span class="hcf-location-mark">[^<]+)(</span>)', re.IGNORECASE), lambda match: match.group(1) + '. ' + match.group(2)), |
| 95 |
# insert ": " between title and text of captions |
| 96 |
( |
| 97 |
re.compile(r'(<em itemtype="text" itemprop="name" class="vhb-title">[^<]+)(</em>)', re.IGNORECASE), |
| 98 |
lambda match: match.group(1) + ': ' + match.group(2), |
| 99 |
), |
| 100 |
# convert "data-src" to "src" attributes |
| 101 |
(re.compile(r'( data-src=")([^"]*")', re.IGNORECASE), lambda match: ' src="' + match.group(2)), |
| 102 |
] |
| 103 |
|
| 104 |
extra_css = '''h2 {font-size: 1em; text-align: left} |
| 105 |
h3 {font-size: 1em; text-align: left} |
| 106 |
h4 {font-size: 1em; text-align: left; margin-bottom: 0em} |
| 107 |
em {font-style: normal; font-weight: bold} |
| 108 |
.vhb-subline {font-weight: normal; text-transform: uppercase} |
| 109 |
.vhb-headline {font-size: 1.6em} |
| 110 |
.vhb-teaser-head {margin-top: 1em; margin-bottom: 1em} |
| 111 |
.vhb-hollow-area--innercontent {font-size: 0.6em} |
| 112 |
.hcf-location-mark {font-weight: bold} |
| 113 |
.lb-post-header {margin-top: 1em} |
| 114 |
.panel-body p {margin-top: 0em}''' |
| 115 |
|
| 116 |
def get_browser(self): |
| 117 |
br = BasicNewsRecipe.get_browser(self) |
| 118 |
if self.username is not None and self.password is not None: |
| 119 |
try: |
| 120 |
br.open(self.login_url) |
| 121 |
br.select_form(nr=0) |
| 122 |
br['login'] = self.username |
| 123 |
br['password'] = self.password |
| 124 |
br.submit() |
| 125 |
except Exception as e: |
| 126 |
self.log('Login failed with error:', str(e)) |
| 127 |
return br |
| 128 |
|
| 129 |
def get_cover_url(self): |
| 130 |
cover_url = '' |
| 131 |
soup = self.index_to_soup(self.cover_source) |
| 132 |
img = soup.find('img', {'alt': 'Handelsblatt - ePaper;', 'src': True}) |
| 133 |
if img and img['src']: |
| 134 |
cover_url = img['src'] |
| 135 |
return cover_url |
| 136 |
|
| 137 |
def print_version(self, url): |
| 138 |
main, sep, name = url.rpartition('/') |
| 139 |
return main + '/v_detail_tab_print/' + name |
| 140 |
|
| 141 |
def preprocess_html(self, soup): |
| 142 |
# remove all articles without relevant content (e.g., videos) |
| 143 |
article = soup.find('div', {'class': 'vhb-article-container'}) |
| 144 |
if article is None: |
| 145 |
self.abort_article() |
| 146 |
return soup |
| 147 |
|
| 148 |
def postprocess_html(self, soup, first_fetch): |
| 149 |
# convert lists of author(s) and date(s) into simple text |
| 150 |
for cap in soup.find_all('div', {'class': re.compile(r'vhb-article-caption')}): |
| 151 |
cap.replace_with(cap.encode_contents().decode('utf-8').strip() + ' ') |
| 152 |
for row in soup.find_all('div', {'class': 'vhb-article-author-row'}): |
| 153 |
for ul in row.find_all('ul'): |
| 154 |
entry = '' |
| 155 |
for li in ul.find_all(lambda tag: tag.name == 'li' and not tag.attrs): |
| 156 |
entry += self.tag_to_string(li).strip() + ', ' |
| 157 |
for li in ul.find_all(lambda tag: tag.name == 'li' and tag.attrs): |
| 158 |
entry += self.tag_to_string(li) |
| 159 |
ul.parent.replace_with(entry) |
| 160 |
# remove all local hyperlinks |
| 161 |
for a in soup.find_all('a', {'href': True}): |
| 162 |
if '.handelsblatt.com/' in a['href']: |
| 163 |
a.replace_with(a.encode_contents().decode('utf-8')) |
| 164 |
# make sure that all figure captions (including the source) are shown |
| 165 |
# without linebreaks by using the alternative text given within <img/> |
| 166 |
# instead of the original text (which is oddly formatted) |
| 167 |
for fig in soup.find_all('figcaption', {'class': 'vhb-inline-picture'}): |
| 168 |
cap = fig.find('img')['alt'] |
| 169 |
fig.find('div', {'class': 'vhb-caption'}).replace_with(cap) |
| 170 |
# remove references to related articles |
| 171 |
for strong in soup.find_all('strong'): |
| 172 |
if strong.string and (re.match(r'^Mehr:? ?', strong.string) or re.match(r'^>>.*', strong.string)): |
| 173 |
p_parent = strong.find_parent('p') |
| 174 |
if p_parent: |
| 175 |
p_parent.decompose() |
| 176 |
# clean up remainders of embedded content |
| 177 |
for div in soup.find_all('div', {'style': True}): |
| 178 |
if len(div.attrs) == 1: |
| 179 |
del div['style'] |
| 180 |
return soup |