Debian Codegraph @

recipes/handelsblatt.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 # License: GPLv3 Copyright: 2016, Aimylios <aimylios at gmx.de>
4
5 from __future__ import absolute_import, division, print_function, unicode_literals
6
7 '''
8 handelsblatt.com
9 '''
10
11 import re
12
13 from calibre.web.feeds.news import BasicNewsRecipe
14
15
16 class Handelsblatt(BasicNewsRecipe):
17 title = 'Handelsblatt'
18 __author__ = 'Aimylios' # based on the work of malfi and Hegi
19 description = 'RSS-Feeds von Handelsblatt.com'
20 publisher = 'Verlagsgruppe Handelsblatt GmbH'
21 publication_type = 'newspaper'
22 needs_subscription = 'optional'
23 language = 'de'
24
25 oldest_article = 2
26 max_articles_per_feed = 15
27 simultaneous_downloads = 10
28 no_stylesheets = True
29 remove_javascript = True
30 remove_empty_feeds = True
31 ignore_duplicate_articles = {'title', 'url'}
32
33 conversion_options = {'smarten_punctuation': True, 'publisher': publisher}
34
35 # uncomment this to reduce file size
36 # compress_news_images = True
37 # compress_news_images_max_size = 16
38
39 login_url = 'https://id.handelsblatt.com/login?service=https://www.handelsblatt.com'
40 cover_source = 'https://www.ikiosk.de/shop/epaper/handelsblatt.html'
41 masthead_url = 'https://www.handelsblatt.com/images/logo_handelsblatt/11002806/8-formatOriginal.png'
42
43 feeds = [
44 ('Top-Themen', 'http://www.handelsblatt.com/contentexport/feed/top-themen'),
45 ('Politik', 'http://www.handelsblatt.com/contentexport/feed/politik'),
46 ('Unternehmen', 'http://www.handelsblatt.com/contentexport/feed/unternehmen'),
47 ('Finanzen', 'http://www.handelsblatt.com/contentexport/feed/finanzen'),
48 ('Technologie', 'http://www.handelsblatt.com/contentexport/feed/technologie'),
49 ('Panorama', 'http://www.handelsblatt.com/contentexport/feed/panorama'),
50 ]
51
52 keep_only_tags = [dict(name='div', attrs={'class': ['vhb-article-container']})]
53
54 remove_tags = [
55 dict(name='span', attrs={'class': ['vhb-colon', 'vhb-label-premium']}),
56 dict(
57 name='aside',
58 attrs={'class': ['vhb-article-element vhb-left', 'vhb-article-element vhb-left vhb-teasergallery', 'vhb-article-element vhb-left vhb-shorttexts']},
59 ),
60 dict(name='aside', attrs={'class': re.compile(r'vhb-club-events')}),
61 dict(name='article', attrs={'class': ['vhb-imagegallery vhb-teaser', 'vhb-teaser vhb-type-video']}),
62 dict(name='small', attrs={'class': ['vhb-credit']}),
63 dict(
64 name='div',
65 attrs={
66 'class': [
67 'dg_health',
68 'fb-post',
69 'header-bar',
70 'lb_consent--placeholder',
71 'lb-item embed',
72 'lb-post-actions',
73 'mod--displaynone',
74 'white_content',
75 'opinary-widget-wrapper',
76 'vhb-article__content-element--shorttextgallery',
77 'vhb-hollow-area vhb-hollow-area--col-1',
78 ]
79 },
80 ),
81 dict(name='div', attrs={'class': re.compile(r'stepstone')}),
82 dict(name='div', attrs={'class': re.compile(r'vhb-imagegallery')}),
83 dict(name='div', attrs={'id': ['highcharts_infografik']}),
84 dict(name='div', attrs={'id': re.compile(r'dax-sentiment')}),
85 dict(name=['div', 'section'], attrs={'class': re.compile(r'slider')}),
86 dict(name='a', attrs={'class': ['twitter-follow-button']}),
87 dict(name='img', attrs={'class': ['highlight-icon', 'lb-author__avatar', 'pin-icon']}),
88 dict(name='img', attrs={'alt': re.compile(r'Handelsblatt Morning Briefing')}),
89 dict(name=['blockquote', 'button', 'link']),
90 ]
91
92 preprocess_regexps = [
93 # insert ". " after the location at the beginning of an article
94 (re.compile(r'(<span class="hcf-location-mark">[^<]+)(</span>)', re.IGNORECASE), lambda match: match.group(1) + '. ' + match.group(2)),
95 # insert ": " between title and text of captions
96 (
97 re.compile(r'(<em itemtype="text" itemprop="name" class="vhb-title">[^<]+)(</em>)', re.IGNORECASE),
98 lambda match: match.group(1) + ': ' + match.group(2),
99 ),
100 # convert "data-src" to "src" attributes
101 (re.compile(r'( data-src=")([^"]*")', re.IGNORECASE), lambda match: ' src="' + match.group(2)),
102 ]
103
104 extra_css = '''h2 {font-size: 1em; text-align: left}
105 h3 {font-size: 1em; text-align: left}
106 h4 {font-size: 1em; text-align: left; margin-bottom: 0em}
107 em {font-style: normal; font-weight: bold}
108 .vhb-subline {font-weight: normal; text-transform: uppercase}
109 .vhb-headline {font-size: 1.6em}
110 .vhb-teaser-head {margin-top: 1em; margin-bottom: 1em}
111 .vhb-hollow-area--innercontent {font-size: 0.6em}
112 .hcf-location-mark {font-weight: bold}
113 .lb-post-header {margin-top: 1em}
114 .panel-body p {margin-top: 0em}'''
115
116 def get_browser(self):
117 br = BasicNewsRecipe.get_browser(self)
118 if self.username is not None and self.password is not None:
119 try:
120 br.open(self.login_url)
121 br.select_form(nr=0)
122 br['login'] = self.username
123 br['password'] = self.password
124 br.submit()
125 except Exception as e:
126 self.log('Login failed with error:', str(e))
127 return br
128
129 def get_cover_url(self):
130 cover_url = ''
131 soup = self.index_to_soup(self.cover_source)
132 img = soup.find('img', {'alt': 'Handelsblatt - ePaper;', 'src': True})
133 if img and img['src']:
134 cover_url = img['src']
135 return cover_url
136
137 def print_version(self, url):
138 main, sep, name = url.rpartition('/')
139 return main + '/v_detail_tab_print/' + name
140
141 def preprocess_html(self, soup):
142 # remove all articles without relevant content (e.g., videos)
143 article = soup.find('div', {'class': 'vhb-article-container'})
144 if article is None:
145 self.abort_article()
146 return soup
147
148 def postprocess_html(self, soup, first_fetch):
149 # convert lists of author(s) and date(s) into simple text
150 for cap in soup.find_all('div', {'class': re.compile(r'vhb-article-caption')}):
151 cap.replace_with(cap.encode_contents().decode('utf-8').strip() + ' ')
152 for row in soup.find_all('div', {'class': 'vhb-article-author-row'}):
153 for ul in row.find_all('ul'):
154 entry = ''
155 for li in ul.find_all(lambda tag: tag.name == 'li' and not tag.attrs):
156 entry += self.tag_to_string(li).strip() + ', '
157 for li in ul.find_all(lambda tag: tag.name == 'li' and tag.attrs):
158 entry += self.tag_to_string(li)
159 ul.parent.replace_with(entry)
160 # remove all local hyperlinks
161 for a in soup.find_all('a', {'href': True}):
162 if '.handelsblatt.com/' in a['href']:
163 a.replace_with(a.encode_contents().decode('utf-8'))
164 # make sure that all figure captions (including the source) are shown
165 # without linebreaks by using the alternative text given within <img/>
166 # instead of the original text (which is oddly formatted)
167 for fig in soup.find_all('figcaption', {'class': 'vhb-inline-picture'}):
168 cap = fig.find('img')['alt']
169 fig.find('div', {'class': 'vhb-caption'}).replace_with(cap)
170 # remove references to related articles
171 for strong in soup.find_all('strong'):
172 if strong.string and (re.match(r'^Mehr:? ?', strong.string) or re.match(r'^>>.*', strong.string)):
173 p_parent = strong.find_parent('p')
174 if p_parent:
175 p_parent.decompose()
176 # clean up remainders of embedded content
177 for div in soup.find_all('div', {'style': True}):
178 if len(div.attrs) == 1:
179 del div['style']
180 return soup