Debian Codegraph @

recipes/zeitde.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 __license__ = 'GPL v3'
4 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
5
6 '''
7 Fetch Zeit-Online.de
8 '''
9 import re
10 from datetime import datetime
11
12 from calibre.web.feeds.news import BasicNewsRecipe
13
14 try:
15 from http.cookiejar import Cookie
16 except ImportError:
17 from cookielib import Cookie
18
19 try:
20 from urllib.error import HTTPError
21 from urllib.request import Request, urlopen
22 except ImportError:
23 from urllib2 import HTTPError, Request, urlopen
24
25 from calibre.ebooks.BeautifulSoup import BeautifulSoup
26
27
28 class ZeitDe(BasicNewsRecipe):
29 __author__ = 'Armin Geller, Thomas Hollstegge' # AGe 2014-02-26
30 title = 'Zeit Online'
31 description = 'German online portal of newspaper Die Zeit'
32 publisher = 'ZEIT ONLINE GmbH'
33 category = 'news, Germany'
34 timefmt = ' [%a, %d %b %Y]'
35 publication_type = 'newspaper'
36 language = 'de'
37 encoding = 'UTF-8'
38
39 oldest_article = 7
40 max_articles_per_feed = 100
41 remove_empty_feeds = True
42 auto_cleanup = True
43 auto_cleanup_keep = '//header[@class="article-header"]|//div[@class="gallery__media-container"]|//div[@class="article__media-container"]'
44
45 masthead_url = 'https://static.zeit.de/p/zeit.web/3.714/images/structured-data-publisher-logo-zon.png'
46
47 extra_css = '''.figure__text { font-size: 0.9em; font-style: italic; }
48 .figure__copyright { font-size: 0.9em; font-style: italic; color: #888; }
49 .article-heading__kicker { font-size: 0.5em; display: block; margin-bottom: 1em; }
50 p.summary { font-size: 1.3em; font-style: italic; } '''
51
52 remove_tags = [
53 dict(name='aside', class_='topicbox'),
54 dict(name='aside', class_='article-toc'),
55 dict(class_='visually-hidden'),
56 dict(name='a', class_='faq-link'),
57 ]
58
59 recipe_specific_options = {
60 'days': {
61 'short': 'Oldest article to download from this news source. In days ',
62 'long': 'For example, 0.5, gives you articles from the past 12 hours',
63 'default': str(oldest_article),
64 }
65 }
66
67 def __init__(self, *args, **kwargs):
68 BasicNewsRecipe.__init__(self, *args, **kwargs)
69 d = self.recipe_specific_options.get('days')
70 if d and isinstance(d, str):
71 self.oldest_article = float(d)
72
73 feeds = [
74 ('Startseite – Die wichtigsten Themen auf einen Blick', 'https://newsfeed.zeit.de/index'),
75 ('Politik – Ausland und Deutschland', 'https://newsfeed.zeit.de/politik/index'),
76 ('Gesellschaft – Gesellschaft und soziales Leben', 'https://newsfeed.zeit.de/gesellschaft/index'),
77 ('Wirtschaft – Wirtschaft und Unternehmen', 'https://newsfeed.zeit.de/wirtschaft/index'),
78 ('Kultur – Literatur, Kunst, Film und Musik', 'https://newsfeed.zeit.de/kultur/index'),
79 ('Wissen – Wissenschaft, Gesundheit, Umwelt und Geschichte', 'https://newsfeed.zeit.de/wissen/index'),
80 ('Digital – Hardware, Software, Internet, Datenschutz', 'https://newsfeed.zeit.de/digital/index'),
81 ('ZEIT Campus Online - Studieren, arbeiten, leben', 'https://newsfeed.zeit.de/campus/index'),
82 ('Arbeit - Umbruch, Neuanfang, Erfolg, Zweifel', 'https://newsfeed.zeit.de/arbeit/index'),
83 ('Entdecken - Die Poesie des Alltäglichen', 'https://newsfeed.zeit.de/entdecken/index'),
84 ('Mobilität – Wie wir uns fortbewegen', 'https://newsfeed.zeit.de/mobilitaet/index'), # AGe 2014-01-09
85 ('Sport – Sieg und Niederlage', 'https://newsfeed.zeit.de/sport/index'),
86 ]
87
88 def get_browser(self):
89 br = BasicNewsRecipe.get_browser(self)
90
91 # Add a cookie indicating we have accepted the cookie
92 # policy
93 ck = Cookie(
94 version=0,
95 name='zonconsent',
96 value=datetime.now().isoformat(),
97 port=None,
98 port_specified=False,
99 domain='.zeit.de',
100 domain_specified=False,
101 domain_initial_dot=True,
102 path='/',
103 path_specified=False,
104 secure=True,
105 expires=None,
106 discard=False,
107 comment=None,
108 comment_url=None,
109 rest={'HttpOnly': None},
110 rfc2109=False,
111 )
112 br.cookiejar.set_cookie(ck)
113 return br
114
115 def print_version(self, url):
116 # If there is a complete page, use that one
117 req = Request(url=url + '/komplettansicht')
118 req.get_method = lambda: 'HEAD'
119 try:
120 urlopen(req)
121 return url + '/komplettansicht'
122 except HTTPError:
123 return url
124
125 def preprocess_raw_html(self, raw_html, url):
126 soup = BeautifulSoup(raw_html)
127
128 # Skip articles behind paywall
129 if soup.find('meta', property='lp:paywall'):
130 self.abort_article()
131
132 # Readability may strip the header for multipage articles, so simply
133 # include it in the main tag
134 if soup.find('body')['data-page-type'] == 'article':
135 body = soup.find('div', {'class': 'article-body'})
136 header = soup.find('header', {'class': 'article-header'})
137 if header is not None:
138 header.extract()
139 body.insert(0, header)
140
141 # Add real img tags for images
142 for container in soup.findAll(class_=re.compile(r'__media-container$')):
143 img = container.find('noscript')
144 if img is not None:
145 img.name = 'div'
146 img['data-src'] = ''
147
148 return soup.prettify()