| 1 |
#!/usr/bin/env python |
| 2 |
# vim:fileencoding=utf-8 |
| 3 |
__license__ = 'GPL v3' |
| 4 |
__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>' |
| 5 |
|
| 6 |
''' |
| 7 |
Fetch Zeit-Online.de |
| 8 |
''' |
| 9 |
import re |
| 10 |
from datetime import datetime |
| 11 |
|
| 12 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 13 |
|
| 14 |
try: |
| 15 |
from http.cookiejar import Cookie |
| 16 |
except ImportError: |
| 17 |
from cookielib import Cookie |
| 18 |
|
| 19 |
try: |
| 20 |
from urllib.error import HTTPError |
| 21 |
from urllib.request import Request, urlopen |
| 22 |
except ImportError: |
| 23 |
from urllib2 import HTTPError, Request, urlopen |
| 24 |
|
| 25 |
from calibre.ebooks.BeautifulSoup import BeautifulSoup |
| 26 |
|
| 27 |
|
| 28 |
class ZeitDe(BasicNewsRecipe): |
| 29 |
__author__ = 'Armin Geller, Thomas Hollstegge' # AGe 2014-02-26 |
| 30 |
title = 'Zeit Online' |
| 31 |
description = 'German online portal of newspaper Die Zeit' |
| 32 |
publisher = 'ZEIT ONLINE GmbH' |
| 33 |
category = 'news, Germany' |
| 34 |
timefmt = ' [%a, %d %b %Y]' |
| 35 |
publication_type = 'newspaper' |
| 36 |
language = 'de' |
| 37 |
encoding = 'UTF-8' |
| 38 |
|
| 39 |
oldest_article = 7 |
| 40 |
max_articles_per_feed = 100 |
| 41 |
remove_empty_feeds = True |
| 42 |
auto_cleanup = True |
| 43 |
auto_cleanup_keep = '//header[@class="article-header"]|//div[@class="gallery__media-container"]|//div[@class="article__media-container"]' |
| 44 |
|
| 45 |
masthead_url = 'https://static.zeit.de/p/zeit.web/3.714/images/structured-data-publisher-logo-zon.png' |
| 46 |
|
| 47 |
extra_css = '''.figure__text { font-size: 0.9em; font-style: italic; } |
| 48 |
.figure__copyright { font-size: 0.9em; font-style: italic; color: #888; } |
| 49 |
.article-heading__kicker { font-size: 0.5em; display: block; margin-bottom: 1em; } |
| 50 |
p.summary { font-size: 1.3em; font-style: italic; } ''' |
| 51 |
|
| 52 |
remove_tags = [ |
| 53 |
dict(name='aside', class_='topicbox'), |
| 54 |
dict(name='aside', class_='article-toc'), |
| 55 |
dict(class_='visually-hidden'), |
| 56 |
dict(name='a', class_='faq-link'), |
| 57 |
] |
| 58 |
|
| 59 |
recipe_specific_options = { |
| 60 |
'days': { |
| 61 |
'short': 'Oldest article to download from this news source. In days ', |
| 62 |
'long': 'For example, 0.5, gives you articles from the past 12 hours', |
| 63 |
'default': str(oldest_article), |
| 64 |
} |
| 65 |
} |
| 66 |
|
| 67 |
def __init__(self, *args, **kwargs): |
| 68 |
BasicNewsRecipe.__init__(self, *args, **kwargs) |
| 69 |
d = self.recipe_specific_options.get('days') |
| 70 |
if d and isinstance(d, str): |
| 71 |
self.oldest_article = float(d) |
| 72 |
|
| 73 |
feeds = [ |
| 74 |
('Startseite – Die wichtigsten Themen auf einen Blick', 'https://newsfeed.zeit.de/index'), |
| 75 |
('Politik – Ausland und Deutschland', 'https://newsfeed.zeit.de/politik/index'), |
| 76 |
('Gesellschaft – Gesellschaft und soziales Leben', 'https://newsfeed.zeit.de/gesellschaft/index'), |
| 77 |
('Wirtschaft – Wirtschaft und Unternehmen', 'https://newsfeed.zeit.de/wirtschaft/index'), |
| 78 |
('Kultur – Literatur, Kunst, Film und Musik', 'https://newsfeed.zeit.de/kultur/index'), |
| 79 |
('Wissen – Wissenschaft, Gesundheit, Umwelt und Geschichte', 'https://newsfeed.zeit.de/wissen/index'), |
| 80 |
('Digital – Hardware, Software, Internet, Datenschutz', 'https://newsfeed.zeit.de/digital/index'), |
| 81 |
('ZEIT Campus Online - Studieren, arbeiten, leben', 'https://newsfeed.zeit.de/campus/index'), |
| 82 |
('Arbeit - Umbruch, Neuanfang, Erfolg, Zweifel', 'https://newsfeed.zeit.de/arbeit/index'), |
| 83 |
('Entdecken - Die Poesie des Alltäglichen', 'https://newsfeed.zeit.de/entdecken/index'), |
| 84 |
('Mobilität – Wie wir uns fortbewegen', 'https://newsfeed.zeit.de/mobilitaet/index'), # AGe 2014-01-09 |
| 85 |
('Sport – Sieg und Niederlage', 'https://newsfeed.zeit.de/sport/index'), |
| 86 |
] |
| 87 |
|
| 88 |
def get_browser(self): |
| 89 |
br = BasicNewsRecipe.get_browser(self) |
| 90 |
|
| 91 |
# Add a cookie indicating we have accepted the cookie |
| 92 |
# policy |
| 93 |
ck = Cookie( |
| 94 |
version=0, |
| 95 |
name='zonconsent', |
| 96 |
value=datetime.now().isoformat(), |
| 97 |
port=None, |
| 98 |
port_specified=False, |
| 99 |
domain='.zeit.de', |
| 100 |
domain_specified=False, |
| 101 |
domain_initial_dot=True, |
| 102 |
path='/', |
| 103 |
path_specified=False, |
| 104 |
secure=True, |
| 105 |
expires=None, |
| 106 |
discard=False, |
| 107 |
comment=None, |
| 108 |
comment_url=None, |
| 109 |
rest={'HttpOnly': None}, |
| 110 |
rfc2109=False, |
| 111 |
) |
| 112 |
br.cookiejar.set_cookie(ck) |
| 113 |
return br |
| 114 |
|
| 115 |
def print_version(self, url): |
| 116 |
# If there is a complete page, use that one |
| 117 |
req = Request(url=url + '/komplettansicht') |
| 118 |
req.get_method = lambda: 'HEAD' |
| 119 |
try: |
| 120 |
urlopen(req) |
| 121 |
return url + '/komplettansicht' |
| 122 |
except HTTPError: |
| 123 |
return url |
| 124 |
|
| 125 |
def preprocess_raw_html(self, raw_html, url): |
| 126 |
soup = BeautifulSoup(raw_html) |
| 127 |
|
| 128 |
# Skip articles behind paywall |
| 129 |
if soup.find('meta', property='lp:paywall'): |
| 130 |
self.abort_article() |
| 131 |
|
| 132 |
# Readability may strip the header for multipage articles, so simply |
| 133 |
# include it in the main tag |
| 134 |
if soup.find('body')['data-page-type'] == 'article': |
| 135 |
body = soup.find('div', {'class': 'article-body'}) |
| 136 |
header = soup.find('header', {'class': 'article-header'}) |
| 137 |
if header is not None: |
| 138 |
header.extract() |
| 139 |
body.insert(0, header) |
| 140 |
|
| 141 |
# Add real img tags for images |
| 142 |
for container in soup.findAll(class_=re.compile(r'__media-container$')): |
| 143 |
img = container.find('noscript') |
| 144 |
if img is not None: |
| 145 |
img.name = 'div' |
| 146 |
img['data-src'] = '' |
| 147 |
|
| 148 |
return soup.prettify() |