Debian Codegraph @

recipes/zeitde_sub.recipe

raw
1 #!/usr/bin/env python
2 # -*- coding: utf-8 mode: python -*-
3 from __future__ import absolute_import, division, print_function, unicode_literals
4
5 __license__ = 'GPL v3'
6 __copyright__ = '2010, Steffen Siebert <calibre at steffensiebert.de>'
7 __docformat__ = 'restructuredtext de'
8 __version__ = '1.5'
9
10 '''
11 Die Zeit EPUB
12 '''
13
14 import io
15 import os
16 import re
17 import zipfile
18
19 from calibre import walk
20 from calibre.ptempfile import PersistentTemporaryFile
21 from calibre.web.feeds.news import BasicNewsRecipe
22
23 try:
24 from urllib.parse import urlparse
25 except ImportError:
26 from urlparse import urlparse
27 from contextlib import closing
28
29 from mechanize import Cookie
30
31 from calibre.utils.magick.draw import save_cover_data_to
32
33
34 class ZeitEPUBAbo(BasicNewsRecipe):
35 title = 'Die Zeit'
36 description = 'Das EPUB Abo der Zeit (needs subscription - use session-cookie as password)'
37 language = 'de'
38 lang = 'de-DE'
39
40 __author__ = 'Steffen Siebert, revised by Tobias Isenberg (with some code by Kovid Goyal), updated by Henning Losert and Fabian Schuller'
41 needs_subscription = True
42
43 conversion_options = {
44 'no_default_epub_cover': True,
45 # fixing the wrong left margin
46 'mobi_ignore_margins': True,
47 'keep_ligatures': True,
48 }
49
50 preprocess_regexps = [
51 # filtering for correct dashes ("Gedankenstrich" and "bis")
52 (re.compile(' (-|\u2212)(?=[ ,])'), lambda match: ' \u2013'), # noqa: RUF039
53 (re.compile(r'(?<=\d)-(?=\d)'), lambda match: '\u2013'), # number-number
54 (re.compile(r'(?<=\d,)-(?= ?\u20AC)'), lambda match: '\u2013'), # ,- Euro
55 # fix the number dash number dash for the title image that was broken
56 # by the previous line
57 (re.compile(r'(?<=\d\d\d\d)\u2013(?=\d?\d\.png)'), lambda match: '-'),
58 # filtering for certain dash cases
59 (re.compile(r'Bild - Zeitung'), lambda match: 'Bild-Zeitung'), # the obvious
60 (re.compile(r'EMail'), lambda match: 'E-Mail'), # the obvious
61 (re.compile(r'SBahn'), lambda match: 'S-Bahn'), # the obvious
62 (re.compile(r'UBoot'), lambda match: 'U-Boot'), # the obvious
63 (re.compile(r'T Shirt'), lambda match: 'T-Shirt'), # the obvious
64 (re.compile(r'TShirt'), lambda match: 'T-Shirt'), # the obvious
65 # the next two lines not only fix errors but also create new ones. this is due to additional errors in
66 # the typesetting such as missing commas or wrongly placed dashes. but
67 # more is fixed than broken.
68 # space too much before a connecting dash
69 (re.compile(r'(?<!und|der|\w\w,) -(?=\w)'), lambda match: '-'),
70 (
71 re.compile(r'(?<=\w)- (?!und\b|oder\b|wie\b|aber\b|auch\b|sondern\b|bis\b|&amp;|&\s|bzw\.|auf\b|eher\b)'),
72 lambda match: '-',
73 ), # space too much after a connecting dash
74 # filtering for missing spaces before the month in long dates
75 (re.compile(r'(?<=\d)\.(?=(Januar|Februar|M\u00E4rz|April|Mai|Juni|Juli|August|September|Oktober|November|Dezember))'), lambda match: '. '),
76 # filtering for other missing spaces
77 (re.compile(r'Stuttgart21'), lambda match: 'Stuttgart 21'), # the obvious
78 (re.compile(r'(?<=\d)(?=\u20AC)'), lambda match: '\u2013'), # Zahl[no space]Euro
79 # missing space after colon
80 (re.compile(r':(?=[^\d\s</])'), lambda match: ': '),
81 # missing space after closing quotation
82 (re.compile(r'\u00AB(?=[^\-\.:;,\?!<\)\s])'), lambda match: '\u00ab '),
83 # missing space before opening quotation
84 (re.compile(r'(?<=[^\s\(>])\u00BB'), lambda match: ' \u00bb'),
85 # missing space before Roman numeral
86 (re.compile(r'(?<=[a-z])(?=(I|II|III|IV|V|VI|VII|VIII|IX|X|XI|XII|XIII|XIV|XV|XVI|XVII|XVIII|XIX|XX)\.)'), lambda match: ' '),
87 # missing space after Roman numeral
88 (re.compile(r'(?<=(I|V|X)\.)(?=[\w])'), lambda match: ' '),
89 # missing space after Roman numeral
90 (re.compile(r'(?<=(II|IV|VI|IX|XI|XV|XX)\.)(?=[\w])'), lambda match: ' '),
91 # missing space after Roman numeral
92 (re.compile(r'(?<=(III|VII|XII|XIV|XVI|XIX)\.)(?=[\w])'), lambda match: ' '),
93 # missing space after Roman numeral
94 (re.compile(r'(?<=(VIII|XIII|XVII)\.)(?=[\w])'), lambda match: ' '),
95 # missing space after Roman numeral
96 (re.compile(r'(?<=(XVIII)\.)(?=[\w])'), lambda match: ' '),
97 (re.compile(r'(?<=[A-Za-zÄÖÜäöü]),(?=[A-Za-zÄÖÜäöü])'), lambda match: ', '), # missing space after comma
98 # missing space after full-stop
99 (re.compile(r'(?<=[a-zäöü])\.(?=[A-ZÄÖÜ][A-Za-zÄÖÜäöü])'), lambda match: '. '),
100 # fix abbreviation that was potentially broken previously
101 (re.compile(r'(?<=[uU]\.) (?=a\.)'), lambda match: '\u2008'),
102 # fix abbreviation that was potentially broken previously
103 (re.compile(r'(?<=[iI]\.) (?=A\.)'), lambda match: '\u2008'),
104 # fix abbreviation that was potentially broken previously
105 (re.compile(r'(?<=[zZ]\.) (?=B\.)'), lambda match: '\u2008'),
106 # fix e-mail address that was potentially broken previously
107 (re.compile(r'(?<=\w\.) (?=[A-Z][a-z]*@)'), lambda match: ''),
108 (re.compile(r'(?<=\d)[Pp]rozent'), lambda match: ' Prozent'),
109 (re.compile(r'\.\.\.\.+'), lambda match: '...'), # too many dots (....)
110 # spaces before ...
111 (re.compile(r'(?<=[^\s])\.\.\.'), lambda match: ' ...'),
112 # spaces after ...
113 (re.compile(r'\.\.\.(?=[^\s])'), lambda match: '... '),
114 # fix special cases of ... in brackets
115 (re.compile(r'(?<=[\[\(]) \.\.\. (?=[\]\)])'), lambda match: '...'),
116 # fix special cases of ... after a quotation mark
117 (re.compile(r'(?<=[\u00BB\u203A]) \.\.\.'), lambda match: '...'),
118 # fix special cases of ... before a quotation mark or comma
119 (re.compile(r'\.\.\. (?=[\u00AB\u2039,])'), lambda match: '...'),
120 # fix missing spaces between numbers and any sort of units, possibly
121 # with dot
122 (
123 re.compile(
124 r'(?<=\d)(?=(Femto|Piko|Nano|Mikro|Milli|Zenti|Dezi|Hekto|Kilo|Mega|Giga|Tera|Peta|Tausend|Trilli|Kubik|Quadrat|Meter|Uhr|Jahr|Schuljahr|Seite))'
125 ),
126 lambda match: ' ',
127 ), # noqa: E501
128 (
129 re.compile(
130 r'(?<=\d\.)(?=(Femto|Piko|Nano|Mikro|Milli|Zenti|Dezi|Hekto|Kilo|Mega|Giga|Tera|Peta|Tausend|Trilli|Kubik|Quadrat|Meter|Uhr|Jahr|Schuljahr|Seite))'
131 ),
132 lambda match: ' ',
133 ), # noqa: E501
134 # fix wrong spaces
135 # at beginning of paragraphs
136 (re.compile(r'(?<=<p class="absatz">[A-ZÄÖÜ]) (?=[a-zäöü\-])'), lambda match: ''),
137 # before closing quotation
138 (re.compile(' \u00ab'), lambda match: '\u00ab '), # noqa: RUF039
139 # after opening quotation
140 (re.compile('\u00bb '), lambda match: ' \u00bb'), # noqa: RUF039
141 # filtering for spaces in large numbers for better readability
142 # end of the number with some character following
143 (re.compile(r'(?<=\d\d)(?=\d\d\d[ ,;\)<\?!-])'), lambda match: '\u2008'),
144 # end of the number with full-stop following, then space is necessary
145 # (avoid file names)
146 (re.compile(r'(?<=\d\d)(?=\d\d\d. )'), lambda match: '\u2008'),
147 (re.compile(r'(?<=\d)(?=\d\d\d\u2008)'), lambda match: '\u2008'), # next level
148 (re.compile(r'(?<=\d)(?=\d\d\d\u2008)'), lambda match: '\u2008'), # next level
149 (re.compile(r'(?<=\d)(?=\d\d\d\u2008)'), lambda match: '\u2008'), # next level
150 (re.compile(r'(?<=\d)(?=\d\d\d\u2008)'), lambda match: '\u2008'), # next level
151 # filtering for unicode characters that are missing on the Kindle,
152 # try to replace them with meaningful work-arounds
153 # subscript-0
154 (re.compile('\u2080'), lambda match: '<span style="font-size: 40%;">0</span>'), # noqa: RUF039
155 # subscript-1
156 (re.compile('\u2081'), lambda match: '<span style="font-size: 40%;">1</span>'), # noqa: RUF039
157 # subscript-2
158 (re.compile('\u2082'), lambda match: '<span style="font-size: 40%;">2</span>'), # noqa: RUF039
159 # subscript-3
160 (re.compile('\u2083'), lambda match: '<span style="font-size: 40%;">3</span>'), # noqa: RUF039
161 # subscript-4
162 (re.compile('\u2084'), lambda match: '<span style="font-size: 40%;">4</span>'), # noqa: RUF039
163 # subscript-5
164 (re.compile('\u2085'), lambda match: '<span style="font-size: 40%;">5</span>'), # noqa: RUF039
165 # subscript-6
166 (re.compile('\u2086'), lambda match: '<span style="font-size: 40%;">6</span>'), # noqa: RUF039
167 # subscript-7
168 (re.compile('\u2087'), lambda match: '<span style="font-size: 40%;">7</span>'), # noqa: RUF039
169 # subscript-8
170 (re.compile('\u2088'), lambda match: '<span style="font-size: 40%;">8</span>'), # noqa: RUF039
171 # subscript-9
172 (re.compile('\u2089'), lambda match: '<span style="font-size: 40%;">9</span>'), # noqa: RUF039
173 # always chance CO2
174 (re.compile(r'CO2'), lambda match: 'CO<span style="font-size: 40%;">2</span>'), # CO2
175 # remove *** paragraphs
176 (re.compile(r'<p class="absatz">\*\*\*</p>'), lambda match: ''),
177 # better layout for the top line of each article
178 (re.compile(r'(?<=DIE ZEIT N\u00B0 \d /) (?=\d\d)'), lambda match: ' 20'), # proper year in edition number
179 (re.compile(r'(?<=DIE ZEIT N\u00B0 \d\d /) (?=\d\d)'), lambda match: ' 20'), # proper year in edition number
180 (re.compile(r'(?<=>)(?=DIE ZEIT N\u00B0 \d\d / 20\d\d)'), lambda match: ' \u2014 '), # m-dash between category and DIE ZEIT
181 ]
182
183 def build_index(self):
184 url = 'https://epaper.zeit.de/abo/diezeit'
185 browser = self.get_browser()
186
187 c = Cookie(
188 version=0,
189 name='zeit_sso_201501',
190 value=self.password,
191 port=None,
192 port_specified=False,
193 domain='.zeit.de',
194 domain_specified=True,
195 domain_initial_dot=True,
196 path='/',
197 path_specified=True,
198 secure=True,
199 expires=None,
200 discard=True,
201 comment=None,
202 comment_url=None,
203 rest={},
204 )
205 browser.cookiejar.set_cookie(c)
206 browser.open(url)
207 # find page for latest issue
208 latestlink = browser.find_link(text_regex=re.compile(r'.*ZUR AKTUELLEN AUSGABE.*'))
209 browser.follow_link(latestlink)
210 # now find the correct file, we will still use the ePub file
211 epublink = browser.find_link(text_regex=re.compile(r'.*EPUB F.*R E-READER LADEN.*')) # change from '.*EPUB FÜR E-READER LADEN.*' in May 2017
212 response = browser.follow_link(epublink)
213 self.report_progress(1, _('next step'))
214
215 tmp = PersistentTemporaryFile(suffix='.epub')
216 self.report_progress(0, _('downloading epub'))
217 tmp.write(response.read())
218 tmp.close()
219
220 zfile = zipfile.ZipFile(tmp.name, 'r')
221 self.report_progress(0, _('extracting epub'))
222
223 zfile.extractall(self.output_dir)
224
225 tmp.close()
226
227 index = os.path.join(self.output_dir, 'content.opf')
228
229 self.report_progress(1, _('epub downloaded and extracted'))
230
231 # doing regular expression filtering
232 for path in walk('.'):
233 shortname, extension = os.path.splitext(path)
234 if extension.lower() in ('.html', '.htm', '.xhtml'):
235 with open(path, 'r+b') as f:
236 raw = f.read()
237 raw = raw.decode('utf-8')
238 for pat, func in self.preprocess_regexps:
239 raw = pat.sub(func, raw)
240 f.seek(0)
241 f.truncate()
242 f.write(raw.encode('utf-8'))
243
244 # adding real cover
245 self.report_progress(0, _('trying to download cover image (titlepage)'))
246 self.download_cover()
247 self.conversion_options['cover'] = self.cover_path
248
249 return index
250
251 # getting url of the cover
252 def get_cover_url(self):
253 self.log.warning('Downloading cover')
254 try:
255 self.log.warning('Trying PDF-based cover')
256 url = 'https://epaper.zeit.de/abo/diezeit'
257 browser = self.get_browser()
258
259 c = Cookie(
260 version=0,
261 name='zeit_sso_201501',
262 value=self.password,
263 port=None,
264 port_specified=False,
265 domain='.zeit.de',
266 domain_specified=True,
267 domain_initial_dot=True,
268 path='/',
269 path_specified=True,
270 secure=True,
271 expires=None,
272 discard=True,
273 comment=None,
274 comment_url=None,
275 rest={},
276 )
277 browser.cookiejar.set_cookie(c)
278 browser.open(url)
279
280 # change into abo section - not needed at the moment
281 # browser.open(url)
282 # abolink = browser.find_link(text_regex=re.compile(
283 # '.*Abo-Bereich.*'))
284 # browser.follow_link(abolink)
285 # find page for latest issue
286 latestlink = browser.find_link(text_regex=re.compile(r'.*ZUR AKTUELLEN AUSGABE.*'))
287 browser.follow_link(latestlink)
288 # actual cover search
289 pdflink = browser.find_link(text_regex=re.compile(r'.*GESAMT-PDF LADEN.*'))
290 cover_url = (
291 urlparse(pdflink.base_url)[0]
292 + '://'
293 + urlparse(pdflink.base_url)[1]
294 + ''
295 + (urlparse(pdflink.url)[2]).replace('ePaper_', '').replace('.pdf', '_001.pdf')
296 )
297 self.log.warning('PDF link found:')
298 self.log.warning(cover_url)
299 # download the cover (has to be here due to new login process)
300 with closing(browser.open(cover_url)) as r:
301 cdata = r.read()
302 from calibre.ebooks.metadata.pdf import get_metadata
303
304 stream = io.BytesIO(cdata)
305 cdata = None
306 mi = get_metadata(stream)
307 if mi.cover_data and mi.cover_data[1]:
308 cdata = mi.cover_data[1]
309
310 cpath = os.path.join(self.output_dir, 'cover.jpg')
311 save_cover_data_to(cdata, cpath)
312 cover_url = cpath
313
314 except Exception:
315 self.log.warning('Trying low-res cover')
316 try:
317 inhalt = self.index_to_soup('http://www.zeit.de/inhalt')
318 cover_url = inhalt.find('div', attrs={'class': 'singlearchive clearfix'}).img['src'].replace('icon_', '')
319 except Exception:
320 self.log.warning('Using static old low-res cover')
321 cover_url = 'http://images.zeit.de/bilder/titelseiten_zeit/1946/001_001.jpg'
322 return cover_url