Debian Codegraph @

recipes/nzz_ger.recipe

raw
1 import json
2 from datetime import datetime
3
4 from mechanize import Request
5
6 from calibre.web.feeds.recipes import BasicNewsRecipe
7
8
9 class Nzz(BasicNewsRecipe):
10 title = 'NZZ'
11 __author__ = 'Claude Henchoz'
12 description = 'Neue Zürcher Zeitung'
13 publisher = 'Neue Zürcher Zeitung'
14 category = 'news, politics'
15 oldest_article = 7
16 max_articles_per_feed = 15
17 language = 'de'
18 no_stylesheets = True
19 use_embedded_content = False
20 encoding = 'utf-8'
21
22 scale_news_images = (600, 400)
23 scale_news_images_to_device = True
24 compress_news_images = True
25 compress_news_images_max_size = 50
26
27 masthead_url = 'https://upload.wikimedia.org/wikipedia/commons/thumb/3/37/Neue_Z%C3%BCrcher_Zeitung.svg/800px-Neue_Z%C3%BCrcher_Zeitung.svg.png'
28
29 keep_only_tags = [dict(name='section', attrs={'class': 'container--article'})]
30
31 remove_tags = [
32 dict(name='div', attrs={'class': 'progressbar__wrapper'}), # Reading progress.
33 dict(name='div', attrs={'class': 'headline__meta'}), # Article meta data.
34 dict(name='div', attrs={'class': 'items-start'}), # Article meta data.
35 dict(name='span', attrs={'class': 'image-description__author-single'}), # Photo accreditation.
36 dict(name='span', attrs={'data-nzz-tid': 'image-description-author'}), # Photo accreditation.
37 dict(name='div', attrs={'class': 'disabled-overlay'}), # "Please enable Javascript".
38 dict(name='section', attrs={'id': 'content-table'}), # Table of contents.
39 dict(name='div', attrs={'componenttype': 'articlelist'}), # Article list inside the article.
40 dict(name='svg'), # Checkmark images.
41 dict(name='div', attrs={'componenttype': 'advertisement'}), # Ads.
42 dict(name='div', attrs={'componenttype': 'alias-teaser-embed'}), # Embedded teasers.
43 ]
44
45 remove_tags_after = [
46 dict(name='p', attrs={'data-vars-danzz-last-article-element': 'true', 'class': 'articlecomponent'}),
47 dict(name='div', attrs={'componenttype': 'sharebox'}), # Remove everything after the social media share box.
48 ]
49
50 # Center and reduce the size of images and image captions.
51 extra_css = '''
52 div.calibre-nuked-tag-figure { font-size: small; text-align: center; }
53 img { display: block; margin: auto; width: 75%; height: auto; }
54 '''
55
56 remove_attributes = ['style', 'font', 'class']
57
58 feeds = [
59 ('Topthemen der Startseite', 'https://www.nzz.ch/startseite.rss'),
60 ('International', 'https://www.nzz.ch/international.rss'),
61 ('Schweiz', 'https://www.nzz.ch/schweiz.rss'),
62 ('Wirtschaft', 'https://www.nzz.ch/wirtschaft.rss'),
63 ('Finanznachrichten', 'https://www.nzz.ch/finanzen.rss'),
64 ('Kultur', 'https://www.nzz.ch/feuilleton.rss'),
65 ('Sport', 'https://www.nzz.ch/sport.rss'),
66 ('Zürich', 'https://www.nzz.ch/zuerich.rss'),
67 ('Panorama', 'https://www.nzz.ch/panorama.rss'),
68 ('Wissenschaft', 'https://www.nzz.ch/wissenschaft.rss'),
69 ('Auto', 'https://www.nzz.ch/mobilitaet/auto-mobil.rss'),
70 ('Technologie', 'https://www.nzz.ch/technologie.rss'),
71 ]
72
73 def get_cover_url(self):
74 # Prepare the date and data
75 today_date = datetime.now().strftime('%Y-%m-%d')
76 json_data = {
77 'editions': [
78 {
79 'publicationDate': today_date,
80 'defId': 6,
81 },
82 ],
83 'startDate': today_date,
84 'maxHits': 1,
85 'direction': 'BACKWARD',
86 }
87
88 # Prepare headers
89 headers = {
90 'Accept': 'application/json',
91 'Accept-Language': 'en-US,en;q=0.9,de;q=0.8',
92 'Content-Type': 'application/json',
93 'Origin': 'https://epaper.nzz.ch',
94 'Referer': 'https://epaper.nzz.ch/',
95 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 Edg/122.0.0.0',
96 'X-Requested-With': 'XMLHttpRequest',
97 }
98
99 # Encode your JSON data
100 encoded_data = json.dumps(json_data).encode('utf-8')
101
102 # Create a mechanize Request object with the target URL, encoded data, and headers
103 req = Request(url='https://epaper.nzz.ch/epaper/1.0/findEditionsFromDate', data=encoded_data, headers=headers, method='POST')
104
105 # Use mechanize to open the request and read the response
106 browser = self.get_browser()
107 response = browser.open(req)
108 response_data = json.loads(response.read())
109
110 # Extract the desired information
111 url = response_data['data'][0]['pages'][0]['pageDocUrl']['PREVIEW']['url']
112
113 return url
114
115 def get_browser(self, *args, **kwargs):
116 kwargs['user_agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36'
117 br = BasicNewsRecipe.get_browser(self, *args, **kwargs)
118 br.addheaders += [('Referer', 'https://www.google.com/'), ('X-Forwarded-For', '66.249.66.1')]
119 return br
120
121 def preprocess_html(self, soup):
122 # Delete images whose parent <div> is of class '.rounded-full'.
123 for img_tag in soup.findAll('img'):
124 if img_tag.find_parent('div', class_='rounded-full'):
125 self.log.debug('Removing suspected commentary profile pic from inside <div class="rounded-full">.')
126 img_tag.decompose()
127
128 # Fix lazy-loading images.
129 for img in soup.findAll('img', attrs={'srcset': True}):
130 img['src'] = img['srcset'].split()[0]
131
132 # To prevent image captions from being displayed as headers in the output, convert them from <h2> to <p>.
133 for caption in soup.findAll('h2', attrs={'data-nzz-tid': 'image-description-caption'}):
134 caption.name = 'p'
135
136 # Remove the social media share box, which should delimit the end of the article.
137 sharebox_div = soup.find('div', attrs={'componenttype': 'sharebox'})
138 self.log.debug("Searching for sharebox <div> with attrs {'componenttype': 'sharebox'}.")
139 if sharebox_div:
140 self.log.debug('Found sharebox <div> that will now be removed.')
141 sharebox_div.decompose()
142
143 return soup