Debian Codegraph @

recipes/parool.recipe

raw
1 #!/usr/bin/env python
2 import json
3 import uuid
4 from contextlib import closing
5
6 from mechanize import Request
7
8 from calibre.web.feeds.recipes import BasicNewsRecipe
9
10
11 class Parool(BasicNewsRecipe):
12 title = 'Het Parool'
13 __author__ = 'Cristi Ghera'
14 max_articles_per_feed = 100
15 description = 'Het Parool - Vrij, Onverveerd'
16 needs_subscription = False
17 language = 'nl'
18 country = 'NL'
19 category = 'news, politics, Netherlands'
20 resolve_internal_links = True
21 remove_tags_before = dict(id='main-content')
22 remove_tags_after = dict(id='main-content')
23 remove_tags = [
24 dict(
25 attrs={
26 'class': [
27 'article-footer__sharing',
28 'artstyle__editorial-tips',
29 'artstyle__advertisement',
30 'artstyle__container__icon',
31 'artstyle__disabled-embed',
32 'container__title__icon',
33 ]
34 }
35 ),
36 dict(attrs={'data-element-id': ['article-element-authors']}),
37 dict(name=['script', 'noscript', 'style']),
38 ]
39 remove_attributes = ['class', 'id', 'name', 'style']
40 encoding = 'utf-8'
41 no_stylesheets = True
42 ignore_duplicate_articles = {'url'}
43
44 def parse_index(self):
45 soup = self.index_to_soup('https://www.parool.nl/privacy-wall/accept?redirectUri=%2Feditie%2Fvandaag%2F&authId=' + str(uuid.uuid4()))
46 containers = soup.findAll('section', attrs={'class': 'section--horizontal'})
47 sections = []
48 for container in containers:
49 section_title = self.tag_to_string(container.find('h2')).strip()
50 articles = []
51
52 for art in container.findAll('article'):
53 a = art.find('a')
54 url = a['href']
55 if url[0] == '/':
56 url = 'https://www.parool.nl' + url
57 if '/editie/' not in url:
58 continue
59 header = a.find('header')
60 teaser_label = self.tag_to_string(header.find('h4').find('span', attrs={'class': 'teaser__label'})).strip()
61 teaser_sublabel = self.tag_to_string(header.find('h4').find('span', attrs={'class': 'teaser__sublabel'})).strip()
62 teaser_title = self.tag_to_string(header.find('h3').find('span', attrs={'class': 'teaser__title__value--short'})).strip()
63 ignore = {'dirkjan', 's1ngle', 'pukkels', 'hein de kort'}
64 if teaser_label.lower() in ignore:
65 continue
66 parts = []
67 if teaser_label:
68 parts.append(teaser_label.upper())
69 if teaser_sublabel:
70 parts.append(teaser_sublabel)
71 if teaser_title:
72 parts.append(teaser_title)
73 article_title = ' \u2022 '.join(parts)
74 articles.append(dict(title=article_title, url=url, content=''))
75
76 sections.append((section_title, articles))
77 return sections
78
79 def preprocess_html(self, soup):
80 for tag in soup():
81 if tag.name == 'img':
82 if tag['src'][0] == '/':
83 tag['src'] = 'https://www.parool.nl' + tag['src']
84 for tag in soup():
85 if tag.name == 'picture':
86 tag.replaceWith(tag.find('img'))
87 comic_articles = {
88 'Alle strips van Dirkjan',
89 'S1NGLE',
90 'Pukkels',
91 'Bekijk hier alle cartoons van Hein de Kort',
92 }
93 if self.tag_to_string(soup.find('h1')).strip() in comic_articles:
94 for node in soup.find('figure').find_next_siblings():
95 node.extract()
96 return soup
97
98 def get_cover_url(self):
99 headers = {
100 'X-Requested-With': 'XMLHttpRequest',
101 'Accept': 'application/json, text/javascript, */*; q=0.01',
102 'DNT': '1',
103 }
104 url = 'https://login-api.e-pages.dk/v1/krant.parool.nl/folders'
105 with closing(self.browser.open(Request(url, None, headers))) as r:
106 folders = json.loads(r.read())
107 return folders['objects'][0]['teaser_medium']
108 return None