| 1 |
#!/usr/bin/env python |
| 2 |
import json |
| 3 |
import uuid |
| 4 |
from contextlib import closing |
| 5 |
|
| 6 |
from mechanize import Request |
| 7 |
|
| 8 |
from calibre.web.feeds.recipes import BasicNewsRecipe |
| 9 |
|
| 10 |
|
| 11 |
class Parool(BasicNewsRecipe): |
| 12 |
title = 'Het Parool' |
| 13 |
__author__ = 'Cristi Ghera' |
| 14 |
max_articles_per_feed = 100 |
| 15 |
description = 'Het Parool - Vrij, Onverveerd' |
| 16 |
needs_subscription = False |
| 17 |
language = 'nl' |
| 18 |
country = 'NL' |
| 19 |
category = 'news, politics, Netherlands' |
| 20 |
resolve_internal_links = True |
| 21 |
remove_tags_before = dict(id='main-content') |
| 22 |
remove_tags_after = dict(id='main-content') |
| 23 |
remove_tags = [ |
| 24 |
dict( |
| 25 |
attrs={ |
| 26 |
'class': [ |
| 27 |
'article-footer__sharing', |
| 28 |
'artstyle__editorial-tips', |
| 29 |
'artstyle__advertisement', |
| 30 |
'artstyle__container__icon', |
| 31 |
'artstyle__disabled-embed', |
| 32 |
'container__title__icon', |
| 33 |
] |
| 34 |
} |
| 35 |
), |
| 36 |
dict(attrs={'data-element-id': ['article-element-authors']}), |
| 37 |
dict(name=['script', 'noscript', 'style']), |
| 38 |
] |
| 39 |
remove_attributes = ['class', 'id', 'name', 'style'] |
| 40 |
encoding = 'utf-8' |
| 41 |
no_stylesheets = True |
| 42 |
ignore_duplicate_articles = {'url'} |
| 43 |
|
| 44 |
def parse_index(self): |
| 45 |
soup = self.index_to_soup('https://www.parool.nl/privacy-wall/accept?redirectUri=%2Feditie%2Fvandaag%2F&authId=' + str(uuid.uuid4())) |
| 46 |
containers = soup.findAll('section', attrs={'class': 'section--horizontal'}) |
| 47 |
sections = [] |
| 48 |
for container in containers: |
| 49 |
section_title = self.tag_to_string(container.find('h2')).strip() |
| 50 |
articles = [] |
| 51 |
|
| 52 |
for art in container.findAll('article'): |
| 53 |
a = art.find('a') |
| 54 |
url = a['href'] |
| 55 |
if url[0] == '/': |
| 56 |
url = 'https://www.parool.nl' + url |
| 57 |
if '/editie/' not in url: |
| 58 |
continue |
| 59 |
header = a.find('header') |
| 60 |
teaser_label = self.tag_to_string(header.find('h4').find('span', attrs={'class': 'teaser__label'})).strip() |
| 61 |
teaser_sublabel = self.tag_to_string(header.find('h4').find('span', attrs={'class': 'teaser__sublabel'})).strip() |
| 62 |
teaser_title = self.tag_to_string(header.find('h3').find('span', attrs={'class': 'teaser__title__value--short'})).strip() |
| 63 |
ignore = {'dirkjan', 's1ngle', 'pukkels', 'hein de kort'} |
| 64 |
if teaser_label.lower() in ignore: |
| 65 |
continue |
| 66 |
parts = [] |
| 67 |
if teaser_label: |
| 68 |
parts.append(teaser_label.upper()) |
| 69 |
if teaser_sublabel: |
| 70 |
parts.append(teaser_sublabel) |
| 71 |
if teaser_title: |
| 72 |
parts.append(teaser_title) |
| 73 |
article_title = ' \u2022 '.join(parts) |
| 74 |
articles.append(dict(title=article_title, url=url, content='')) |
| 75 |
|
| 76 |
sections.append((section_title, articles)) |
| 77 |
return sections |
| 78 |
|
| 79 |
def preprocess_html(self, soup): |
| 80 |
for tag in soup(): |
| 81 |
if tag.name == 'img': |
| 82 |
if tag['src'][0] == '/': |
| 83 |
tag['src'] = 'https://www.parool.nl' + tag['src'] |
| 84 |
for tag in soup(): |
| 85 |
if tag.name == 'picture': |
| 86 |
tag.replaceWith(tag.find('img')) |
| 87 |
comic_articles = { |
| 88 |
'Alle strips van Dirkjan', |
| 89 |
'S1NGLE', |
| 90 |
'Pukkels', |
| 91 |
'Bekijk hier alle cartoons van Hein de Kort', |
| 92 |
} |
| 93 |
if self.tag_to_string(soup.find('h1')).strip() in comic_articles: |
| 94 |
for node in soup.find('figure').find_next_siblings(): |
| 95 |
node.extract() |
| 96 |
return soup |
| 97 |
|
| 98 |
def get_cover_url(self): |
| 99 |
headers = { |
| 100 |
'X-Requested-With': 'XMLHttpRequest', |
| 101 |
'Accept': 'application/json, text/javascript, */*; q=0.01', |
| 102 |
'DNT': '1', |
| 103 |
} |
| 104 |
url = 'https://login-api.e-pages.dk/v1/krant.parool.nl/folders' |
| 105 |
with closing(self.browser.open(Request(url, None, headers))) as r: |
| 106 |
folders = json.loads(r.read()) |
| 107 |
return folders['objects'][0]['teaser_medium'] |
| 108 |
return None |