| 1 |
#!/usr/bin/env python |
| 2 |
import datetime |
| 3 |
import json |
| 4 |
import re |
| 5 |
from contextlib import closing |
| 6 |
|
| 7 |
from mechanize import Request |
| 8 |
|
| 9 |
from calibre.web.feeds.recipes import BasicNewsRecipe |
| 10 |
|
| 11 |
|
| 12 |
class NRC(BasicNewsRecipe): |
| 13 |
title = 'NRC' |
| 14 |
__author__ = 'Cristi Ghera' |
| 15 |
max_articles_per_feed = 100 |
| 16 |
description = 'NRC - Nieuws, achtergronden en onderzoeksjournalistiek' |
| 17 |
needs_subscription = False |
| 18 |
language = 'nl' |
| 19 |
country = 'NL' |
| 20 |
category = 'news, politics, Netherlands' |
| 21 |
resolve_internal_links = True |
| 22 |
remove_tags_before = {'class': 'article__header-and-content'} |
| 23 |
remove_tags_after = {'class': 'article__header-and-content'} |
| 24 |
remove_tags = [ |
| 25 |
dict( |
| 26 |
attrs={ |
| 27 |
'class': [ |
| 28 |
'article__footer', |
| 29 |
'lees-ook', |
| 30 |
'luister-naar', |
| 31 |
'print-layout-warning', |
| 32 |
'newslettersignup', |
| 33 |
'article__byline', |
| 34 |
'article__published-in', |
| 35 |
'article__featured-image__caption__producer', |
| 36 |
'metabox', |
| 37 |
] |
| 38 |
} |
| 39 |
), |
| 40 |
dict(name=['script', 'noscript', 'style']), |
| 41 |
] |
| 42 |
remove_attributes = ['class', 'id', 'name', 'style'] |
| 43 |
encoding = 'utf-8' |
| 44 |
no_stylesheets = True |
| 45 |
ignore_duplicate_articles = {'url'} |
| 46 |
delay = 0.3 |
| 47 |
|
| 48 |
touchscreen = True |
| 49 |
|
| 50 |
frontpage = None |
| 51 |
|
| 52 |
title_regexp = None |
| 53 |
|
| 54 |
@staticmethod |
| 55 |
def _monthly_list_url(date, fmt='%Y/%m/'): |
| 56 |
return 'https://www.nrc.nl/de/data/NH/' + date.strftime(fmt) |
| 57 |
|
| 58 |
def _clean_article_title(self, title): |
| 59 |
if not title: |
| 60 |
return title |
| 61 |
if self.title_regexp is None: |
| 62 |
self.title_regexp = re.compile(r'<span class="keyword">([^<]+)</span>\s*') |
| 63 |
return self.title_regexp.sub(r'\1 ', title) |
| 64 |
|
| 65 |
def parse_index(self): |
| 66 |
sections = [] |
| 67 |
today = datetime.date.today() |
| 68 |
headers = { |
| 69 |
'X-Requested-With': 'XMLHttpRequest', |
| 70 |
'Accept': 'application/json, text/javascript, */*; q=0.01', |
| 71 |
'DNT': '1', |
| 72 |
} |
| 73 |
monthly_list_urls = [self._monthly_list_url(today), self._monthly_list_url(datetime.date(today.year, today.month, 1) - datetime.timedelta(days=1))] |
| 74 |
issue_url = None |
| 75 |
issue_date = None |
| 76 |
for monthly_list_url in monthly_list_urls: |
| 77 |
with closing(self.browser.open(Request(monthly_list_url, None, headers))) as r: |
| 78 |
issues = json.loads(r.read()) |
| 79 |
if len(issues) > 0: |
| 80 |
issue_date = datetime.datetime.strptime(issues[0]['published_at'], '%Y-%m-%dT%H:%M:%SZ') |
| 81 |
issue_url = self._monthly_list_url(issue_date, '%Y/%m/%d/') |
| 82 |
self.frontpage = issues[0]['frontpage'] |
| 83 |
break |
| 84 |
if issue_url is None: |
| 85 |
return [] |
| 86 |
with closing(self.browser.open(Request(issue_url, None, headers))) as r: |
| 87 |
edition = json.loads(r.read()) |
| 88 |
documents = {} |
| 89 |
for headline in edition['paperheadlines']: |
| 90 |
item = headline['item'] |
| 91 |
documents[headline['document_id']] = dict(url=item['full_url'], headline=self._clean_article_title(item['headline'])) |
| 92 |
for section in edition['sections']: |
| 93 |
articles = [] |
| 94 |
for doc in section['document_ids']: |
| 95 |
if doc not in documents: |
| 96 |
self.log.warn('Document not found:', doc) |
| 97 |
continue |
| 98 |
articles.append(dict(title=documents[doc]['headline'], url=documents[doc]['url'])) |
| 99 |
sections.append((section['name'], articles)) |
| 100 |
return sections |
| 101 |
|
| 102 |
def preprocess_html(self, soup): |
| 103 |
for tag in soup(): |
| 104 |
if tag.name == 'img': |
| 105 |
if tag.has_attr('data-src-medium'): |
| 106 |
tag['src'] = tag['data-src-medium'].split('|')[0] |
| 107 |
elif tag.has_attr('data-src'): |
| 108 |
tag['src'] = tag['data-src'].split('|')[0] |
| 109 |
if tag['src'].startswith('//'): |
| 110 |
tag['src'] = 'https:' + tag['src'] |
| 111 |
elif tag['src'].startswith('/'): |
| 112 |
tag['src'] = 'https://www.nrc.nl' + tag['src'] |
| 113 |
if self.browser.cookiejar: |
| 114 |
self.browser.cookiejar.clear() |
| 115 |
return soup |
| 116 |
|
| 117 |
def get_cover_url(self): |
| 118 |
return self.frontpage |