Debian Codegraph @

recipes/nrc.nl.recipe

raw
1 #!/usr/bin/env python
2 import datetime
3 import json
4 import re
5 from contextlib import closing
6
7 from mechanize import Request
8
9 from calibre.web.feeds.recipes import BasicNewsRecipe
10
11
12 class NRC(BasicNewsRecipe):
13 title = 'NRC'
14 __author__ = 'Cristi Ghera'
15 max_articles_per_feed = 100
16 description = 'NRC - Nieuws, achtergronden en onderzoeksjournalistiek'
17 needs_subscription = False
18 language = 'nl'
19 country = 'NL'
20 category = 'news, politics, Netherlands'
21 resolve_internal_links = True
22 remove_tags_before = {'class': 'article__header-and-content'}
23 remove_tags_after = {'class': 'article__header-and-content'}
24 remove_tags = [
25 dict(
26 attrs={
27 'class': [
28 'article__footer',
29 'lees-ook',
30 'luister-naar',
31 'print-layout-warning',
32 'newslettersignup',
33 'article__byline',
34 'article__published-in',
35 'article__featured-image__caption__producer',
36 'metabox',
37 ]
38 }
39 ),
40 dict(name=['script', 'noscript', 'style']),
41 ]
42 remove_attributes = ['class', 'id', 'name', 'style']
43 encoding = 'utf-8'
44 no_stylesheets = True
45 ignore_duplicate_articles = {'url'}
46 delay = 0.3
47
48 touchscreen = True
49
50 frontpage = None
51
52 title_regexp = None
53
54 @staticmethod
55 def _monthly_list_url(date, fmt='%Y/%m/'):
56 return 'https://www.nrc.nl/de/data/NH/' + date.strftime(fmt)
57
58 def _clean_article_title(self, title):
59 if not title:
60 return title
61 if self.title_regexp is None:
62 self.title_regexp = re.compile(r'<span class="keyword">([^<]+)</span>\s*')
63 return self.title_regexp.sub(r'\1 ', title)
64
65 def parse_index(self):
66 sections = []
67 today = datetime.date.today()
68 headers = {
69 'X-Requested-With': 'XMLHttpRequest',
70 'Accept': 'application/json, text/javascript, */*; q=0.01',
71 'DNT': '1',
72 }
73 monthly_list_urls = [self._monthly_list_url(today), self._monthly_list_url(datetime.date(today.year, today.month, 1) - datetime.timedelta(days=1))]
74 issue_url = None
75 issue_date = None
76 for monthly_list_url in monthly_list_urls:
77 with closing(self.browser.open(Request(monthly_list_url, None, headers))) as r:
78 issues = json.loads(r.read())
79 if len(issues) > 0:
80 issue_date = datetime.datetime.strptime(issues[0]['published_at'], '%Y-%m-%dT%H:%M:%SZ')
81 issue_url = self._monthly_list_url(issue_date, '%Y/%m/%d/')
82 self.frontpage = issues[0]['frontpage']
83 break
84 if issue_url is None:
85 return []
86 with closing(self.browser.open(Request(issue_url, None, headers))) as r:
87 edition = json.loads(r.read())
88 documents = {}
89 for headline in edition['paperheadlines']:
90 item = headline['item']
91 documents[headline['document_id']] = dict(url=item['full_url'], headline=self._clean_article_title(item['headline']))
92 for section in edition['sections']:
93 articles = []
94 for doc in section['document_ids']:
95 if doc not in documents:
96 self.log.warn('Document not found:', doc)
97 continue
98 articles.append(dict(title=documents[doc]['headline'], url=documents[doc]['url']))
99 sections.append((section['name'], articles))
100 return sections
101
102 def preprocess_html(self, soup):
103 for tag in soup():
104 if tag.name == 'img':
105 if tag.has_attr('data-src-medium'):
106 tag['src'] = tag['data-src-medium'].split('|')[0]
107 elif tag.has_attr('data-src'):
108 tag['src'] = tag['data-src'].split('|')[0]
109 if tag['src'].startswith('//'):
110 tag['src'] = 'https:' + tag['src']
111 elif tag['src'].startswith('/'):
112 tag['src'] = 'https://www.nrc.nl' + tag['src']
113 if self.browser.cookiejar:
114 self.browser.cookiejar.clear()
115 return soup
116
117 def get_cover_url(self):
118 return self.frontpage