| 1 |
from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag |
| 2 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 3 |
|
| 4 |
|
| 5 |
def new_tag(soup, name, attrs=()): |
| 6 |
impl = getattr(soup, 'new_tag', None) |
| 7 |
if impl is not None: |
| 8 |
return impl(name, attrs=dict(attrs)) |
| 9 |
return Tag(soup, name, attrs=attrs or None) |
| 10 |
|
| 11 |
|
| 12 |
class NrcNextRecipe(BasicNewsRecipe): |
| 13 |
__license__ = 'GPL v3' |
| 14 |
__author__ = 'kwetal' |
| 15 |
language = 'nl' |
| 16 |
country = 'NL' |
| 17 |
version = 2 |
| 18 |
|
| 19 |
title = 'nrcnext' |
| 20 |
publisher = 'NRC Media' |
| 21 |
category = 'News, Opinion, the Netherlands' |
| 22 |
description = 'Dutch newsblog from the Dutch daily newspaper nrcnext.' |
| 23 |
|
| 24 |
conversion_options = {'comments': description, 'language': language, 'publisher': publisher} |
| 25 |
|
| 26 |
no_stylesheets = True |
| 27 |
remove_javascript = True |
| 28 |
|
| 29 |
keep_only_tags = [dict(name='div', attrs={'id': 'main'})] |
| 30 |
|
| 31 |
remove_tags = [] |
| 32 |
remove_tags.append(dict(name='div', attrs={'class': 'meta'})) |
| 33 |
remove_tags.append(dict(name='p', attrs={'class': 'meta'})) |
| 34 |
remove_tags.append(dict(name='div', attrs={'class': 'datumlabel'})) |
| 35 |
remove_tags.append(dict(name='div', attrs={'class': 'sharing-is-caring'})) |
| 36 |
remove_tags.append(dict(name='div', attrs={'class': 'navigation'})) |
| 37 |
remove_tags.append(dict(name='div', attrs={'class': 'reageer'})) |
| 38 |
remove_tags.append(dict(name='div', attrs={'class': 'comment odd alt thread-odd thread-alt depth-1 reactie '})) |
| 39 |
remove_tags.append(dict(name='div', attrs={'class': 'comment even thread-even depth-1 reactie '})) |
| 40 |
remove_tags.append(dict(name='ul', attrs={'class': 'cats single'})) |
| 41 |
remove_tags.append(dict(name='ul', attrs={'class': 'cats onderwerpen'})) |
| 42 |
remove_tags.append(dict(name='ul', attrs={'class': 'cats rubrieken'})) |
| 43 |
remove_tags.append(dict(name='h3', attrs={'class': 'reacties'})) |
| 44 |
|
| 45 |
extra_css = ''' |
| 46 |
body {font-family: verdana, arial, helvetica, geneva, sans-serif; text-align: left;} |
| 47 |
p.wp-caption-text {font-size: x-small; color: #666666;} |
| 48 |
h2.sub_title {font-size: medium; color: #696969;} |
| 49 |
h2.vlag {font-size: small; font-weight: bold;} |
| 50 |
''' |
| 51 |
|
| 52 |
def parse_index(self): |
| 53 |
# Use the website as an index. Their RSS feeds can be out of date. |
| 54 |
feeds = {} |
| 55 |
feeds['columnisten'] = 'http://www.nrcnext.nl/columnisten/' |
| 56 |
feeds['koken'] = 'http://www.nrcnext.nl/koken/' |
| 57 |
feeds['geld & werk'] = 'http://www.nrcnext.nl/geld-en-werk/' |
| 58 |
feeds['vandaag'] = 'http://www.nrcnext.nl' |
| 59 |
# feeds[u'city life in afrika'] = u'http://www.nrcnext.nl/city-life-in-afrika/' |
| 60 |
answer = [] |
| 61 |
articles = {} |
| 62 |
indices = [] |
| 63 |
|
| 64 |
for index, feed in feeds.items(): |
| 65 |
soup = self.index_to_soup(feed) |
| 66 |
for post in soup.findAll(True, attrs={'class': 'post '}): |
| 67 |
# Find the links to the actual articles and remember the location |
| 68 |
# they're pointing to and the title |
| 69 |
a = post.find('a', attrs={'rel': 'bookmark'}) |
| 70 |
href = a['href'] |
| 71 |
title = self.tag_to_string(a) |
| 72 |
if index == 'columnisten': |
| 73 |
# In this feed/page articles can be written by more than one author. |
| 74 |
# It is nice to see their names in the titles. |
| 75 |
flag = post.find('h2', attrs={'class': 'vlag'}) |
| 76 |
author = flag.contents[0].renderContents().decode('utf-8') |
| 77 |
completeTitle = ''.join([author, ': ', title]) |
| 78 |
else: |
| 79 |
completeTitle = title |
| 80 |
|
| 81 |
# Add the article to a temporary list |
| 82 |
article = {'title': completeTitle, 'date': '', 'url': href, 'description': '<p> </p>'} |
| 83 |
if index not in articles: |
| 84 |
articles[index] = [] |
| 85 |
articles[index].append(article) |
| 86 |
|
| 87 |
# Add the index title to a temporary list |
| 88 |
indices.append(index) |
| 89 |
|
| 90 |
# Now, sort the temporary list of feeds in the order they appear on the website |
| 91 |
# indices = self.sort_index_by(indices, {u'columnisten' : 1, u'koken' : 3, u'geld & werk' : 2, u'vandaag' : 0, u'city life in afrika' : 4}) |
| 92 |
indices = self.sort_index_by(indices, {'columnisten': 1, 'koken': 3, 'geld & werk': 2, 'vandaag': 0}) |
| 93 |
# Apply this sort order to the actual list of feeds and articles |
| 94 |
answer = [(key, articles[key]) for key in indices if key in articles] |
| 95 |
|
| 96 |
return answer |
| 97 |
|
| 98 |
def preprocess_html(self, soup): |
| 99 |
if soup.find('div', attrs={'id': 'main', 'class': 'single'}): |
| 100 |
tag = soup.find('div', attrs={'class': 'post'}) |
| 101 |
if tag: |
| 102 |
h2 = tag.find('h2', 'vlag') |
| 103 |
if h2: |
| 104 |
new_h2 = new_tag(soup, 'h2', attrs=[('class', 'vlag')]) |
| 105 |
new_h2.append(self.tag_to_string(h2)) |
| 106 |
h2.replaceWith(new_h2) |
| 107 |
else: |
| 108 |
h2 = tag.find('h2') |
| 109 |
if h2: |
| 110 |
new_h2 = new_tag(soup, 'h2', attrs=[('class', 'sub_title')]) |
| 111 |
new_h2.append(self.tag_to_string(h2)) |
| 112 |
h2.replaceWith(new_h2) |
| 113 |
|
| 114 |
h1 = tag.find('h1') |
| 115 |
if h1: |
| 116 |
new_h1 = new_tag(soup, 'h1') |
| 117 |
new_h1.append(self.tag_to_string(h1)) |
| 118 |
h1.replaceWith(new_h1) |
| 119 |
|
| 120 |
# Slows down my reader. |
| 121 |
for movie in tag.findAll('span', attrs={'class': 'vvqbox vvqvimeo'}): |
| 122 |
movie.extract() |
| 123 |
for movie in tag.findAll('span', attrs={'class': 'vvqbox vvqyoutube'}): |
| 124 |
movie.extract() |
| 125 |
for iframe in tag.findAll('iframe'): |
| 126 |
iframe.extract() |
| 127 |
|
| 128 |
fresh_soup = self.getFreshSoup(soup) |
| 129 |
fresh_soup.body.append(tag) |
| 130 |
|
| 131 |
return fresh_soup |
| 132 |
else: |
| 133 |
# This should never happen and other famous last words... |
| 134 |
return soup |
| 135 |
|
| 136 |
def getFreshSoup(self, oldSoup): |
| 137 |
freshSoup = BeautifulSoup('<html><head><title></title></head><body></body></html>') |
| 138 |
if oldSoup.head.title: |
| 139 |
freshSoup.head.title.append(self.tag_to_string(oldSoup.head.title)) |
| 140 |
return freshSoup |