Debian Codegraph @

recipes/ncrnext.recipe

raw
1 from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag
2 from calibre.web.feeds.news import BasicNewsRecipe
3
4
5 def new_tag(soup, name, attrs=()):
6 impl = getattr(soup, 'new_tag', None)
7 if impl is not None:
8 return impl(name, attrs=dict(attrs))
9 return Tag(soup, name, attrs=attrs or None)
10
11
12 class NrcNextRecipe(BasicNewsRecipe):
13 __license__ = 'GPL v3'
14 __author__ = 'kwetal'
15 language = 'nl'
16 country = 'NL'
17 version = 2
18
19 title = 'nrcnext'
20 publisher = 'NRC Media'
21 category = 'News, Opinion, the Netherlands'
22 description = 'Dutch newsblog from the Dutch daily newspaper nrcnext.'
23
24 conversion_options = {'comments': description, 'language': language, 'publisher': publisher}
25
26 no_stylesheets = True
27 remove_javascript = True
28
29 keep_only_tags = [dict(name='div', attrs={'id': 'main'})]
30
31 remove_tags = []
32 remove_tags.append(dict(name='div', attrs={'class': 'meta'}))
33 remove_tags.append(dict(name='p', attrs={'class': 'meta'}))
34 remove_tags.append(dict(name='div', attrs={'class': 'datumlabel'}))
35 remove_tags.append(dict(name='div', attrs={'class': 'sharing-is-caring'}))
36 remove_tags.append(dict(name='div', attrs={'class': 'navigation'}))
37 remove_tags.append(dict(name='div', attrs={'class': 'reageer'}))
38 remove_tags.append(dict(name='div', attrs={'class': 'comment odd alt thread-odd thread-alt depth-1 reactie '}))
39 remove_tags.append(dict(name='div', attrs={'class': 'comment even thread-even depth-1 reactie '}))
40 remove_tags.append(dict(name='ul', attrs={'class': 'cats single'}))
41 remove_tags.append(dict(name='ul', attrs={'class': 'cats onderwerpen'}))
42 remove_tags.append(dict(name='ul', attrs={'class': 'cats rubrieken'}))
43 remove_tags.append(dict(name='h3', attrs={'class': 'reacties'}))
44
45 extra_css = '''
46 body {font-family: verdana, arial, helvetica, geneva, sans-serif; text-align: left;}
47 p.wp-caption-text {font-size: x-small; color: #666666;}
48 h2.sub_title {font-size: medium; color: #696969;}
49 h2.vlag {font-size: small; font-weight: bold;}
50 '''
51
52 def parse_index(self):
53 # Use the website as an index. Their RSS feeds can be out of date.
54 feeds = {}
55 feeds['columnisten'] = 'http://www.nrcnext.nl/columnisten/'
56 feeds['koken'] = 'http://www.nrcnext.nl/koken/'
57 feeds['geld & werk'] = 'http://www.nrcnext.nl/geld-en-werk/'
58 feeds['vandaag'] = 'http://www.nrcnext.nl'
59 # feeds[u'city life in afrika'] = u'http://www.nrcnext.nl/city-life-in-afrika/'
60 answer = []
61 articles = {}
62 indices = []
63
64 for index, feed in feeds.items():
65 soup = self.index_to_soup(feed)
66 for post in soup.findAll(True, attrs={'class': 'post '}):
67 # Find the links to the actual articles and remember the location
68 # they're pointing to and the title
69 a = post.find('a', attrs={'rel': 'bookmark'})
70 href = a['href']
71 title = self.tag_to_string(a)
72 if index == 'columnisten':
73 # In this feed/page articles can be written by more than one author.
74 # It is nice to see their names in the titles.
75 flag = post.find('h2', attrs={'class': 'vlag'})
76 author = flag.contents[0].renderContents().decode('utf-8')
77 completeTitle = ''.join([author, ': ', title])
78 else:
79 completeTitle = title
80
81 # Add the article to a temporary list
82 article = {'title': completeTitle, 'date': '', 'url': href, 'description': '<p>&nbsp;</p>'}
83 if index not in articles:
84 articles[index] = []
85 articles[index].append(article)
86
87 # Add the index title to a temporary list
88 indices.append(index)
89
90 # Now, sort the temporary list of feeds in the order they appear on the website
91 # indices = self.sort_index_by(indices, {u'columnisten' : 1, u'koken' : 3, u'geld & werk' : 2, u'vandaag' : 0, u'city life in afrika' : 4})
92 indices = self.sort_index_by(indices, {'columnisten': 1, 'koken': 3, 'geld & werk': 2, 'vandaag': 0})
93 # Apply this sort order to the actual list of feeds and articles
94 answer = [(key, articles[key]) for key in indices if key in articles]
95
96 return answer
97
98 def preprocess_html(self, soup):
99 if soup.find('div', attrs={'id': 'main', 'class': 'single'}):
100 tag = soup.find('div', attrs={'class': 'post'})
101 if tag:
102 h2 = tag.find('h2', 'vlag')
103 if h2:
104 new_h2 = new_tag(soup, 'h2', attrs=[('class', 'vlag')])
105 new_h2.append(self.tag_to_string(h2))
106 h2.replaceWith(new_h2)
107 else:
108 h2 = tag.find('h2')
109 if h2:
110 new_h2 = new_tag(soup, 'h2', attrs=[('class', 'sub_title')])
111 new_h2.append(self.tag_to_string(h2))
112 h2.replaceWith(new_h2)
113
114 h1 = tag.find('h1')
115 if h1:
116 new_h1 = new_tag(soup, 'h1')
117 new_h1.append(self.tag_to_string(h1))
118 h1.replaceWith(new_h1)
119
120 # Slows down my reader.
121 for movie in tag.findAll('span', attrs={'class': 'vvqbox vvqvimeo'}):
122 movie.extract()
123 for movie in tag.findAll('span', attrs={'class': 'vvqbox vvqyoutube'}):
124 movie.extract()
125 for iframe in tag.findAll('iframe'):
126 iframe.extract()
127
128 fresh_soup = self.getFreshSoup(soup)
129 fresh_soup.body.append(tag)
130
131 return fresh_soup
132 else:
133 # This should never happen and other famous last words...
134 return soup
135
136 def getFreshSoup(self, oldSoup):
137 freshSoup = BeautifulSoup('<html><head><title></title></head><body></body></html>')
138 if oldSoup.head.title:
139 freshSoup.head.title.append(self.tag_to_string(oldSoup.head.title))
140 return freshSoup