| 1 |
import re |
| 2 |
|
| 3 |
from calibre.ebooks.BeautifulSoup import Tag |
| 4 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 5 |
|
| 6 |
|
| 7 |
def new_tag(soup, name, attrs=()): |
| 8 |
impl = getattr(soup, 'new_tag', None) |
| 9 |
if impl is not None: |
| 10 |
return impl(name, attrs=dict(attrs)) |
| 11 |
return Tag(soup, name, attrs=attrs or None) |
| 12 |
|
| 13 |
|
| 14 |
class JoopRecipe(BasicNewsRecipe): |
| 15 |
__license__ = 'GPL v3' |
| 16 |
__author__ = 'kwetal' |
| 17 |
language = 'nl' |
| 18 |
country = 'NL' |
| 19 |
version = 1 |
| 20 |
|
| 21 |
title = 'Joop' |
| 22 |
publisher = 'Vara' |
| 23 |
category = 'News, Politics, Discussion' |
| 24 |
description = 'Political blog from the Netherlands' |
| 25 |
|
| 26 |
oldest_article = 7 |
| 27 |
max_articles_per_feed = 100 |
| 28 |
use_embedded_content = False |
| 29 |
|
| 30 |
no_stylesheets = True |
| 31 |
remove_javascript = True |
| 32 |
|
| 33 |
keep_only_tags = [] |
| 34 |
keep_only_tags.append(dict(name='div', attrs={'class': 'author_head clearfix photo'})) |
| 35 |
keep_only_tags.append(dict(name='h2', attrs={'class': 'columnhead smallline'})) |
| 36 |
keep_only_tags.append(dict(name='div', attrs={'class': re.compile(r'article.*')})) |
| 37 |
|
| 38 |
extra_css = ''' |
| 39 |
body {font-family: verdana, arial, helvetica, geneva, sans-serif;} |
| 40 |
img {margin-right: 0.4em;} |
| 41 |
h3 {font-size: medium; font-style: italic; font-weight: normal;} |
| 42 |
h2 {font-size: xx-large; font-weight: bold} |
| 43 |
sub {color: #666666; font-size: x-small; font-weight: normal;} |
| 44 |
div.joop_byline {font-size: large} |
| 45 |
div.joop_byline_job {font-size: small; color: #696969;} |
| 46 |
div.joop_date {font-size: x-small; font-style: italic; margin-top: 0.6em} |
| 47 |
''' |
| 48 |
|
| 49 |
INDEX = 'http://www.joop.nl' |
| 50 |
|
| 51 |
conversion_options = {'comments': description, 'tags': category, 'language': language, 'publisher': publisher} |
| 52 |
|
| 53 |
def parse_index(self): |
| 54 |
sections = ['Politiek', 'Wereld', 'Economie', 'Groen', 'Media', 'Leven', 'Show', 'Opinies'] |
| 55 |
soup = self.index_to_soup(self.INDEX) |
| 56 |
answer = [] |
| 57 |
|
| 58 |
div = soup.find('div', attrs={'id': 'footer'}) |
| 59 |
for section in sections: |
| 60 |
articles = [] |
| 61 |
h2 = div.find(lambda tag: tag.name == 'h2' and tag.renderContents().decode('utf-8') == section) |
| 62 |
if h2: |
| 63 |
ul = h2.findNextSibling('ul', 'linklist') |
| 64 |
if ul: |
| 65 |
for li in ul.findAll('li'): |
| 66 |
title = self.tag_to_string(li.a) |
| 67 |
url = self.INDEX + li.a['href'] |
| 68 |
articles.append({'title': title, 'date': None, 'url': url, 'description': ''}) |
| 69 |
|
| 70 |
answer.append((section, articles)) |
| 71 |
|
| 72 |
return answer |
| 73 |
|
| 74 |
def preprocess_html(self, soup): |
| 75 |
div = soup.find('div', 'author_head clearfix photo') |
| 76 |
if div: |
| 77 |
h2 = soup.find('h2') |
| 78 |
if h2: |
| 79 |
h2.name = 'div' |
| 80 |
h2['class'] = 'joop_byline' |
| 81 |
span = h2.find('span') |
| 82 |
if span: |
| 83 |
span.name = 'div' |
| 84 |
span['class'] = 'joop_byline_job' |
| 85 |
div.replaceWith(h2) |
| 86 |
|
| 87 |
h2 = soup.find('h2', attrs={'class': 'columnhead smallline'}) |
| 88 |
if h2: |
| 89 |
txt = None |
| 90 |
span = h2.find('span', 'info') |
| 91 |
if span: |
| 92 |
txt = span.find(text=True) |
| 93 |
div = new_tag(soup, 'div', attrs=[('class', 'joop_date')]) |
| 94 |
div.append(txt) |
| 95 |
h2.replaceWith(div) |
| 96 |
|
| 97 |
return soup |