| 1 |
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai |
| 2 |
|
| 3 |
import re |
| 4 |
|
| 5 |
from calibre.ebooks.BeautifulSoup import Comment |
| 6 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 7 |
|
| 8 |
|
| 9 |
class Dziennik_pl(BasicNewsRecipe): |
| 10 |
title = 'Dziennik.pl' |
| 11 |
__author__ = 'fenuks' |
| 12 |
description = 'Wiadomości z kraju i ze świata. Wiadomości gospodarcze. Znajdziesz u nas informacje, wydarzenia, komentarze, opinie.' |
| 13 |
category = 'newspaper' |
| 14 |
language = 'pl' |
| 15 |
masthead_url = 'http://5.s.dziennik.pl/images/logos.png' |
| 16 |
cover_url = 'http://5.s.dziennik.pl/images/logos.png' |
| 17 |
no_stylesheets = True |
| 18 |
oldest_article = 7 |
| 19 |
max_articles_per_feed = 100 |
| 20 |
remove_javascript = True |
| 21 |
remove_empty_feeds = True |
| 22 |
ignore_duplicate_articles = {'title', 'url'} |
| 23 |
extra_css = 'ul {list-style: none; padding: 0; margin: 0;} .foto {float: left;} .clr {clear: both;}' |
| 24 |
preprocess_regexps = [ |
| 25 |
(re.compile(r'Komentarze:'), lambda m: ''), |
| 26 |
(re.compile(r'<p><strong><a href=".*?">>>> CZYTAJ TAKŻE: ".*?"</a></strong></p>'), lambda m: ''), |
| 27 |
] |
| 28 |
keep_only_tags = [dict(id='article')] |
| 29 |
remove_tags = [ |
| 30 |
dict( |
| 31 |
name='div', |
| 32 |
attrs={ |
| 33 |
'class': [ |
| 34 |
'art_box_dodatki', |
| 35 |
'new_facebook_icons2', |
| 36 |
'leftArt', |
| 37 |
'article_print', |
| 38 |
'quiz-widget', |
| 39 |
'belka-spol', |
| 40 |
'belka-spol belka-spol-bottom', |
| 41 |
'art_data_tags', |
| 42 |
'cl_right', |
| 43 |
'boxRounded gal_inside', |
| 44 |
] |
| 45 |
}, |
| 46 |
), |
| 47 |
dict(name='a', attrs={'class': ['komentarz', 'article_icon_addcommnent']}), |
| 48 |
dict(name='ins'), |
| 49 |
dict(name='br'), |
| 50 |
] # noqa: E501 |
| 51 |
feeds = [ |
| 52 |
('Wszystko', 'http://rss.dziennik.pl/Dziennik-PL/'), |
| 53 |
('Wiadomości', 'http://rss.dziennik.pl/Dziennik-Wiadomosci'), |
| 54 |
('Gospodarka', 'http://rss.dziennik.pl/Dziennik-Gospodarka'), |
| 55 |
('Kobieta', 'http://rss.dziennik.pl/Dziennik-Kobieta'), |
| 56 |
('Auto', 'http://rss.dziennik.pl/Dziennik-Auto'), |
| 57 |
('Rozrywka', 'http://rss.dziennik.pl/Dziennik-Rozrywka'), |
| 58 |
('Film', 'http://rss.dziennik.pl/Dziennik-Film'), |
| 59 |
('Muzyka', 'http://rss.dziennik.pl/Dziennik-Muzyka'), |
| 60 |
('Kultura', 'http://rss.dziennik.pl/Dziennik-Kultura'), |
| 61 |
('Nauka', 'http://rss.dziennik.pl/Dziennik-Nauka'), |
| 62 |
('Podróże', 'http://rss.dziennik.pl/Dziennik-Podroze/'), |
| 63 |
('Nieruchomości', 'http://rss.dziennik.pl/Dziennik-Nieruchomosci'), |
| 64 |
] |
| 65 |
|
| 66 |
def skip_ad_pages(self, soup): |
| 67 |
tag = soup.find(name='a', attrs={'title': 'CZYTAJ DALEJ'}) |
| 68 |
if tag: |
| 69 |
new_soup = self.index_to_soup(tag['href'], raw=True) |
| 70 |
return new_soup |
| 71 |
|
| 72 |
def append_page(self, soup, appendtag): |
| 73 |
tag = soup.find('a', attrs={'class': 'page_next'}) |
| 74 |
if tag: |
| 75 |
appendtag.find('div', attrs={'class': 'article_paginator'}).extract() |
| 76 |
while tag: |
| 77 |
soup2 = self.index_to_soup(tag['href']) |
| 78 |
tag = soup2.find('a', attrs={'class': 'page_next'}) |
| 79 |
if not tag: |
| 80 |
for r in appendtag.findAll('div', attrs={'class': 'art_src'}): |
| 81 |
r.extract() |
| 82 |
pagetext = soup2.find(name='div', attrs={'class': 'article_body'}) |
| 83 |
for dictionary in self.remove_tags: |
| 84 |
v = pagetext.findAll(name=dictionary['name'], attrs=dictionary['attrs']) |
| 85 |
for delete in v: |
| 86 |
delete.extract() |
| 87 |
comments = pagetext.findAll(text=lambda text: isinstance(text, Comment)) |
| 88 |
for comment in comments: |
| 89 |
comment.extract() |
| 90 |
pos = len(appendtag.contents) |
| 91 |
appendtag.insert(pos, pagetext) |
| 92 |
if appendtag.find('div', attrs={'class': 'article_paginator'}): |
| 93 |
appendtag.find('div', attrs={'class': 'article_paginator'}).extract() |
| 94 |
|
| 95 |
def preprocess_html(self, soup): |
| 96 |
self.append_page(soup, soup.body) |
| 97 |
return soup |