Debian Codegraph @

recipes/dziennik_pl.recipe

raw
1 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
2
3 import re
4
5 from calibre.ebooks.BeautifulSoup import Comment
6 from calibre.web.feeds.news import BasicNewsRecipe
7
8
9 class Dziennik_pl(BasicNewsRecipe):
10 title = 'Dziennik.pl'
11 __author__ = 'fenuks'
12 description = 'Wiadomości z kraju i ze świata. Wiadomości gospodarcze. Znajdziesz u nas informacje, wydarzenia, komentarze, opinie.'
13 category = 'newspaper'
14 language = 'pl'
15 masthead_url = 'http://5.s.dziennik.pl/images/logos.png'
16 cover_url = 'http://5.s.dziennik.pl/images/logos.png'
17 no_stylesheets = True
18 oldest_article = 7
19 max_articles_per_feed = 100
20 remove_javascript = True
21 remove_empty_feeds = True
22 ignore_duplicate_articles = {'title', 'url'}
23 extra_css = 'ul {list-style: none; padding: 0; margin: 0;} .foto {float: left;} .clr {clear: both;}'
24 preprocess_regexps = [
25 (re.compile(r'Komentarze:'), lambda m: ''),
26 (re.compile(r'<p><strong><a href=".*?">&gt;&gt;&gt; CZYTAJ TAKŻE: ".*?"</a></strong></p>'), lambda m: ''),
27 ]
28 keep_only_tags = [dict(id='article')]
29 remove_tags = [
30 dict(
31 name='div',
32 attrs={
33 'class': [
34 'art_box_dodatki',
35 'new_facebook_icons2',
36 'leftArt',
37 'article_print',
38 'quiz-widget',
39 'belka-spol',
40 'belka-spol belka-spol-bottom',
41 'art_data_tags',
42 'cl_right',
43 'boxRounded gal_inside',
44 ]
45 },
46 ),
47 dict(name='a', attrs={'class': ['komentarz', 'article_icon_addcommnent']}),
48 dict(name='ins'),
49 dict(name='br'),
50 ] # noqa: E501
51 feeds = [
52 ('Wszystko', 'http://rss.dziennik.pl/Dziennik-PL/'),
53 ('Wiadomości', 'http://rss.dziennik.pl/Dziennik-Wiadomosci'),
54 ('Gospodarka', 'http://rss.dziennik.pl/Dziennik-Gospodarka'),
55 ('Kobieta', 'http://rss.dziennik.pl/Dziennik-Kobieta'),
56 ('Auto', 'http://rss.dziennik.pl/Dziennik-Auto'),
57 ('Rozrywka', 'http://rss.dziennik.pl/Dziennik-Rozrywka'),
58 ('Film', 'http://rss.dziennik.pl/Dziennik-Film'),
59 ('Muzyka', 'http://rss.dziennik.pl/Dziennik-Muzyka'),
60 ('Kultura', 'http://rss.dziennik.pl/Dziennik-Kultura'),
61 ('Nauka', 'http://rss.dziennik.pl/Dziennik-Nauka'),
62 ('Podróże', 'http://rss.dziennik.pl/Dziennik-Podroze/'),
63 ('Nieruchomości', 'http://rss.dziennik.pl/Dziennik-Nieruchomosci'),
64 ]
65
66 def skip_ad_pages(self, soup):
67 tag = soup.find(name='a', attrs={'title': 'CZYTAJ DALEJ'})
68 if tag:
69 new_soup = self.index_to_soup(tag['href'], raw=True)
70 return new_soup
71
72 def append_page(self, soup, appendtag):
73 tag = soup.find('a', attrs={'class': 'page_next'})
74 if tag:
75 appendtag.find('div', attrs={'class': 'article_paginator'}).extract()
76 while tag:
77 soup2 = self.index_to_soup(tag['href'])
78 tag = soup2.find('a', attrs={'class': 'page_next'})
79 if not tag:
80 for r in appendtag.findAll('div', attrs={'class': 'art_src'}):
81 r.extract()
82 pagetext = soup2.find(name='div', attrs={'class': 'article_body'})
83 for dictionary in self.remove_tags:
84 v = pagetext.findAll(name=dictionary['name'], attrs=dictionary['attrs'])
85 for delete in v:
86 delete.extract()
87 comments = pagetext.findAll(text=lambda text: isinstance(text, Comment))
88 for comment in comments:
89 comment.extract()
90 pos = len(appendtag.contents)
91 appendtag.insert(pos, pagetext)
92 if appendtag.find('div', attrs={'class': 'article_paginator'}):
93 appendtag.find('div', attrs={'class': 'article_paginator'}).extract()
94
95 def preprocess_html(self, soup):
96 self.append_page(soup, soup.body)
97 return soup