Debian Codegraph @

recipes/dzieje_pl.recipe

raw
1 from calibre.ebooks.BeautifulSoup import Comment
2 from calibre.web.feeds.news import BasicNewsRecipe
3
4
5 class Dzieje(BasicNewsRecipe):
6 title = 'dzieje.pl'
7 __author__ = 'fenuks'
8 description = 'Dzieje.pl - najlepszy portal informacyjno-edukacyjny dotyczący historii Polski XX wieku. Archiwalne fotografie, filmy, katalog postaci, quizy i konkursy.' # noqa: E501
9 cover_url = 'http://www.dzieje.pl/sites/default/files/dzieje_logo.png'
10 category = 'history'
11 language = 'pl'
12 ignore_duplicate_articles = {'title', 'url'}
13 extra_css = '.imagecache-default {float:left; margin-right:20px;}'
14 index = 'http://dzieje.pl'
15 oldest_article = 8
16 max_articles_per_feed = 100
17 remove_javascript = True
18 no_stylesheets = True
19 keep_only_tags = [dict(name='h1', attrs={'class': 'title'}), dict(id='content-area')]
20 remove_tags = [dict(attrs={'class': 'field field-type-computed field-field-tagi'}), dict(id='dogory'), dict(name='blockquote')]
21
22 def append_page(self, soup, appendtag):
23 tag = appendtag.find('li', attrs={'class': 'pager-next'})
24 if tag:
25 while tag:
26 url = tag.a['href']
27 if not url.startswith('http'):
28 url = 'http://dzieje.pl' + tag.a['href']
29 soup2 = self.index_to_soup(url)
30 pagetext = soup2.find(id='content-area').find(attrs={'class': 'content'})
31 for r in pagetext.findAll(
32 attrs={
33 'class': [
34 'fieldgroup group-groupkul',
35 'fieldgroup group-zdjeciekult',
36 'fieldgroup group-zdjecieciekaw',
37 'fieldgroup group-zdjecieksiazka',
38 'fieldgroup group-zdjeciedu',
39 'field field-type-filefield field-field-zdjecieglownawyd',
40 ]
41 }
42 ): # noqa: E501
43 r.extract()
44 comments = pagetext.findAll(text=lambda text: isinstance(text, Comment))
45 # appendtag.insert(pos, pagetext)
46 tag = soup2.find('li', attrs={'class': 'pager-next'})
47 for r in appendtag.findAll(
48 attrs={
49 'class': [
50 'item-list',
51 'field field-type-computed field-field-tagi',
52 ]
53 }
54 ):
55 r.extract()
56 comments = appendtag.findAll(text=lambda text: isinstance(text, Comment))
57 for comment in comments:
58 comment.extract()
59
60 def find_articles(self, url):
61 articles = []
62 soup = self.index_to_soup(url)
63 tag = soup.find(id='content-area').div.div
64 for i in tag.findAll('div', recursive=False):
65 temp = i.find(attrs={'class': 'views-field-title'}).span.a
66 title = temp.string
67 url = self.index + temp['href']
68 # i.find(attrs={'class':'views-field-created'}).span.string
69 date = ''
70 articles.append({'title': title, 'url': url, 'date': date, 'description': ''})
71 return articles
72
73 def parse_index(self):
74 feeds = []
75 feeds.append(('Wiadomości', self.find_articles('http://dzieje.pl/wiadomosci')))
76 feeds.append(('Kultura i sztuka', self.find_articles('http://dzieje.pl/kulturaisztuka')))
77 feeds.append(('Film', self.find_articles('http://dzieje.pl/kino')))
78 feeds.append(('Rozmaitości historyczne', self.find_articles('http://dzieje.pl/rozmaitości')))
79 feeds.append(('Książka', self.find_articles('http://dzieje.pl/ksiazka')))
80 feeds.append(('Wystawa', self.find_articles('http://dzieje.pl/wystawa')))
81 feeds.append(('Edukacja', self.find_articles('http://dzieje.pl/edukacja')))
82 feeds.append(('Dzieje się', self.find_articles('http://dzieje.pl/wydarzenia')))
83 return feeds
84
85 def preprocess_html(self, soup):
86 for a in soup('a', href=True):
87 if not a['href'].startswith('http'):
88 a['href'] = self.index + a['href']
89 self.append_page(soup, soup.body)
90 return soup