Debian Codegraph @

recipes/gosc_full.recipe

raw
1 # -*- coding: utf-8 -*-
2 from __future__ import unicode_literals
3
4 __license__ = 'GPL v3'
5 __copyright__ = '2011, Piotr Kontek, [email protected] 2013-2018, Tomasz Długosz, [email protected]'
6
7 import re
8 import time
9
10 from lxml import html
11
12 from calibre.web.feeds.news import BasicNewsRecipe
13
14
15 class GN(BasicNewsRecipe):
16 __author__ = 'Piotr Kontek, Tomasz Długosz'
17 title = 'Gość Niedzielny - pełny numer'
18 publisher = 'Wydawnictwo Kurii Metropolitalnej w Katowicach'
19 description = 'Ogólnopolski tygodnik katolicki - pełny numer sprzed 4 tygodni'
20 encoding = 'utf-8'
21 no_stylesheets = True
22 language = 'pl'
23 remove_javascript = True
24 ignore_duplicate_articles = {'url'}
25 masthead_url = 'http://gosc.pl/static/images/base/logotypes/100x68/gosc-niedzielny.png'
26
27 def find_last_issue(self):
28 raw = self.index_to_soup('http://gosc.pl/wyszukaj/wydania/3.Gosc-Niedzielny/', raw=True)
29 doc = html.fromstring(raw)
30 page = doc.xpath('//div[@class="search-result release-result"]/div[1]/div[1]/a/@href')
31
32 if time.strftime('%w') in ['3', '4']:
33 return page[5]
34 else:
35 return page[4]
36
37 def parse_index(self):
38 self.last_issue = self.find_last_issue()
39 soup = self.index_to_soup('http://gosc.pl' + self.last_issue)
40 self.cover_url = 'http://www.gosc.pl' + soup.find('div', attrs={'class': 'fl-w100 release-wp'}).findAll('a')[-4].contents[0]['src']
41 feeds = []
42 # editorial:
43 a = soup.find('div', attrs={'class': 'release-wp-b'})
44 art = a.find('a')
45 articles = [
46 {
47 'title': self.tag_to_string(art),
48 'url': 'http://www.gosc.pl' + art['href'],
49 'description': self.tag_to_string(a.find('p', attrs={'class': 'b lead'})),
50 }
51 ]
52 feeds.append(('Na dobry początek', articles))
53 # columns:
54 for addr in soup.findAll('a', attrs={'href': re.compile(r'kategoria')}):
55 if not addr.span:
56 main_block = self.index_to_soup('http://www.gosc.pl' + addr['href'])
57 articles = list(self.find_articles(main_block))
58 if len(articles) > 0:
59 section = addr.contents[0]
60 feeds.append((section, articles))
61 # not assigned content:
62 page = 1
63 not_assigned = []
64 while True:
65 soup = self.index_to_soup('http://gosc.pl' + self.last_issue.replace('przeglad', 'wszystko') + '/' + str(page))
66 articles = list(self.find_articles(soup))
67 not_assigned.extend(articles)
68 page += 1
69 pages = soup.find('span', attrs={'class': 'pgr_nrs'})
70 if str(page) not in [self.tag_to_string(x) for x in pages.findAll('a')]:
71 break
72
73 feeds.append(('Nieprzypisane', not_assigned))
74 return feeds
75
76 def find_articles(self, main_block):
77 for a in main_block.findAll('div', attrs={'class': ['infoBox']}):
78 art = a.find('a')
79 yield {
80 'title': self.tag_to_string(art),
81 'url': 'http://www.gosc.pl' + art['href'].split('.')[0],
82 'date': self.tag_to_string(a.find('b', attrs={'class': 'time'})).replace('DODANE', ' '),
83 'description': self.tag_to_string(a.find('div', attrs={'class': 'txt'})),
84 }
85
86 def append_page(self, soup, appendtag):
87 chpage = appendtag.find(attrs={'class': 'pgr_nrs'})
88 if chpage:
89 for page in chpage.findAll('a'):
90 soup2 = self.index_to_soup('http://gosc.pl' + page['href'])
91 pagetext = soup2.find(attrs={'class': 'intextAd'})
92 pos = len(appendtag.contents)
93 appendtag.insert(pos, pagetext)
94
95 def preprocess_html(self, soup):
96 self.append_page(soup, soup.body)
97 r = soup.find(attrs={'class': 'lightbox'})
98 if r:
99 r.contents[0]['src'] = r['href']
100 return soup
101
102 def postprocess_html(self, soup, first_fetch):
103 for r in soup.findAll(attrs={'class': 'pgr'}):
104 r.extract()
105 for r in soup.findAll(attrs={'class': ['di_dr', 'doc_image']}):
106 del r['style']
107 '''
108 for r in soup.findAll(attrs={'class': 'cm-i-a'}):
109 r.replaceWith('<div style="clear:both"></div>' +
110 r.prettify() + '<div style="clear:both"></div>')
111 '''
112 return soup
113
114 keep_only_tags = [dict(name='div', attrs={'class': ['cf txt ', 'cf txt att-audio']})]
115
116 remove_tags = [
117 dict(name='p', attrs={'class': 'l l-2 doc-source'}),
118 dict(name='span', attrs={'class': 'wykop'}),
119 dict(
120 name='div', attrs={'class': ['doc_actions', 'cf', 'fr1_cl', 'txt__social-icons', 'txt__tags', 'fb-like fb_iframe_widget', 'jp-audio', 'jp-jplayer']}
121 ),
122 dict(name='div', attrs={'id': 'vote'}),
123 dict(name='link'),
124 dict(name='a', attrs={'class': 'img_enlarge'}),
125 ]
126
127 extra_css = '''
128 h1 {font-size:150%}
129 p.limiter {font-size:150%; font-weight: bold}
130 p.gn_subtitle {font-weight: bold}
131 span.cm-i-a {text-transform:uppercase;font-size:50%}
132 span.cm-i-p {font-style:italic; font-size:70%;text-align:right}
133 span.gn_brb {color: red; font-weight: bold}
134 div.txt__lead {font-weight: bold; font-size:150%}
135 '''