Debian Codegraph @

recipes/gry_online_pl.recipe

raw
1 from __future__ import print_function
2
3 import time
4
5 from calibre.ebooks.BeautifulSoup import Comment
6 from calibre.web.feeds.recipes import BasicNewsRecipe
7
8
9 class GryOnlinePl(BasicNewsRecipe):
10 title = 'Gry-Online.pl'
11 __author__ = 'fenuks'
12 description = 'WiadomoĊ›ci o grach, recenzje, zapowiedzi. Encyklopedia Gier zawiera opisy gier na PC, konsole Xbox360, PS3 i inne platformy.'
13 category = 'games'
14 language = 'pl'
15 oldest_article = 13
16 INDEX = 'http://www.gry-online.pl/'
17 masthead_url = 'http://www.gry-online.pl/im/gry-online-logo.png'
18 cover_url = 'http://www.gry-online.pl/im/gry-online-logo.png'
19 max_articles_per_feed = 100
20 no_stylesheets = True
21 keep_only_tags = [dict(name='div', attrs={'class': ['gc660', 'gc660 S013', 'news_endpage_tit', 'news_container', 'news']})]
22 remove_tags = [
23 {
24 'class': [
25 'nav-social',
26 'add-info',
27 'smlb',
28 'lista lista3 lista-gry',
29 'S013po',
30 'S013-npb',
31 'zm_gfx_cnt_bottom',
32 'ocen-txt',
33 'wiecej-txt',
34 'wiecej-txt2',
35 'social-for-old-news',
36 'social-for-old-rec',
37 ]
38 }
39 ] # noqa: E501
40 feeds = [('Newsy', 'http://www.gry-online.pl/rss/news.xml'), ('Teksty', 'http://www.gry-online.pl/rss/teksty.xml')]
41
42 def append_page(self, soup, appendtag):
43 tag = appendtag.find('div', attrs={'class': 'n5p'})
44 if tag:
45 nexturls = tag.findAll('a')
46 url_part = soup.find('link', attrs={'rel': 'canonical'})['href']
47 url_part = url_part[25:].rpartition('?')[0]
48 for nexturl in nexturls[1:-1]:
49 finalurl = 'http://www.gry-online.pl/' + url_part + nexturl['href']
50 for i in range(10):
51 try:
52 soup2 = self.index_to_soup(finalurl)
53 break
54 except Exception:
55 print('retrying in 0.5s')
56 time.sleep(0.5)
57 pagetext = soup2.find(attrs={'class': 'gc660'})
58 for r in pagetext.findAll(name='header'):
59 r.extract()
60 for r in pagetext.findAll(attrs={'itemprop': 'description'}):
61 r.extract()
62
63 pos = len(appendtag.contents)
64 appendtag.insert(pos, pagetext)
65 for r in appendtag.findAll(attrs={'class': ['n5p', 'add-info', 'twitter-share-button', 'lista lista3 lista-gry', 'imh10b']}):
66 r.extract()
67 comments = appendtag.findAll(text=lambda text: isinstance(text, Comment))
68 for comment in comments:
69 comment.extract()
70 else:
71 tag = appendtag.find('div', attrs={'class': 'S018stronyr'})
72 if tag:
73 nexturl = tag.a
74 url_part = soup.find('link', attrs={'rel': 'canonical'})['href']
75 url_part = url_part[25:].rpartition('?')[0]
76 while tag:
77 end = tag.find(attrs={'class': 'right left-dead'})
78 if end:
79 break
80 else:
81 nexturl = tag.a
82 finalurl = 'http://www.gry-online.pl/' + url_part + nexturl['href']
83 for i in range(10):
84 try:
85 soup2 = self.index_to_soup(finalurl)
86 break
87 except Exception:
88 print('retrying in 0.5s')
89 time.sleep(0.5)
90 tag = soup2.find('div', attrs={'class': 'S018stronyr'})
91 pagetext = soup2.find(attrs={'class': 'gc660'})
92 for r in pagetext.findAll(name='header'):
93 r.extract()
94 for r in pagetext.findAll(attrs={'itemprop': 'description'}):
95 r.extract()
96
97 comments = pagetext.findAll(text=lambda text: isinstance(text, Comment))
98 [comment.extract() for comment in comments]
99 pos = len(appendtag.contents)
100 appendtag.insert(pos, pagetext)
101 for r in appendtag.findAll(attrs={'class': ['n5p', 'add-info', 'twitter-share-button', 'lista lista3 lista-gry', 'S018strony', 'imh10b']}):
102 r.extract()
103 comments = appendtag.findAll(text=lambda text: isinstance(text, Comment))
104 for comment in comments:
105 comment.extract()
106
107 def image_url_processor(self, baseurl, url):
108 if url.startswith('..'):
109 return url[2:]
110 else:
111 return url
112
113 def preprocess_html(self, soup):
114 self.append_page(soup, soup.body)
115 return soup