| 1 |
from __future__ import print_function |
| 2 |
|
| 3 |
import time |
| 4 |
|
| 5 |
from calibre.ebooks.BeautifulSoup import Comment |
| 6 |
from calibre.web.feeds.recipes import BasicNewsRecipe |
| 7 |
|
| 8 |
|
| 9 |
class GryOnlinePl(BasicNewsRecipe): |
| 10 |
title = 'Gry-Online.pl' |
| 11 |
__author__ = 'fenuks' |
| 12 |
description = 'WiadomoĊci o grach, recenzje, zapowiedzi. Encyklopedia Gier zawiera opisy gier na PC, konsole Xbox360, PS3 i inne platformy.' |
| 13 |
category = 'games' |
| 14 |
language = 'pl' |
| 15 |
oldest_article = 13 |
| 16 |
INDEX = 'http://www.gry-online.pl/' |
| 17 |
masthead_url = 'http://www.gry-online.pl/im/gry-online-logo.png' |
| 18 |
cover_url = 'http://www.gry-online.pl/im/gry-online-logo.png' |
| 19 |
max_articles_per_feed = 100 |
| 20 |
no_stylesheets = True |
| 21 |
keep_only_tags = [dict(name='div', attrs={'class': ['gc660', 'gc660 S013', 'news_endpage_tit', 'news_container', 'news']})] |
| 22 |
remove_tags = [ |
| 23 |
{ |
| 24 |
'class': [ |
| 25 |
'nav-social', |
| 26 |
'add-info', |
| 27 |
'smlb', |
| 28 |
'lista lista3 lista-gry', |
| 29 |
'S013po', |
| 30 |
'S013-npb', |
| 31 |
'zm_gfx_cnt_bottom', |
| 32 |
'ocen-txt', |
| 33 |
'wiecej-txt', |
| 34 |
'wiecej-txt2', |
| 35 |
'social-for-old-news', |
| 36 |
'social-for-old-rec', |
| 37 |
] |
| 38 |
} |
| 39 |
] # noqa: E501 |
| 40 |
feeds = [('Newsy', 'http://www.gry-online.pl/rss/news.xml'), ('Teksty', 'http://www.gry-online.pl/rss/teksty.xml')] |
| 41 |
|
| 42 |
def append_page(self, soup, appendtag): |
| 43 |
tag = appendtag.find('div', attrs={'class': 'n5p'}) |
| 44 |
if tag: |
| 45 |
nexturls = tag.findAll('a') |
| 46 |
url_part = soup.find('link', attrs={'rel': 'canonical'})['href'] |
| 47 |
url_part = url_part[25:].rpartition('?')[0] |
| 48 |
for nexturl in nexturls[1:-1]: |
| 49 |
finalurl = 'http://www.gry-online.pl/' + url_part + nexturl['href'] |
| 50 |
for i in range(10): |
| 51 |
try: |
| 52 |
soup2 = self.index_to_soup(finalurl) |
| 53 |
break |
| 54 |
except Exception: |
| 55 |
print('retrying in 0.5s') |
| 56 |
time.sleep(0.5) |
| 57 |
pagetext = soup2.find(attrs={'class': 'gc660'}) |
| 58 |
for r in pagetext.findAll(name='header'): |
| 59 |
r.extract() |
| 60 |
for r in pagetext.findAll(attrs={'itemprop': 'description'}): |
| 61 |
r.extract() |
| 62 |
|
| 63 |
pos = len(appendtag.contents) |
| 64 |
appendtag.insert(pos, pagetext) |
| 65 |
for r in appendtag.findAll(attrs={'class': ['n5p', 'add-info', 'twitter-share-button', 'lista lista3 lista-gry', 'imh10b']}): |
| 66 |
r.extract() |
| 67 |
comments = appendtag.findAll(text=lambda text: isinstance(text, Comment)) |
| 68 |
for comment in comments: |
| 69 |
comment.extract() |
| 70 |
else: |
| 71 |
tag = appendtag.find('div', attrs={'class': 'S018stronyr'}) |
| 72 |
if tag: |
| 73 |
nexturl = tag.a |
| 74 |
url_part = soup.find('link', attrs={'rel': 'canonical'})['href'] |
| 75 |
url_part = url_part[25:].rpartition('?')[0] |
| 76 |
while tag: |
| 77 |
end = tag.find(attrs={'class': 'right left-dead'}) |
| 78 |
if end: |
| 79 |
break |
| 80 |
else: |
| 81 |
nexturl = tag.a |
| 82 |
finalurl = 'http://www.gry-online.pl/' + url_part + nexturl['href'] |
| 83 |
for i in range(10): |
| 84 |
try: |
| 85 |
soup2 = self.index_to_soup(finalurl) |
| 86 |
break |
| 87 |
except Exception: |
| 88 |
print('retrying in 0.5s') |
| 89 |
time.sleep(0.5) |
| 90 |
tag = soup2.find('div', attrs={'class': 'S018stronyr'}) |
| 91 |
pagetext = soup2.find(attrs={'class': 'gc660'}) |
| 92 |
for r in pagetext.findAll(name='header'): |
| 93 |
r.extract() |
| 94 |
for r in pagetext.findAll(attrs={'itemprop': 'description'}): |
| 95 |
r.extract() |
| 96 |
|
| 97 |
comments = pagetext.findAll(text=lambda text: isinstance(text, Comment)) |
| 98 |
[comment.extract() for comment in comments] |
| 99 |
pos = len(appendtag.contents) |
| 100 |
appendtag.insert(pos, pagetext) |
| 101 |
for r in appendtag.findAll(attrs={'class': ['n5p', 'add-info', 'twitter-share-button', 'lista lista3 lista-gry', 'S018strony', 'imh10b']}): |
| 102 |
r.extract() |
| 103 |
comments = appendtag.findAll(text=lambda text: isinstance(text, Comment)) |
| 104 |
for comment in comments: |
| 105 |
comment.extract() |
| 106 |
|
| 107 |
def image_url_processor(self, baseurl, url): |
| 108 |
if url.startswith('..'): |
| 109 |
return url[2:] |
| 110 |
else: |
| 111 |
return url |
| 112 |
|
| 113 |
def preprocess_html(self, soup): |
| 114 |
self.append_page(soup, soup.body) |
| 115 |
return soup |