Debian Codegraph @

recipes/blikk.recipe

raw
1 """
2 Fetch Blikk
3 """
4
5 import re
6
7 from calibre.web.feeds.news import BasicNewsRecipe
8
9
10 class blikk(BasicNewsRecipe):
11 title = 'Blikk'
12 description = ''
13 __author__ = 'bubak'
14 use_embedded_content = False
15 oldest_article = 2
16 max_articles_per_feed = 40
17 no_stylesheets = True
18 language = 'hu'
19 simultaneous_downloads = 5
20
21 remove_javascript = True
22 cover_url = 'http://imagestore1.btl.hu/57_10_2465_576f066aec514d35e7fee2081dde5ffe_07ce26_701.jpg'
23
24 feeds = [('Blikk.hu hírek', 'http://blikk.hu/rss')]
25
26 extra_css = '''
27 '''
28
29 preprocess_regexps = [(re.compile(r'href="[^"]*["]', re.DOTALL | re.IGNORECASE), lambda match: '')]
30
31 remove_tags = [
32 dict(name='span', attrs={'class': ['hirdetes']}),
33 dict(name='div', attrs={'class': ['cikkFejlec']}),
34 dict(name='div', attrs={'id': re.compile(r'div-gpt-ad|adoceanindex', re.IGNORECASE)}),
35 dict(name='a', attrs={'class': re.compile(r'gap-2', re.IGNORECASE)}),
36 dict(name='p', attrs={'class': re.compile(r'py-2', re.IGNORECASE)}),
37 dict(name='div', attrs={'id': ['subscription-box-container', 'articleOfferFlag']}),
38 dict(name='div', attrs={'data-tagrecommendation': '/next-topmodel-hungary'}),
39 dict(name='div', attrs={'data-blockIndex'}),
40 dict(name='header', attrs={'id': ['header']}),
41 dict(name='section', attrs={'class': re.compile(r'rightside|google-news|breadcrumb|pb-3', re.IGNORECASE)}),
42 dict(name='section', attrs={'class': lambda x: x and 'rightSide' in x.split()}),
43 dict(name='section', attrs={'id': 'articleEnd'}),
44 dict(name='img', attrs={'alt': 'Szólj hozzá!'}),
45 ]
46
47 remove_tags_after = dict(name='section', attrs={'id': 'articleEnd'})
48
49 def skip_ad_pages(self, soup):
50 # ads have fewer links
51 self.log.debug('Number of links: ', len(soup.findAll('a')))
52 self.log.debug('Links: ', soup.findAll('a'))
53 if len(soup.findAll('a')) < 20:
54 a = soup.find('a', text=re.compile(r'^click here')).findParents('a')
55 href = a[0]['href']
56 self.log.debug('Skipping to: ' + href)
57 new = self.browser.open(href).read().decode('utf-8', 'ignore')
58 self.log.debug('Finished: ' + href)
59 return new
60 return None