Debian Codegraph @

recipes/syrzdarma.recipe

raw
1 import re
2
3 from calibre.web.feeds.recipes import BasicNewsRecipe
4
5
6 class syrzdarmaRecipe(BasicNewsRecipe):
7 __author__ = 'bubak'
8 title = 'Sýr zdarma'
9 description = 'syrzdarma.cz'
10 oldest_article = 1
11 max_articles_per_feed = 20
12
13 sections = [('Sýr zdarma', 'https://www.syrzdarma.cz')]
14
15 language = 'cs'
16 cover_url = 'https://www.syrzdarma.cz/assets/logo-text-optimized.svg'
17 remove_javascript = False
18 no_stylesheets = True
19 use_embedded_content = False
20 recursions = 0
21 articles_are_obfuscated = True
22
23 ignore_duplicate_articles = {'title', 'url'}
24
25 remove_attributes = []
26 remove_tags_before = dict(name='h1', attrs={'class': ['AAAtitulek-clanku']})
27 remove_tags = [dict(name='img'), dict(name='div', attrs={'AAAclass': re.compile(r'sidebar-column|post-share|no-login-icon', re.IGNORECASE)})]
28
29 visited_urls = {}
30
31 def get_article_url(self, article):
32 url = BasicNewsRecipe.get_article_url(self, article)
33 if url in self.visited_urls:
34 self.log.debug('Ignoring duplicate: ' + url)
35 return None
36 else:
37 self.visited_urls[url] = True
38 self.log.debug('Accepting: ' + url)
39 return url
40
41 def get_obfuscated_article(self, url):
42 result = None
43 response = self.browser.open(url)
44 html = response.read().decode('utf-8')
45
46 s1 = re.sub(r'^.*cdk-overlay-container', '', html, flags=re.DOTALL)
47 s2 = re.sub(r'^.*articleComponent_article', '', s1, flags=re.DOTALL)
48 s2a = re.sub(r'\n.*', '', s2)
49 s3 = re.sub(r'\\&q;', '', s2a)
50 s4 = re.sub(r',*{insert:([^}]*)},*', r'\1', s3)
51 s5 = re.sub(r',*{attributes:{([^:]*)[^}]*},insert:([^}]*)}', r'<\1>\2</\1>', s4)
52 s5a = re.sub(r',*{attributes:{nbsp:{}},insert:([^}]*)}', '&nbsp;', s5)
53 s6 = re.sub(r'\\\\n', '<br>', s5a)
54 s6a = re.sub(r'\\n', '<br>', s6)
55 s7 = s6a.replace(r'bold>', 'b>')
56 s8 = s7.replace(r'italic>', 'i>')
57 s9 = re.sub(r'].*', '', s8)
58 s9a = s9.replace('&s;', "'")
59 s10 = re.sub(r'^.*title&q;:&q;([^&]*)&.*perex&q;:&q;([^&]*)&q.*\[', r'<h1>\1</h1><i>\2</i>', s9a)
60
61 result = s10
62 return {'data': result, 'url': url}
63
64 def parse_index(self):
65 res = []
66 for section in self.sections:
67 self.log('\tTrying section ' + section[0])
68 soup = self.index_to_soup(section[1])
69 content = soup.find_all('h1')
70 if content is None:
71 raise ValueError('Could not find category content')
72 seen_urls = set()
73 articles = []
74 for article in content:
75 a = article.find('a')
76 if a.get('href'):
77 url = a['href']
78 if url.startswith('/'):
79 url = 'https://syrzdarma.cz' + url
80 if url in seen_urls:
81 continue
82 seen_urls.add(url)
83 title = a.contents[0]
84 self.log('\tFound article:', title, url)
85 articles.append({'title': title, 'url': url, 'description': '', 'date': ''})
86 res.append((section[0], articles))
87 return res
88
89 def image_url_processor(b, u):
90 None