Debian Codegraph @

recipes/telex.recipe

raw
1 import re
2
3 from calibre.web.feeds.recipes import BasicNewsRecipe
4
5
6 class telexRecipe(BasicNewsRecipe):
7 __author__ = 'bubak'
8 title = 'telex'
9 description = 'telex.hu'
10 oldest_article = 1
11 max_articles_per_feed = 20
12
13 sections = [
14 ('BELFÖLD', 'https://telex.hu/rovat/belfold'),
15 ('KÜLFÖLD', 'https://telex.hu/rovat/kulfold'),
16 ('GAZDASÁG', 'https://telex.hu/rovat/gazdasag'),
17 ('ÉLET', 'https://telex.hu/rovat/eletmod'),
18 ('TECHTUD', 'https://telex.hu/rovat/techtud'),
19 ('ENGLISH', 'https://telex.hu/english'),
20 ]
21
22 language = 'hu'
23 no_stylesheets = True
24 use_embedded_content = False
25
26 preprocess_regexps = [(re.compile(r'<a\s[^>]*>([^<]*)</a>', re.DOTALL | re.IGNORECASE), lambda match: match.group(1))]
27
28 remove_attributes = []
29 remove_tags = [
30 dict(name='div', attrs={'class': ['navigation-container', 'content-wrapper', 'hint_box', 'journalist']}),
31 dict(name='div', attrs={'class': re.compile(r'navigation--ov26', re.IGNORECASE)}),
32 ]
33 remove_tags_after = dict(name='div', attrs={'class': 'support-box'})
34
35 visited_urls = {}
36
37 def get_browser(self, *a, **kw):
38 kw['verify_ssl_certificates'] = False
39 return super().get_browser(*a, **kw)
40
41 def parse_index(self):
42 res = []
43 for section in self.sections:
44 self.log('\tTrying section ' + section[0])
45 soup = self.index_to_soup(section[1])
46 content = soup.findAll('div', attrs={'class': re.compile(r'list__highlight__item__title', re.IGNORECASE)})
47 content += soup.findAll('a', attrs={'class': re.compile(r'list__item__title', re.IGNORECASE)})
48
49 if content is None:
50 raise ValueError('Could not find section content')
51 seen_urls = set()
52 articles = []
53 for a in content:
54 if a.a:
55 url = a.a['href']
56 if url.startswith('/'):
57 url = 'https://telex.hu' + url
58 if url in seen_urls:
59 continue
60 seen_urls.add(url)
61 self.log('\tFound ' + url)
62 title = a.a.contents[0]
63 self.log('\tFound article:', title, url)
64 articles.append({'title': title, 'url': url, 'description': '', 'date': ''})
65 else:
66 url = a['href']
67 if url.startswith('/'):
68 url = 'https://telex.hu' + url
69 if url in seen_urls:
70 continue
71 seen_urls.add(url)
72 self.log('\tFound ' + url)
73 title = a.find('div').find('div').find('div').contents[0]
74 self.log('\tFound article:', title, url)
75 articles.append({'title': title, 'url': url, 'description': '', 'date': ''})
76 res.append((section[0], articles))
77 return res
78
79 def populate_article_metadata(self, article, soup, first):
80 self.log('Article date ', article.date)