Debian Codegraph @

recipes/bama.recipe

raw
1 """
2 Fetch Bama
3 """
4
5 import re
6
7 from calibre.web.feeds.news import BasicNewsRecipe
8
9
10 class bama(BasicNewsRecipe):
11 title = 'Bama'
12 description = ''
13 __author__ = 'bubak'
14 oldest_article = 1
15 max_articles_per_feed = 30
16 no_stylesheets = True
17 language = 'hu'
18
19 remove_javascript = True
20 cover_url = 'https://www.bama.hu/wp-content/themes/megyeilapok2017/images/bama_footer_red.svg'
21
22 sections = [
23 ('Ország - világ', 'https://www.bama.hu/rovat/orszag-vilag', 1),
24 ('Pécs', 'https://www.bama.hu/regio/pecs/', 1),
25 ('Közélet', 'https://www.bama.hu/rovat/kozelet', 1),
26 ('Gazdaság', 'https://www.bama.hu/rovat/gazdasag', 1),
27 ('Kultúra', 'https://www.bama.hu/rovat/kultura/', 1),
28 ('Kék hírek', 'https://www.bama.hu/rovat/kek-hirek', 1),
29 ]
30
31 keep_only_tags = [dict(name='section', attrs={'class': 'article'})]
32 remove_tags_after = dict(name='app-newsletter')
33 remove_tags = [dict(name='div', attrs={'class': ['info', 'article-info', 'article-tags-share-block', 'content-block']}), dict(name='app-google-news')]
34 preprocess_regexps = [
35 (re.compile(r'<script.*?</script>', re.DOTALL), lambda match: ''),
36 (re.compile(r'<div id="fb-root".*', re.DOTALL), lambda match: '</body>'),
37 (re.compile(r'<div class="ad .*?</div>', re.DOTALL), lambda match: ''),
38 (re.compile(r'<div class="cikkshare cf.*?Mik ezek.*?id="miertjo".*?</form>.*?</div>', re.DOTALL), lambda match: ''),
39 ]
40
41 def get_browser(self, *a, **kw):
42 kw['user_agent'] = 'Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/119.0'
43 br = BasicNewsRecipe.get_browser(self, *a, **kw)
44 self.log.debug('entry User-Agent: ')
45 return br
46
47 def parse_index(self):
48 res = []
49 seen_titles = set()
50 for section in self.sections:
51 self.log('\tTrying section ' + section[0])
52 self.log('\tTrying url ' + section[1])
53 articles = []
54 soup = self.index_to_soup(section[1])
55 if section[2] == 0:
56 content = soup.findAll('div', attrs={'class': re.compile(r'enews-article-offerer-content-container', re.IGNORECASE)})
57 elif section[2] == 1:
58 content = soup.findAll('div', attrs={'class': 'article-data'})
59 else:
60 content = soup.findAll('div', attrs={'class': 'article-data'})
61 if content is None:
62 raise ValueError('Could not find category content')
63 for c in content:
64 article = c.find('a', text=re.compile(r'.'))
65 if article and article.get('href'):
66 url = article['href']
67 # title = re.sub(r'^\s*(\S.*\S)\s*$', r'\1', article.text)
68 title = article.text
69 if url.startswith('/'):
70 url = 'https://www.bama.hu' + url
71 if title in seen_titles:
72 self.log('\tAlready seen: ', title)
73 continue
74 seen_titles.add(title)
75 self.log('\tFound article:', title, 'at', url)
76 articles.append({'title': title, 'url': url, 'description': '', 'date': ''})
77 res.append((section[0], articles))
78 return res