| 1 |
""" |
| 2 |
Fetch Bama |
| 3 |
""" |
| 4 |
|
| 5 |
import re |
| 6 |
|
| 7 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 8 |
|
| 9 |
|
| 10 |
class bama(BasicNewsRecipe): |
| 11 |
title = 'Bama' |
| 12 |
description = '' |
| 13 |
__author__ = 'bubak' |
| 14 |
oldest_article = 1 |
| 15 |
max_articles_per_feed = 30 |
| 16 |
no_stylesheets = True |
| 17 |
language = 'hu' |
| 18 |
|
| 19 |
remove_javascript = True |
| 20 |
cover_url = 'https://www.bama.hu/wp-content/themes/megyeilapok2017/images/bama_footer_red.svg' |
| 21 |
|
| 22 |
sections = [ |
| 23 |
('Ország - világ', 'https://www.bama.hu/rovat/orszag-vilag', 1), |
| 24 |
('Pécs', 'https://www.bama.hu/regio/pecs/', 1), |
| 25 |
('Közélet', 'https://www.bama.hu/rovat/kozelet', 1), |
| 26 |
('Gazdaság', 'https://www.bama.hu/rovat/gazdasag', 1), |
| 27 |
('Kultúra', 'https://www.bama.hu/rovat/kultura/', 1), |
| 28 |
('Kék hírek', 'https://www.bama.hu/rovat/kek-hirek', 1), |
| 29 |
] |
| 30 |
|
| 31 |
keep_only_tags = [dict(name='section', attrs={'class': 'article'})] |
| 32 |
remove_tags_after = dict(name='app-newsletter') |
| 33 |
remove_tags = [dict(name='div', attrs={'class': ['info', 'article-info', 'article-tags-share-block', 'content-block']}), dict(name='app-google-news')] |
| 34 |
preprocess_regexps = [ |
| 35 |
(re.compile(r'<script.*?</script>', re.DOTALL), lambda match: ''), |
| 36 |
(re.compile(r'<div id="fb-root".*', re.DOTALL), lambda match: '</body>'), |
| 37 |
(re.compile(r'<div class="ad .*?</div>', re.DOTALL), lambda match: ''), |
| 38 |
(re.compile(r'<div class="cikkshare cf.*?Mik ezek.*?id="miertjo".*?</form>.*?</div>', re.DOTALL), lambda match: ''), |
| 39 |
] |
| 40 |
|
| 41 |
def get_browser(self, *a, **kw): |
| 42 |
kw['user_agent'] = 'Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/119.0' |
| 43 |
br = BasicNewsRecipe.get_browser(self, *a, **kw) |
| 44 |
self.log.debug('entry User-Agent: ') |
| 45 |
return br |
| 46 |
|
| 47 |
def parse_index(self): |
| 48 |
res = [] |
| 49 |
seen_titles = set() |
| 50 |
for section in self.sections: |
| 51 |
self.log('\tTrying section ' + section[0]) |
| 52 |
self.log('\tTrying url ' + section[1]) |
| 53 |
articles = [] |
| 54 |
soup = self.index_to_soup(section[1]) |
| 55 |
if section[2] == 0: |
| 56 |
content = soup.findAll('div', attrs={'class': re.compile(r'enews-article-offerer-content-container', re.IGNORECASE)}) |
| 57 |
elif section[2] == 1: |
| 58 |
content = soup.findAll('div', attrs={'class': 'article-data'}) |
| 59 |
else: |
| 60 |
content = soup.findAll('div', attrs={'class': 'article-data'}) |
| 61 |
if content is None: |
| 62 |
raise ValueError('Could not find category content') |
| 63 |
for c in content: |
| 64 |
article = c.find('a', text=re.compile(r'.')) |
| 65 |
if article and article.get('href'): |
| 66 |
url = article['href'] |
| 67 |
# title = re.sub(r'^\s*(\S.*\S)\s*$', r'\1', article.text) |
| 68 |
title = article.text |
| 69 |
if url.startswith('/'): |
| 70 |
url = 'https://www.bama.hu' + url |
| 71 |
if title in seen_titles: |
| 72 |
self.log('\tAlready seen: ', title) |
| 73 |
continue |
| 74 |
seen_titles.add(title) |
| 75 |
self.log('\tFound article:', title, 'at', url) |
| 76 |
articles.append({'title': title, 'url': url, 'description': '', 'date': ''}) |
| 77 |
res.append((section[0], articles)) |
| 78 |
return res |