Debian Codegraph @

recipes/portfolio.recipe

raw
1 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
2
3 import re
4
5 from calibre.web.feeds.news import BasicNewsRecipe
6
7
8 class AdvancedUserRecipe1348146838(BasicNewsRecipe):
9 title = 'Portfolio.hu'
10 __author__ = 'bubak'
11 oldest_article = 1
12 language = 'hu'
13
14 author = 'bubak'
15 simultaneous_downloads = 1
16 max_articles_per_feed = 50
17
18 feeds = [
19 ('Összes hír', 'https://www.portfolio.hu/rss/all.xml'),
20 ('Globál hírek', 'https://www.portfolio.hu/rss/global.xml'),
21 ('Uniós források hírek', 'https://www.portfolio.hu/rss/unios-forrasok.xml'),
22 ('Gazdaság', 'http://www.portfolio.hu/rss/cikkek/gazdasag.xml'),
23 ('Ingatlan', 'http://www.portfolio.hu/rss/cikkek/ingatlan.xml'),
24 ('Pénzcentrum', 'http://www.penzcentrum.hu/rss/all.xml'),
25 ('All news', 'https://www.portfolio.hu/en/rss/all.xml'),
26 ]
27
28 remove_tags = [
29 dict(name='div', attrs={'id': ['prevent-box']}),
30 dict(name='div', attrs={'class': re.compile(r'conf-short', re.IGNORECASE)}),
31 dict(name='img', attrs={'src': re.compile(r'\.(jpg|svg|gif)', re.IGNORECASE)}),
32 dict(name='div', attrs={'class': re.compile(r'event.*linked|hidden|blogger-block|grid-block', re.IGNORECASE)}),
33 dict(name='article', attrs={'class': re.compile(r'article-list-item', re.IGNORECASE)}),
34 dict(name='section', attrs={'class': re.compile(r'pfblogger-block|recentnews|pfforum-box|newslettersubs|ai-summary', re.IGNORECASE)}),
35 dict(name='section', attrs={'id': ['BlockPodcast', 'BlockCimlaprol', 'BlockTrader', 'BlockCalendar', 'BlockIngatlan']}),
36 dict(name='ul', attrs={'class': 'tags'}),
37 dict(name='div', attrs={'class': ['event-share', 'trc_rbox_container', 'related', 'author-name']}),
38 dict(name='span', attrs={'class': 'footer-wrapper'}),
39 ]
40 remove_tags_before = dict(name='h1')
41
42 preprocess_regexps = [(re.compile(r'<a\s[^>]*>([^<]*)</a>', re.DOTALL | re.IGNORECASE), lambda match: match.group(1))]
43
44 visited_urls = {}
45
46 def get_article_url(self, article):
47 url = BasicNewsRecipe.get_article_url(self, article)
48 if url in self.visited_urls:
49 self.log.debug('Ignoring duplicate: ' + url)
50 return None
51 else:
52 self.visited_urls[url] = True
53 self.log.debug('Accepting: ' + url)
54 return url