Debian Codegraph @

recipes/modoros.recipe

raw
1 import os
2 import os.path
3 import re
4
5 from calibre.constants import CONFIG_DIR_MODE, config_dir
6 from calibre.web.feeds.recipes import BasicNewsRecipe
7
8 try:
9 from urllib.parse import quote
10 except ImportError:
11 from urllib import quote
12 from hashlib import md5
13
14
15 class ModorosBlogHu(BasicNewsRecipe):
16 __author__ = 'Zsolt Botykai'
17 title = 'Modoros Blog'
18 description = 'Modoros.blog.hu'
19 oldest_article = 10000
20 max_articles_per_feed = 10000
21 reverse_article_order = True
22 language = 'hu'
23 remove_javascript = True
24 remove_empty_feeds = True
25 no_stylesheets = True
26 feeds = [('Modoros Blog', 'http://modoros.blog.hu/rss')]
27 use_embedded_content = False
28 preprocess_regexps = [
29 (re.compile(r'<!--megosztas -->.*?</body>', re.DOTALL | re.IGNORECASE), lambda match: '</body>'),
30 (re.compile(r'<p align="left"'), lambda m: '<p'),
31 (re.compile(r'<noscript.+?noscript>', re.DOTALL | re.IGNORECASE), lambda m: ''),
32 (re.compile(r'<img style="position: absolute;top:-10px.+?>', re.DOTALL | re.IGNORECASE), lambda m: ''),
33 (re.compile(r'<p>( |&nbsp;)*?</p>', re.DOTALL | re.IGNORECASE), lambda match: ''),
34 ]
35 extra_css = '''
36 body { background-color: white; color: black }
37 '''
38
39 remove_tags = [
40 dict(name='div', attrs={'id': ['csucs']}),
41 dict(name='img', attrs={'style': ['position: absolute;top:-10px;left:-10px;']}),
42 dict(name='div', attrs={'class': ['tovabb-is-van', 'page-break', 'clear']}),
43 dict(name='span', attrs={'class': ['hozzaszolas-szamlalo']}),
44 ]
45
46 masthead_url = 'http://modoros.blog.hu/media/skins/modoros-neon/img/modorosblog-felirat.png'
47
48 def get_cover_url(self):
49 return 'http://modoros.blog.hu/media/skins/modoros-neon/img/modorosblog-felirat.png'
50
51 # As seen here:
52 # http://www.mobileread.com/forums/showpost.php?p=1295505&postcount=10
53 def parse_feeds(self):
54 recipe_dir = os.path.join(config_dir, 'recipes')
55 hash_dir = os.path.join(recipe_dir, 'recipe_storage')
56 feed_dir = os.path.join(hash_dir, self.title.encode('utf-8').replace('/', ':'))
57 if not os.path.isdir(feed_dir):
58 os.makedirs(feed_dir, mode=CONFIG_DIR_MODE)
59
60 feeds = BasicNewsRecipe.parse_feeds(self)
61
62 for feed in feeds:
63 feed_hash = quote(feed.title.encode('utf-8'), safe='')
64 feed_fn = os.path.join(feed_dir, feed_hash)
65
66 past_items = set()
67 if os.path.exists(feed_fn):
68 with open(feed_fn) as f:
69 for h in f:
70 past_items.add(h.strip())
71
72 cur_items = set()
73 for article in feed.articles[:]:
74 item_hash = md5()
75 if article.content:
76 item_hash.update(article.content.encode('utf-8'))
77 if article.summary:
78 item_hash.update(article.summary.encode('utf-8'))
79 item_hash = item_hash.hexdigest()
80 if article.url:
81 item_hash = article.url + ':' + item_hash
82 cur_items.add(item_hash)
83 if item_hash in past_items:
84 feed.articles.remove(article)
85 with open(feed_fn, 'w') as f:
86 for h in cur_items:
87 f.write(h + '\n')
88
89 remove = [fl for fl in feeds if len(fl) == 0 and self.remove_empty_feeds]
90 for f in remove:
91 feeds.remove(f)
92
93 return feeds