Debian Codegraph @

recipes/truthout.recipe

raw
1 from calibre.web.feeds.news import BasicNewsRecipe
2
3
4 class Truthout_Sage(BasicNewsRecipe):
5 title = 'Truthout'
6 __author__ = 'PaulB223'
7 language = 'en'
8 encoding = 'utf-8'
9 oldest_article = 4
10 max_articles_per_feed = 100
11
12 no_stylesheets = True
13 remove_javascript = True
14 remove_attributes = ['style', 'width', 'height']
15
16 feeds = [
17 ('Truthout News', 'http://truthout.org/feed?format=feed'),
18 ]
19
20 remove_tags = [
21 dict(name=['nav', 'header', 'footer', 'aside']),
22 dict(name=['link', 'meta', 'style', 'svg', 'input', 'source', 'noscript', 'button']),
23 dict(
24 attrs={
25 'class': lambda x: (
26 x and any(c in x.lower() for c in ['social', 'donate', 'share', 'related', 'topics', 'author-wrapper', 'banner', 'newsletter', 'appeal'])
27 )
28 }
29 ),
30 dict(id=['menu', 'donate-region', 'bannerandheader', 'search-container']),
31 dict(attrs={'class': lambda x: x and 'republish' in x.lower()}),
32 ]
33
34 remove_tags_after = [dict(attrs={'class': lambda x: x and 'author-bio' in x.split()}), dict(attrs={'class': lambda x: x and 'entry-content' in x.split()})]
35
36 def get_browser(self, *a, **kw):
37 from calibre import random_user_agent
38
39 kw['user_agent'] = random_user_agent(allow_ie=False)
40 return BasicNewsRecipe.get_browser(self, *a, **kw)
41
42 def preprocess_html(self, soup):
43 for h1 in soup.findAll('h1'):
44 h1.name = 'h2'
45
46 for img in soup.findAll('img'):
47 if img.has_attr('data-src'):
48 img['src'] = img['data-src']
49
50 return soup