Debian Codegraph @

recipes/american_thinker.recipe

raw
1 __license__ = 'GPL v3'
2 __copyright__ = '2010, Walt Anthony <workshop.northpole at gmail.com>'
3 '''
4 www.americanthinker.com
5 '''
6 import html5lib
7 from lxml import etree
8
9 from calibre.utils.cleantext import clean_xml_chars
10 from calibre.web.feeds.news import BasicNewsRecipe
11
12
13 class AmericanThinker(BasicNewsRecipe):
14 title = 'American Thinker'
15 description = 'American Thinker is a daily internet publication devoted to the thoughtful exploration of issues of importance to Americans.'
16 __author__ = 'Walt Anthony'
17 publisher = 'Thomas Lifson'
18 category = 'news, politics, USA'
19 oldest_article = 7 # days
20 max_articles_per_feed = 50
21 summary_length = 150
22 language = 'en_US'
23 ignore_duplicate_articles = {'title', 'url'}
24
25 remove_javascript = True
26 auto_cleanup = True
27
28 conversion_options = {'comment': description, 'tags': category, 'publisher': publisher, 'language': language, 'linearize_tables': True}
29
30 def preprocess_raw_html(self, raw, url):
31 root = html5lib.parse(clean_xml_chars(raw), treebuilder='lxml', namespaceHTMLElements=False)
32 for x in root.xpath(
33 "descendant-or-self::*[@class and contains(concat(' ', normalize-space(@class), ' '), ' article_body ')"
34 " and (@class and contains(concat(' ', normalize-space(@class), ' '), ' bottom '))]"
35 ):
36 x.getparent().remove(x)
37 return etree.tostring(root, encoding='unicode')
38
39 feeds = [('http://feeds.feedburner.com/americanthinker'), ('http://feeds.feedburner.com/AmericanThinkerBlog')]