Debian Codegraph @

recipes/ars_technica.recipe

raw
1 #!/usr/bin/env python
2 __license__ = 'GPL v3'
3 __copyright__ = '2008-2012, Darko Miletic <darko.miletic at gmail.com>'
4 '''
5 arstechnica.com
6 '''
7
8 import re
9
10 from calibre.web.feeds.news import BasicNewsRecipe
11
12
13 def classes(classes):
14 q = frozenset(classes.split(' '))
15 return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)})
16
17
18 class ArsTechnica(BasicNewsRecipe):
19 title = 'Ars Technica'
20 language = 'en'
21 __author__ = 'Darko Miletic, Sujata Raman, Alexis Rohou, Tom Sparks'
22 description = 'Ars Technica: Serving the technologist for 1.2 decades'
23 publisher = 'Conde Nast Publications'
24 oldest_article = 5
25 max_articles_per_feed = 100
26 no_stylesheets = True
27 encoding = 'utf-8'
28 use_embedded_content = False
29 remove_empty_feeds = True
30 ignore_duplicate_articles = {'url', 'title'}
31 masthead_url = 'https://upload.wikimedia.org/wikipedia/commons/5/51/Ars_Technica_logo_%282016%29.svg'
32 extra_css = '''
33 body {font-family: Arial,sans-serif}
34 .heading{font-family: "Times New Roman",serif}
35 .byline{font-weight: bold; line-height: 1em; font-size: 0.625em; text-decoration: none}
36 img{display: block}
37 .caption-text{font-size:small; font-style:italic}
38 .caption-byline{font-size:small; font-style:italic; font-weight:bold}
39 .video, .page-numbers, .story-sidebar { display: none }
40 .image { display: block }
41 '''
42
43 recipe_specific_options = {
44 'days': {
45 'short': 'Oldest article to download from this news source. In days ',
46 'long': 'For example, 0.5, gives you articles from the past 12 hours',
47 'default': str(oldest_article),
48 }
49 }
50
51 def __init__(self, *args, **kwargs):
52 BasicNewsRecipe.__init__(self, *args, **kwargs)
53 d = self.recipe_specific_options.get('days')
54 if d and isinstance(d, str):
55 self.oldest_article = float(d)
56
57 keep_only_tags = [
58 dict(itemprop=['article-header', 'article-body']),
59 classes('main post-meta article-guts standalone'),
60 ]
61
62 remove_tags = [
63 classes(
64 'xf_thread_iframe_wrapper topnav-sections nav-previous nav-next ars-gallery-caption-credit'
65 ' ars-gallery-caption-text ars-gallery-caption-arrow font-impact sr-only author-mini-bio'
66 ' taboola-container comments site-header site-footer video corner-info article-expander'
67 ' left-column related-stories article-social text-settings-dropdown-story intro-image'
68 ),
69 dict(name=['object', 'link', 'embed', 'iframe', 'meta']),
70 dict(id=['social-left', 'article-footer-wrap']),
71 dict(name='nav', attrs={'class': 'subheading'}),
72 ]
73 remove_attributes = ['lang', 'style']
74
75 # Feed are found here: http://arstechnica.com/rss-feeds/
76 feeds = [
77 ('Ars Technica', 'https://feeds.arstechnica.com/arstechnica/index'),
78 ('Features', 'https://feeds.arstechnica.com/arstechnica/features'),
79 ('Technology Lab', 'https://feeds.arstechnica.com/arstechnica/technology-lab'),
80 ('Gear & Gadgets', 'https://feeds.arstechnica.com/arstechnica/gadgets'),
81 ('Law & Disorder', 'https://feeds.arstechnica.com/arstechnica/tech-policy'),
82 # ('Health', 'https://arstechnica.com/health/feed/'),
83 # ('Policy', 'https://arstechnica.com/tech-policy/feed/'),
84 # ('Security', 'https://arstechnica.com/security/feed/'),
85 ('Infinite Loop', 'https://feeds.arstechnica.com/arstechnica/apple'),
86 ('Opposable Thumbs', 'https://feeds.arstechnica.com/arstechnica/gaming'),
87 ('Scientific Method', 'https://feeds.arstechnica.com/arstechnica/science'),
88 ('Cars Technica', 'https://feeds.arstechnica.com/arstechnica/cars'),
89 ('Staff', 'https://feeds.arstechnica.com/arstechnica/staff'),
90 # ('AI', 'https://arstechnica.com/staff/ai/feed/'),
91 # ('Space', 'https://arstechnica.com/space/feed/'),
92 ('Ars Cardboard', 'https://feeds.arstechnica.com/arstechnica/cardboard'),
93 ]
94
95 recursions = 1
96
97 def is_link_wanted(self, url, tag):
98 return re.search(r'/[0-9]/$', url) is not None
99
100 def postprocess_html(self, soup, first_fetch):
101 if not first_fetch:
102 for x in soup.findAll(itemprop=['headline', 'description']):
103 x.extract()
104 for x in soup.findAll(**classes('post-meta')):
105 x.extract()
106 return soup