Debian Codegraph @

recipes/slate.recipe

raw
1 #!/usr/bin/env python
2 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
3
4 __license__ = 'GPL v3'
5
6 '''
7 calibre recipe for slate.com
8 '''
9
10 from calibre.web.feeds.recipes import BasicNewsRecipe
11
12
13 def classes(classes):
14 q = frozenset(classes.split(' '))
15 return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)})
16
17
18 class Slate(BasicNewsRecipe):
19 title = 'Slate'
20 description = 'A general-interest publication offering analysis and commentary about politics, news and culture.'
21 __author__ = 'unkn0wn'
22 no_stylesheets = True
23 language = 'en'
24 encoding = 'utf-8'
25 remove_attributes = ['style', 'height', 'width']
26 INDEX = 'https://slate.com/'
27 resolve_internal_links = True
28 remove_empty_feeds = True
29 ignore_duplicate_articles = {'url'}
30
31 extra_css = '''
32 .article__rubric { font-size:small; color:#404040; }
33 .article__byline, time { font-size:small; }
34 .article__top-image, .image__credit, .image__meta { font-size:small; text-align:center; }
35 em, blockquote, .alternativeHeadline { color:#202020; }
36 '''
37
38 keep_only_tags = [
39 classes('article__header article__top-image article__content article-hotseats__body'),
40 ]
41 remove_tags = [
42 dict(name=['aside', 'svg', 'iframe']),
43 classes('social-share slate-ad newsletter-signup in-article-recirc'),
44 ]
45
46 def preprocess_html(self, soup):
47 for h2 in soup.findAll('h2'):
48 h2.name = 'h4'
49 for img in soup.findAll('img', attrs={'data-srcset': True}):
50 img['src'] = img['data-src'] + '&width=600'
51 return soup
52
53 def parse_index(self):
54 ans = []
55 for sectitle, url in (
56 ('News & Politics', 'news-and-politics'),
57 ('Culture', 'culture'),
58 ('Technology', 'technology'),
59 ('Business', 'business'),
60 ('Life', 'life'),
61 ('Advice', 'advice'),
62 ):
63 url = self.INDEX + url
64 self.log('\nFound section:', sectitle, url)
65 articles = self.slate_section_articles(url)
66 if articles:
67 ans.append((sectitle, articles))
68 return ans
69
70 def slate_section_articles(self, url):
71 from datetime import date
72
73 soup = self.index_to_soup(url)
74 ans = []
75 dt = date.today().strftime('/%Y/')
76 for a in soup.findAll('a', attrs={'href': lambda x: x and x.startswith(url + dt)}):
77 url = a['href']
78 head = a.find(
79 attrs={
80 'class': [
81 'section-feed-two-column__card-headline',
82 'section-feed-three-column__teaser-headline',
83 'section-feed-two-column__teaser-headline',
84 'topic-story__hed',
85 ]
86 }
87 )
88 if head:
89 title = self.tag_to_string(head).strip()
90 self.log('\t' + title)
91 self.log('\t\t' + url)
92 ans.append({'title': title, 'url': url})
93 return ans
94
95 def populate_article_metadata(self, article, soup, first):
96 summ = soup.find(attrs={'class': 'article__dek'})
97 if summ:
98 article.summary = article.text_summary = self.tag_to_string(summ)