Debian Codegraph @

recipes/foreign_policy.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 from __future__ import absolute_import, division, print_function, unicode_literals
4
5 __license__ = 'GPL v3'
6 __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
7
8 from collections import defaultdict
9
10 from calibre.web.feeds.news import BasicNewsRecipe
11
12
13 def classes(classes):
14 q = frozenset(classes.split(' '))
15 return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)})
16
17
18 class ForeignPolicy(BasicNewsRecipe):
19 title = 'Foreign Policy'
20 language = 'en'
21 __author__ = 'Kovid Goyal'
22 description = 'International News'
23 no_stylesheets = True
24 remove_javascript = True
25 remove_empty_feeds = True
26 resolve_internal_links = True
27 encoding = 'utf-8'
28 remove_attributes = ['style', 'height', 'width']
29
30 extra_css = '''
31 img {display:block; margin:0 auto;}
32 .department-meta { font-size:small; color:#404040; }
33 .dek-heading, .author-bio, .date-time { font-size:small; color:#202020; }
34 .figure-image, .caption, .wp-caption { font-size:small; text-align:center; }
35 '''
36
37 keep_only_tags = [dict(name='article', attrs={'class': 'article'})]
38 remove_tags = [
39 dict(name=['meta', 'link', 'svg', 'button', 'iframe', 'aside']),
40 classes(
41 'share-links content-ungated related-articles fp-lightbox--overlay more-text myfp-article '
42 'editors-note-in-post--v2 author-photo related-articles-carousel sidebar-box_right '
43 ),
44 ]
45 remove_tags_after = [classes('post-content-main')]
46
47 recipe_specific_options = {'issue': {'short': 'Enter the Issue ID you want to download ', 'long': 'For example, 411131563'}}
48
49 def parse_index(self):
50 issue_url = 'https://foreignpolicy.com/the-magazine'
51 d = self.recipe_specific_options.get('issue')
52 if d and isinstance(d, str):
53 issue_url = issue_url + '/?issue_id=' + d
54
55 soup = self.index_to_soup(issue_url)
56 img = soup.find('img', attrs={'src': lambda x: x and '-cover' in x})
57 if img:
58 self.cover_url = img['src'].split('?')[0] + '?w=800?quality=90'
59 current_section = None
60 feeds_dict = defaultdict(list)
61 soup = soup.find('main')
62 for x in soup.findAll(name=('h2', 'h3')):
63 if x.name == 'h2':
64 current_section = self.tag_to_string(x)
65 self.log(current_section)
66 if current_section.lower() == 'recent issues':
67 break
68 else:
69 title = self.tag_to_string(x)
70 a = x.parent
71 url = a['href']
72 desc = ''
73 meta = a.findNext(attrs={'class': 'meta-data -excerpt'})
74 if meta:
75 desc += self.tag_to_string(meta)
76 dek = a.findNext(attrs={'class': 'dek-heading -excerpt'})
77 if dek:
78 desc += ' | ' + self.tag_to_string(dek)
79 self.log('\t', title, url, '\n\t', desc)
80 feeds_dict[current_section].append({'title': title, 'url': url, 'description': desc})
81 return list(feeds_dict.items())
82
83 def preprocess_html(self, soup):
84 for img in soup.findAll('img', attrs={'src': True}):
85 img['src'] = img['src'].split('?')[0] + '?w=600?quality=90'
86 return soup