Debian Codegraph @

recipes/project_syndicate.recipe

raw
1 #!/usr/bin/env python
2
3 from calibre.web.feeds.news import BasicNewsRecipe, classes
4
5
6 class projectsynd(BasicNewsRecipe):
7 title = 'Project Syndicate'
8 __author__ = 'unkn0wn'
9 description = (
10 'Project Syndicate produces and delivers original, high-quality commentaries to a global audience. '
11 'Featuring exclusive contributions by prominent political leaders, policymakers, scholars, business '
12 'leaders, and civic activists from around the world, we provide news media and their readers with cutting-edge '
13 'analysis and insight, regardless of ability to pay. Our membership includes over 500 media outlets – more '
14 'than half of which receive our commentaries for free or at subsidized rates – in 156 countries. '
15 'Download Weekly.'
16 )
17 language = 'en'
18 no_stylesheets = True
19 remove_javascript = True
20 masthead_url = 'https://www.project-syndicate.org/images/PS_Logotype.svg'
21 ignore_duplicate_articles = {'title', 'url'}
22 resolve_internal_links = True
23 remove_empty_feeds = True
24 remove_attributes = ['style', 'height', 'width']
25 oldest_article = 7 # days
26 delay = 2
27
28 def get_browser(self, *args, **kwargs):
29 kwargs['user_agent'] = 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
30 br = BasicNewsRecipe.get_browser(self, *args, **kwargs)
31 br.addheaders += [
32 ('Referer', 'https://www.google.com/'),
33 ('X-Forwarded-For', '66.249.66.1'),
34 ]
35 return br
36
37 recipe_specific_options = {
38 'days': {
39 'short': 'Oldest article to download from this news source. In days ',
40 'long': 'For example, 0.5, gives you articles from the past 12 hours',
41 'default': str(oldest_article),
42 }
43 }
44
45 def __init__(self, *args, **kwargs):
46 BasicNewsRecipe.__init__(self, *args, **kwargs)
47 d = self.recipe_specific_options.get('days')
48 if d and isinstance(d, str):
49 self.oldest_article = float(d)
50
51 extra_css = '''
52 [aria-label="Author information"], .article__byline, .image--lead { font-size: small; }
53 .article__abs { font-style: italic; }
54 em { font-style:italic; color:#202020; }
55 '''
56
57 keep_only_tags = [
58 classes('image--lead article_title article__byline article__title--main article__abs article__body'),
59 dict(attrs={'aria-label': 'Author information'}),
60 ]
61
62 remove_tags = [
63 dict(name='source'),
64 classes('inlay--slide slide__container editorpick-container special--generic mrf--promotion'),
65 ]
66
67 feeds = [
68 'https://www.project-syndicate.org/rss/section/economics',
69 'https://www.project-syndicate.org/rss/section/politics-world-affairs',
70 'https://www.project-syndicate.org/rss/onpoint',
71 'https://www.project-syndicate.org/rss/section/environment-sustainability',
72 'https://www.project-syndicate.org/rss/section/global-health-development',
73 'https://www.project-syndicate.org/rss/section/culture-society',
74 'https://www.project-syndicate.org/rss/section/innovation-technology',
75 'https://www.project-syndicate.org/rss',
76 ]
77
78 def preprocess_html(self, soup):
79 if div := soup.find(**classes('article__abs')):
80 if p := div.find('p'):
81 div.name = 'p'
82 div.clear()
83 div.string = p.get_text()
84 for h2 in soup.findAll(['h2', 'h3']):
85 h2.name = 'h4'
86 for img in soup.findAll('img', attrs={'src': True}):
87 img['src'] = img['src'].replace('medium', 'xlarge')
88 return soup