Debian Codegraph @

recipes/wash_post.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 """
4 www.washingtonpost.com
5 """
6
7 import json
8
9 from html5_parser import parse
10
11 from calibre.web.feeds.news import BasicNewsRecipe
12
13
14 class TheWashingtonPost(BasicNewsRecipe):
15 title = 'The Washington Post'
16 __author__ = 'unkn0wn'
17 description = (
18 'Leading source for news, video and opinion on politics, business, '
19 'world and national news, science, travel, entertainment and more. '
20 'Our local coverage includes reporting on education, crime, weather, '
21 'traffic, real estate, jobs and cars for DC, Maryland and Virginia. '
22 'Offering award-winning opinion writing, entertainment information '
23 'and restaurant reviews.'
24 )
25 publisher = 'The Washington Post Company'
26 category = 'news, politics, USA'
27 oldest_article = 1.2
28 max_articles_per_feed = 200
29 no_stylesheets = True
30 encoding = 'utf8'
31 use_embedded_content = False
32 language = 'en_US'
33 remove_empty_feeds = True
34 resolve_internal_links = True
35 ignore_duplicate_articles = {'url'}
36 masthead_url = 'https://upload.wikimedia.org/wikipedia/commons/9/93/The_Logo_of_The_Washington_Post_Newspaper.svg'
37 publication_type = 'newspaper'
38 remove_attributes = ['style', 'width', 'height']
39
40 recipe_specific_options = {
41 'days': {
42 'short': 'Oldest article to download from this news source. In days ',
43 'long': 'For example, 0.5, gives you articles from the past 12 hours',
44 'default': str(oldest_article),
45 }
46 }
47
48 def __init__(self, *args, **kwargs):
49 BasicNewsRecipe.__init__(self, *args, **kwargs)
50 d = self.recipe_specific_options.get('days')
51 if d and isinstance(d, str):
52 self.oldest_article = float(d)
53
54 extra_css = '''
55 .img { text-align:center; font-size:small; }
56 .auth { font-weight:bold; font-size:small; }
57 .time { font-size:small; color: #202020; }
58 .subt { font-style: italic; }
59 '''
60
61 def get_cover_url(self):
62 soup = self.index_to_soup('https://www.washingtonpost.com/todays_paper/updates/')
63 if img := soup.find('img', attrs={'src': lambda x: x and x.endswith('_FrontPage.png')}):
64 return img['src']
65
66 def get_browser(self, *args, **kwargs):
67 kwargs['user_agent'] = 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
68 br = BasicNewsRecipe.get_browser(self, *args, **kwargs)
69 br.addheaders += [
70 ('Referer', 'https://www.google.com/'),
71 ('X-Forwarded-For', '66.249.66.1'),
72 ]
73 return br
74
75 # Official feeds: https://www.washingtonpost.com/discussions/2018/10/12/washington-post-rss-feeds/
76 feeds = [
77 ('Politics', 'http://feeds.washingtonpost.com/rss/politics'),
78 ('Opinions', 'http://feeds.washingtonpost.com/rss/opinions'),
79 ('Local', 'http://feeds.washingtonpost.com/rss/local'),
80 ('Sports', 'http://feeds.washingtonpost.com/rss/sports'),
81 ('Technology', 'http://feeds.washingtonpost.com/rss/business/technology'),
82 ('National', 'http://feeds.washingtonpost.com/rss/national'),
83 ('World', 'http://feeds.washingtonpost.com/rss/world'),
84 ('Business', 'http://feeds.washingtonpost.com/rss/business'),
85 ('Lifestyle', 'http://feeds.washingtonpost.com/rss/lifestyle'),
86 ('Entertainment', 'http://feeds.washingtonpost.com/rss/entertainment'),
87 # Undocumented feeds.
88 ('White House', 'http://feeds.washingtonpost.com/rss/politics/whitehouse'),
89 ('Commanders', 'http://feeds.washingtonpost.com/rss/sports/redskins'),
90 ]
91
92 def preprocess_raw_html(self, raw, url):
93 root = parse(raw)
94 if '/interactive/' in url:
95 return (
96 '<html><body><article><h1>' + root.xpath('//h1')[0].text + '</h1><em>'
97 'This article is supposed to be read in a browser.'
98 '</em></article></body></html>'
99 )
100 m = root.xpath('//script[@id="__NEXT_DATA__"]')
101
102 data = json.loads(m[0].text)
103 data = data['props']['pageProps']['globalContent']
104
105 text = data.get('label', {}).get('basic', {}).get('text', '')
106 label = f'<p class="time">{text}</p>' if text else ''
107 title = '<h1>' + data['headlines']['basic'] + '</h1>'
108 subhead = '<p class="subt">' + data['description'].get('basic', '') + '</h3>'
109
110 promo_img = ''
111 if data.get('promo_items', {}).get('basic', {}).get('type', '') == 'image':
112 pi = data['promo_items']['basic']
113 promo_img = '<p><div class="img"><img src="{}"><div>{}</div></div></p>'.format(pi['url'], pi['credits_caption_display'])
114
115 author = ''
116 if 'credits' in data:
117 author = (
118 '<div><span class="auth">'
119 + 'By '
120 + ', '.join(x['name'] for x in data['credits']['by'])
121 + '</span> | <span class="time">'
122 + data['publish_date'][:-14]
123 + '</span></div>'
124 )
125
126 body = ''
127 for x in data['content_elements']:
128 if x['type'] == 'text':
129 body += '<p>' + x['content'] + '</p>'
130 elif x['type'] == 'video':
131 if 'promo_image' in x:
132 body += '<p><div class="img"><img src="{}"><div>{}</div></div></p>'.format(x['promo_image']['url'], x['description'].get('basic', ''))
133 elif x['type'] == 'image':
134 img_ = '<p><div class="img"><img src="{}"><div>{}</div></div></p>'.format(x['url'], x['credits_caption_display'])
135 if img_ != promo_img:
136 body += img_
137 elif x['type'] == 'list':
138 body += '<ul>'
139 for li in x['items']:
140 if li.get('content', '') != '':
141 body += f'<li>{li["content"]}</li>'
142 body += '</ul>'
143
144 return '<html><body><div>' + label + title + subhead + promo_img + author + body + '</div></body></html>'
145
146 def preprocess_html(self, soup):
147 for img in soup.findAll('img', attrs={'src': True}):
148 img['src'] = 'https://www.washingtonpost.com/wp-apps/imrs.php?src=' + img['src'] + '&w=600'
149 return soup