Debian Codegraph @

recipes/wash_post_print.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 """
4 washingtonpost.com
5 """
6
7 import json
8
9 from html5_parser import parse
10
11 from calibre.web.feeds.news import BasicNewsRecipe
12
13
14 class wapoprint(BasicNewsRecipe):
15 title = 'The Washington Post | Print Edition'
16 __author__ = 'unkn0wn'
17 description = (
18 'Leading source for news, video and opinion on politics, business, '
19 'world and national news, science, travel, entertainment and more. '
20 'Our local coverage includes reporting on education, crime, weather, '
21 'traffic, real estate, jobs and cars for DC, Maryland and Virginia. '
22 'Offering award-winning opinion writing, entertainment information '
23 'and restaurant reviews.'
24 )
25 publisher = 'The Washington Post Company'
26 category = 'news, politics, USA'
27 no_stylesheets = True
28 use_embedded_content = False
29 encoding = 'utf-8'
30 language = 'en_US'
31 remove_attributes = ['style', 'height', 'width']
32 publication_type = 'newspaper'
33 resolve_internal_links = True
34 ignore_duplicate_articles = {'title', 'url'}
35 masthead_url = 'https://upload.wikimedia.org/wikipedia/commons/9/93/The_Logo_of_The_Washington_Post_Newspaper.svg'
36 extra_css = '''
37 .img { text-align:center; font-size:small; }
38 .auth { font-weight:bold; font-size:small; }
39 .time { font-size:small; color: #202020; }
40 .subt { font-style: italic; }
41 '''
42
43 def get_browser(self, *args, **kwargs):
44 kwargs['user_agent'] = 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
45 br = BasicNewsRecipe.get_browser(self, *args, **kwargs)
46 br.addheaders += [
47 ('Referer', 'https://www.google.com/'),
48 ('X-Forwarded-For', '66.249.66.1'),
49 ]
50 return br
51
52 def parse_index(self):
53 soup = self.index_to_soup('https://www.washingtonpost.com/todays_paper/updates/')
54 if img := soup.find('img', attrs={'src': lambda x: x and x.endswith('_FrontPage.png')}):
55 self.cover_url = img['src']
56 if h2 := soup.find('h2', attrs={'class': lambda x: x and 'font--subhead' in x.split()}):
57 self.title = 'WaPo Print | ' + self.tag_to_string(h2)
58
59 feeds = []
60
61 for div in soup.findAll('section', attrs={'id': True}):
62 secname = self.tag_to_string(div.find('label')).strip()
63 self.log(secname)
64 articles = []
65 for a in div.findAll('a', href=True):
66 url = a['href']
67 title = self.tag_to_string(a).strip()
68 if not title or not url:
69 continue
70 self.log('\t', title)
71 self.log('\t\t', url)
72 articles.append({'title': title, 'url': url})
73 if articles:
74 feeds.append((secname, articles))
75 return feeds
76
77 def preprocess_raw_html(self, raw, url):
78 root = parse(raw)
79 if '/interactive/' in url:
80 return (
81 '<html><body><article><h1>' + root.xpath('//h1')[0].text + '</h1><em>'
82 'This article is supposed to be read in a browser.'
83 '</em></article></body></html>'
84 )
85 m = root.xpath('//script[@id="__NEXT_DATA__"]')
86
87 data = json.loads(m[0].text)
88 data = data['props']['pageProps']['globalContent']
89
90 text = data.get('label', {}).get('basic', {}).get('text', '')
91 label = f'<p class="time">{text}</p>' if text else ''
92 title = '<h1>' + data['headlines']['basic'] + '</h1>'
93 subhead = '<h3 class="subhead">' + data['description'].get('basic', '') + '</h3>'
94
95 promo_img = ''
96 if data.get('promo_items', {}).get('basic', {}).get('type', '') == 'image':
97 pi = data['promo_items']['basic']
98 promo_img = '<p><div class="img"><img src="{}"><div class="caption">{}</div></div></p>'.format(pi['url'], pi['credits_caption_display'])
99
100 author = ''
101 if 'credits' in data:
102 author = (
103 '<div><span class="auth">'
104 + 'By '
105 + ', '.join(x['name'] for x in data['credits']['by'])
106 + '</span> | <span class="time">'
107 + data['publish_date'][:-14]
108 + '</span></div>'
109 )
110
111 body = ''
112 for x in data['content_elements']:
113 if x['type'] == 'text':
114 body += '<p>' + x['content'] + '</p>'
115 elif x['type'] == 'video':
116 if 'promo_image' in x:
117 body += '<p><div class="img"><img src="{}"><div class="caption">{}</div></div></p>'.format(
118 x['promo_image']['url'], x['description'].get('basic', '')
119 )
120 elif x['type'] == 'image':
121 img_ = '<p><div class="img"><img src="{}"><div class="caption">{}</div></div></p>'.format(x['url'], x['credits_caption_display'])
122 if img_ != promo_img:
123 body += img_
124 elif x['type'] == 'list':
125 body += '<ul>'
126 for li in x['items']:
127 if li.get('content', '') != '':
128 body += f'<li>{li["content"]}</li>'
129 body += '</ul>'
130
131 return '<html><body><div>' + label + title + subhead + promo_img + author + body + '</div></body></html>'
132
133 def preprocess_html(self, soup):
134 for img in soup.findAll('img', attrs={'src': True}):
135 img['src'] = 'https://www.washingtonpost.com/wp-apps/imrs.php?src=' + img['src'] + '&w=600'
136 return soup
137
138 def populate_article_metadata(self, article, soup, first):
139 article.summary = article.text_summary = self.tag_to_string(soup.find('p', attrs={'class': 'subt'}))