Debian Codegraph @

recipes/atlantic_com.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 # License: GPLv3 Copyright: 2015, Kovid Goyal <kovid at kovidgoyal.net>
4 import json
5 from xml.sax.saxutils import escape, quoteattr
6
7 from calibre.utils.iso8601 import parse_iso8601
8 from calibre.web.feeds.news import BasicNewsRecipe, classes
9 from calibre.web.feeds.news import prefixed_classes as prefix_classes
10
11 web_version = True
12 test_article = None
13 # test_article = 'https://www.theatlantic.com/health/archive/2020/12/covid-19-second-surge/617415/?utm_source=feed'
14
15
16 # parse article JSON {{{
17 def process_image_block(lines, block):
18 caption = block.get('captionText')
19 caption_lines = []
20 if caption:
21 if block.get('attributionText', '').strip():
22 caption += ' (' + block['attributionText'] + ')'
23 caption_lines.append('<p style="font-style: italic">' + caption + '</p>')
24 lines.append('<div style="text-align: center"><img src={}/>'.format(quoteattr(block['url'])))
25 lines.extend(caption_lines)
26 lines.append('</div>')
27
28
29 def json_to_html(raw):
30 data = json.loads(raw)
31 # open('/t/p.json', 'w').write(json.dumps(data, indent=2))
32 data = max((v['data'] for v in data['props']['pageProps']['urqlState'].values()), key=len)
33 article = json.loads(data)['article']
34 # open('/t/p.json', 'w').write(json.dumps(article, indent=2))
35 lines = []
36 lines.append('<h1 style="align: center">' + escape(article['title']) + '</h1>')
37 lines.append('<h2 style="align: center">' + escape(article['dek']) + '</h2>')
38 auts = ', '.join(x['displayName'] for x in article['authors'])
39 if auts:
40 lines.append('<p style="align: center">by ' + escape(auts) + '</p>')
41 dp = article['datePublished']
42 dt = parse_iso8601(dp)
43 lines.append('<p style="align: center">' + escape(dt.strftime('%a, %d %b %Y')) + '</p>')
44 if article.get('leadArt') and 'image' in article['leadArt']:
45 process_image_block(lines, article['leadArt']['image'])
46 for item in article['content']:
47 tn = item.get('__typename', '')
48 if tn.endswith('Image'):
49 process_image_block(lines, item)
50 continue
51 html = item.get('innerHtml')
52 if html is None or '</iframe>' in html:
53 continue
54 if 'innerHtml' not in item:
55 continue
56 tagname = item.get('tagName', 'P').lower()
57 lines.append('<{0}>{1}</{0}>'.format(tagname, html))
58 return '<html><body><div id="from-json-by-calibre">' + '\n'.join(lines) + '</div></body></html>'
59
60
61 class NoJSON(ValueError):
62 pass
63
64
65 def extract_html(soup):
66 script = soup.findAll('script', id='__NEXT_DATA__')
67 if not script:
68 raise NoJSON('No script tag with JSON data found')
69 raw = script[0].contents[0]
70 return json_to_html(raw)
71
72
73 # }}}
74
75
76 class TheAtlantic(BasicNewsRecipe):
77 if web_version:
78 title = 'TheAtlantic.com'
79 description = 'News and editorial about politics, culture, entertainment, tech, etc. Contains many articles not seen in The Atlantic magazine'
80 else:
81 title = 'The Atlantic'
82 description = 'Current affairs and politics focussed on the US'
83 INDEX = 'https://www.theatlantic.com/magazine/'
84
85 __author__ = 'Kovid Goyal'
86 language = 'en'
87 encoding = 'utf-8'
88
89 compress_news_images = True
90 scale_news_images_to_device = True
91
92 keep_only_tags = [
93 dict(itemprop=['headline']),
94 classes(
95 'c-article-header__hed c-rubric article-header c-article-meta c-lead-media'
96 ' lead-img article-cover-extra article-body article-magazine article-cover-content'
97 ),
98 prefix_classes('ArticleHeader_root__ ArticleLayoutSection_main__ ArticleHero_timestamp__ ArticleBody_root__'),
99 dict(itemprop='articleBody'),
100 # these are for photos articles
101 dict(id=['article-header', 'from-json-by-calibre']),
102 classes('photos'),
103 ]
104 remove_tags = [
105 classes(
106 'c-ad c-share-social c-recirculation-link social-kit-top letter-writer-info callout secondary-byline embed-wrapper'
107 ' offset-wrapper boxtop-most-popular social-icons hints read-more c-article-writer__social'
108 ),
109 prefix_classes('ArticleRecirc_inline__'),
110 {'name': ['meta', 'link', 'noscript', 'aside', 'h3']},
111 {'attrs': {'class': ['offset-wrapper', 'boxtop-most-popular']}},
112 {'attrs': {'class': lambda x: x and 'article-tools' in x}},
113 {'src': lambda x: x and 'spotxchange.com' in x},
114 ]
115 remove_tags_after = classes('article-body')
116
117 no_stylesheets = True
118 remove_attributes = ['style']
119 extra_css = '''
120 .credit { text-align: right; font-size: 75%; display: block }
121 .figcaption { font-size: 75% }
122 .caption { font-size: 75% }
123 .lead-img { display: block }
124 p.dropcap:first-letter {
125 float: left; text-transform: uppercase; font-weight: bold; font-size: 5.55em; line-height: 0.83;
126 margin: 0; padding-right: 7px; margin-bottom: -2px; text-align: center;
127 }
128 '''
129
130 def get_browser(self):
131 br = BasicNewsRecipe.get_browser(self)
132 br.set_cookie('inEuropeanUnion', '0', '.theatlantic.com')
133 return br
134
135 def preprocess_raw_html(self, raw_html, url):
136 try:
137 return extract_html(self.index_to_soup(raw_html))
138 except NoJSON:
139 self.log.warn('No JSON found in: {} falling back to HTML'.format(url))
140 except Exception:
141 self.log.exception('Failed to extract JSON data from: {} falling back to HTML'.format(url))
142 return raw_html
143
144 def preprocess_html(self, soup):
145 for img in soup.findAll('img', attrs={'data-srcset': True}):
146 # img['src'] = img['data-srcset'].split()[0]
147 data_srcset = img['data-srcset']
148 if ',' in data_srcset:
149 img['src'] = data_srcset.split(',')[0]
150 else:
151 img['src'] = data_srcset.split()[0]
152 for img in soup.findAll('img', attrs={'data-src': True}):
153 img['src'] = img['data-src']
154 return soup
155
156 def print_version(self, url):
157 ans = url.partition('?')[0] + '?single_page=true'
158 if '/video/' in ans:
159 ans = None
160 return ans
161
162 if web_version and not test_article:
163 use_embedded_content = False
164
165 feeds = [
166 ('The Atlantic', 'https://www.theatlantic.com/feed/all/'),
167 ('Best of The Atlantic', 'https://www.theatlantic.com/feed/best-of/'),
168 ('Politics | The Atlantic', 'https://www.theatlantic.com/feed/channel/politics/'),
169 ('Business | The Atlantic', 'https://www.theatlantic.com/feed/channel/business/'),
170 ('Culture | The Atlantic', 'https://www.theatlantic.com/feed/channel/entertainment/'),
171 ('Global | The Atlantic', 'https://www.theatlantic.com/feed/channel/international/'),
172 ('Technology | The Atlantic', 'https://www.theatlantic.com/feed/channel/technology/'),
173 ('U.S. | The Atlantic', 'https://www.theatlantic.com/feed/channel/national/'),
174 ('Health | The Atlantic', 'https://www.theatlantic.com/feed/channel/health/'),
175 ('Video | The Atlantic', 'https://www.theatlantic.com/feed/channel/video/'),
176 ('Sexes | The Atlantic', 'https://www.theatlantic.com/feed/channel/sexes/'),
177 ('Education | The Atlantic', 'https://www.theatlantic.com/feed/channel/education/'),
178 ('Science | The Atlantic', 'https://www.theatlantic.com/feed/channel/science/'),
179 ('News | The Atlantic', 'https://www.theatlantic.com/feed/channel/news/'),
180 ('Press Releases | The Atlantic', 'https://www.theatlantic.com/feed/channel/press-releases/'),
181 ('Newsletters | The Atlantic', 'https://www.theatlantic.com/feed/channel/newsletters/'),
182 ('The Atlantic Photo', 'https://feeds.feedburner.com/theatlantic/infocus'),
183 ('Notes | The Atlantic', 'https://feeds.feedburner.com/TheAtlanticNotes'),
184 ]
185 else:
186
187 def parse_index(self):
188 if test_article:
189 return [('Articles', [{'title': 'Test article', 'url': test_article}])]
190 soup = self.index_to_soup(self.INDEX)
191 img = soup.find(**prefix_classes('IssueDescription_cover__'))
192 if img is not None:
193 self.cover_url = img['src']
194 current_section, current_articles = 'Cover Story', []
195 feeds = []
196 for x in soup.findAll(**prefix_classes('TocFeaturedSection_heading__ TocSection_heading__ TocHeroGridItem_hedLink___ TocGridItem_hedLink__')):
197 cls = x['class']
198 if not isinstance(cls, str):
199 cls = ' '.join(cls)
200 title = self.tag_to_string(x).strip()
201 if 'Section' in cls:
202 if current_articles:
203 feeds.append((current_section, current_articles))
204 current_section, current_articles = title, []
205 self.log(current_section)
206 continue
207 url = x['href']
208 current_articles.append({'title': title, 'url': url})
209 self.log('\t', title, url)
210 if current_articles:
211 feeds.append((current_section, current_articles))
212 return feeds
213
214
215 if __name__ == '__main__':
216 import sys
217
218 from calibre.ebooks.BeautifulSoup import BeautifulSoup
219
220 print(extract_html(BeautifulSoup(open(sys.argv[-1]).read())))