Debian Codegraph @

recipes/atlantic.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 # License: GPLv3 Copyright: 2015, Kovid Goyal <kovid at kovidgoyal.net>
4 import json
5 from xml.sax.saxutils import escape, quoteattr
6
7 from calibre.utils.iso8601 import parse_iso8601
8 from calibre.web.feeds.news import BasicNewsRecipe, classes
9 from calibre.web.feeds.news import prefixed_classes as prefix_classes
10
11 web_version = False
12 test_article = None
13 # test_article = 'https://www.theatlantic.com/health/archive/2020/12/covid-19-second-surge/617415/?utm_source=feed'
14
15
16 # parse article JSON {{{
17 def process_image_block(lines, block):
18 caption = block.get('captionText')
19 caption_lines = []
20 if caption:
21 if block.get('attributionText', '').strip():
22 caption += ' (' + block['attributionText'] + ')'
23 caption_lines.append('<p style="font-style: italic">' + caption + '</p>')
24 lines.append('<div style="text-align: center"><img src={}/>'.format(quoteattr(block['url'])))
25 lines.extend(caption_lines)
26 lines.append('</div>')
27
28
29 def json_to_html(raw):
30 data = json.loads(raw)
31 # open('/t/p.json', 'w').write(json.dumps(data, indent=2))
32 data = max((v['data'] for v in data['props']['pageProps']['urqlState'].values()), key=len)
33 article = json.loads(data)['article']
34 # open('/t/p.json', 'w').write(json.dumps(article, indent=2))
35 lines = []
36 lines.append('<h1 style="align: center">' + escape(article['title']) + '</h1>')
37 lines.append('<h2 style="align: center">' + escape(article['dek']) + '</h2>')
38 auts = ', '.join(x['displayName'] for x in article['authors'])
39 if auts:
40 lines.append('<p style="align: center">by ' + escape(auts) + '</p>')
41 dp = article['datePublished']
42 dt = parse_iso8601(dp)
43 lines.append('<p style="align: center">' + escape(dt.strftime('%a, %d %b %Y')) + '</p>')
44 if article.get('leadArt') and 'image' in article['leadArt']:
45 process_image_block(lines, article['leadArt']['image'])
46 for item in article['content']:
47 tn = item.get('__typename', '')
48 if tn.endswith('Image'):
49 process_image_block(lines, item)
50 continue
51 html = item.get('innerHtml')
52 if html is None or '</iframe>' in html:
53 continue
54 if 'innerHtml' not in item:
55 continue
56 tagname = item.get('tagName', 'P').lower()
57 lines.append('<{0}>{1}</{0}>'.format(tagname, html))
58 return '<html><body><div id="from-json-by-calibre">' + '\n'.join(lines) + '</div></body></html>'
59
60
61 class NoJSON(ValueError):
62 pass
63
64
65 def extract_html(soup):
66 script = soup.findAll('script', id='__NEXT_DATA__')
67 if not script:
68 raise NoJSON('No script tag with JSON data found')
69 raw = script[0].contents[0]
70 return json_to_html(raw)
71
72
73 # }}}
74
75
76 class TheAtlantic(BasicNewsRecipe):
77 if web_version:
78 title = 'TheAtlantic.com'
79 description = 'News and editorial about politics, culture, entertainment, tech, etc. Contains many articles not seen in The Atlantic magazine'
80 else:
81 title = 'The Atlantic'
82 description = 'Current affairs and politics focused on the US'
83 INDEX = 'https://www.theatlantic.com/magazine/'
84
85 __author__ = 'Kovid Goyal'
86 language = 'en'
87 encoding = 'utf-8'
88
89 recipe_specific_options = {'date': {'short': 'The date of the edition to download (YYYY/MM format)', 'long': 'For example, 2024/05'}}
90
91 def __init__(self, *args, **kwargs):
92 BasicNewsRecipe.__init__(self, *args, **kwargs)
93 d = self.recipe_specific_options.get('date')
94 if d and isinstance(d, str):
95 self.INDEX = 'https://www.theatlantic.com/magazine/toc/' + d + '/'
96
97 keep_only_tags = [
98 dict(itemprop=['headline']),
99 classes(
100 'c-article-header__hed c-rubric article-header c-article-meta c-lead-media'
101 ' lead-img article-cover-extra article-body article-magazine article-cover-content'
102 ),
103 prefix_classes('ArticleHeader_root__ ArticleLayoutSection_main__ ArticleHero_timestamp__ ArticleBody_root__'),
104 dict(itemprop='articleBody'),
105 # these are for photos articles
106 dict(id=['article-header', 'from-json-by-calibre']),
107 classes('photos'),
108 ]
109 remove_tags = [
110 classes(
111 'c-ad c-share-social c-recirculation-link social-kit-top letter-writer-info callout secondary-byline embed-wrapper'
112 ' offset-wrapper boxtop-most-popular social-icons hints read-more c-article-writer__social'
113 ),
114 prefix_classes('ArticleRecirc_inline__'),
115 {'name': ['meta', 'link', 'noscript', 'aside', 'h3']},
116 {'attrs': {'class': ['offset-wrapper', 'boxtop-most-popular']}},
117 {'attrs': {'class': lambda x: x and 'article-tools' in x}},
118 {'src': lambda x: x and 'spotxchange.com' in x},
119 ]
120 remove_tags_after = classes('article-body')
121
122 no_stylesheets = True
123 remove_attributes = ['style']
124 extra_css = '''
125 .credit { text-align: right; font-size: 75%; display: block }
126 .figcaption { font-size: 75% }
127 .caption { font-size: 75% }
128 .lead-img { display: block }
129 p.dropcap:first-letter {
130 float: left; text-transform: uppercase; font-weight: bold; font-size: 5.55em; line-height: 0.83;
131 margin: 0; padding-right: 7px; margin-bottom: -2px; text-align: center;
132 }
133 '''
134
135 def get_browser(self):
136 br = BasicNewsRecipe.get_browser(self)
137 br.set_cookie('inEuropeanUnion', '0', '.theatlantic.com')
138 return br
139
140 def preprocess_raw_html(self, raw_html, url):
141 try:
142 return extract_html(self.index_to_soup(raw_html))
143 except NoJSON:
144 self.log.warn('No JSON found in: {} falling back to HTML'.format(url))
145 except Exception:
146 self.log.exception('Failed to extract JSON data from: {} falling back to HTML'.format(url))
147 return raw_html
148
149 def preprocess_html(self, soup):
150 for img in soup.findAll('img', attrs={'data-srcset': True}):
151 # img['src'] = img['data-srcset'].split()[0]
152 data_srcset = img['data-srcset']
153 if ',' in data_srcset:
154 img['src'] = data_srcset.split(',')[0]
155 else:
156 img['src'] = data_srcset.split()[0]
157 for img in soup.findAll('img', attrs={'data-src': True}):
158 img['src'] = img['data-src']
159 return soup
160
161 def print_version(self, url):
162 ans = url.partition('?')[0] + '?single_page=true'
163 if '/video/' in ans:
164 ans = None
165 return ans
166
167 if web_version and not test_article:
168 use_embedded_content = False
169
170 feeds = [
171 ('The Atlantic', 'https://www.theatlantic.com/feed/all/'),
172 ('Best of The Atlantic', 'https://www.theatlantic.com/feed/best-of/'),
173 ('Politics | The Atlantic', 'https://www.theatlantic.com/feed/channel/politics/'),
174 ('Business | The Atlantic', 'https://www.theatlantic.com/feed/channel/business/'),
175 ('Culture | The Atlantic', 'https://www.theatlantic.com/feed/channel/entertainment/'),
176 ('Global | The Atlantic', 'https://www.theatlantic.com/feed/channel/international/'),
177 ('Technology | The Atlantic', 'https://www.theatlantic.com/feed/channel/technology/'),
178 ('U.S. | The Atlantic', 'https://www.theatlantic.com/feed/channel/national/'),
179 ('Health | The Atlantic', 'https://www.theatlantic.com/feed/channel/health/'),
180 ('Video | The Atlantic', 'https://www.theatlantic.com/feed/channel/video/'),
181 ('Sexes | The Atlantic', 'https://www.theatlantic.com/feed/channel/sexes/'),
182 ('Education | The Atlantic', 'https://www.theatlantic.com/feed/channel/education/'),
183 ('Science | The Atlantic', 'https://www.theatlantic.com/feed/channel/science/'),
184 ('News | The Atlantic', 'https://www.theatlantic.com/feed/channel/news/'),
185 ('Press Releases | The Atlantic', 'https://www.theatlantic.com/feed/channel/press-releases/'),
186 ('Newsletters | The Atlantic', 'https://www.theatlantic.com/feed/channel/newsletters/'),
187 ('The Atlantic Photo', 'https://feeds.feedburner.com/theatlantic/infocus'),
188 ('Notes | The Atlantic', 'https://feeds.feedburner.com/TheAtlanticNotes'),
189 ]
190 else:
191
192 def parse_index(self):
193 if test_article:
194 return [('Articles', [{'title': 'Test article', 'url': test_article}])]
195 soup = self.index_to_soup(self.INDEX)
196 img = soup.find(**prefix_classes('IssueDescription_cover__'))
197 if img is not None:
198 self.cover_url = img['src']
199 current_section, current_articles = 'Cover Story', []
200 feeds = []
201 for x in soup.findAll(
202 **prefix_classes('TocFeaturedSection_heading__ TocSection_heading__ TocHeroGridItem_hedLink__ TocGridItem_hedLink__ RiverGridItem_hedLink__')
203 ):
204 cls = x['class']
205 if not isinstance(cls, str):
206 cls = ' '.join(cls)
207 title = self.tag_to_string(x).strip()
208 if 'Section' in cls:
209 if current_articles:
210 feeds.append((current_section, current_articles))
211 current_section, current_articles = title, []
212 self.log(current_section)
213 continue
214 url = x['href']
215 current_articles.append({'title': title, 'url': url})
216 self.log('\t', title, url)
217 if current_articles:
218 feeds.append((current_section, current_articles))
219 return feeds
220
221
222 if __name__ == '__main__':
223 import sys
224
225 from calibre.ebooks.BeautifulSoup import BeautifulSoup
226
227 print(extract_html(BeautifulSoup(open(sys.argv[-1]).read())))