Debian Codegraph @

recipes/foreignaffairs.recipe

raw
1 #!/usr/bin/env python
2 import json
3 import re
4
5 import mechanize
6
7 from calibre.web.feeds.news import BasicNewsRecipe, classes
8
9
10 def as_article(source, log):
11 url = source['url']
12 title = source['title']
13 desc = ''
14 if source.get('field_subtitle'):
15 desc += source['field_subtitle']
16 if source.get('field_display_authors'):
17 desc += ' by ' + source['field_display_authors']
18 log(title, url)
19 return {'url': url, 'title': title, 'description': desc}
20
21
22 def get_issue_data(br, log, node_id='1126213', year='2020', volnum='99', issue_vol='5'):
23 headers = {
24 'Accept': 'application/json, text/plain, */*',
25 'Content-Type': 'application/json;charset=UTF-8',
26 'Origin': 'https://www.foreignaffairs.com',
27 'Referer': 'https://www.foreignaffairs.com',
28 }
29
30 def make_query(**kwds):
31 size = kwds.pop('size', 1)
32 is_filter = kwds.pop('filter', None)
33 if is_filter:
34 q = {'filter': [{'terms': {k: v}} for k, v in kwds.items()]}
35 else:
36 q = {'must': [{'term': {k: v}} for k, v in kwds.items()]}
37 return {
38 'from': 0,
39 'post_filter': {'bool': q},
40 '_source': {
41 'includes': [
42 'nid',
43 'path',
44 'title',
45 'field_subtitle',
46 'field_display_authors',
47 'fa_node_type_or_subtype',
48 'field_issue_sspecial_articles__nid',
49 'field_issue_sspecial_header',
50 ]
51 },
52 'query': {'match_all': {}},
53 'sort': [{'field_sequence': 'asc'}, {'fa_normalized_date': 'desc'}],
54 'size': size,
55 }
56
57 def get_data(data):
58 search_url = 'https://www.foreignaffairs.com/fa-search.php'
59 req = mechanize.Request(url=search_url, data=json.dumps(data), headers=headers, method='POST')
60 res = br.open(req)
61 data = json.loads(res.read())
62 return data['hits']['hits']
63
64 feeds = []
65
66 def as_article(source):
67 title = source['title'][0]
68 desc = ''
69 fs = source.get('field_subtitle')
70 if fs:
71 desc = fs[0]
72 aus = source.get('field_display_authors')
73 if aus:
74 desc += ' By ' + aus[0]
75 url = 'https://www.foreignaffairs.com' + source['path'][0]
76 return {'title': title, 'description': desc, 'url': url}
77
78 issue_data = get_data(make_query(fa_node_type_or_subtype='Issue', field_issue_volume=issue_vol, field_issue_year=year, field_issue_volume_number=volnum))[
79 0
80 ]['_source']
81
82 if 'field_issue_sspecial_articles__nid' in issue_data:
83 main_sec_title = issue_data['field_issue_sspecial_header'][0]
84 main_sec_nids = issue_data['field_issue_sspecial_articles__nid']
85 articles_data = get_data(make_query(nid=main_sec_nids, filter=True, size=len(main_sec_nids)))
86 articles = []
87 log(main_sec_title)
88 for entry in articles_data:
89 source = entry['_source']
90 articles.append(as_article(source))
91 log('\t', articles[-1]['title'], articles[-1]['url'])
92 feeds.append((main_sec_title, articles))
93
94 articles_data = get_data(make_query(field_issue__nid=node_id, size=50))
95 ans = {}
96 for entry in articles_data:
97 source = entry['_source']
98 section = source['fa_node_type_or_subtype'][0]
99 ans.setdefault(section, []).append(as_article(source))
100 for sectitle in sorted(ans):
101 articles = ans[sectitle]
102 log(sectitle)
103 if articles:
104 for art in articles:
105 log('\t', art['title'], art['url'])
106 feeds.append((sectitle, articles))
107
108 return feeds
109
110
111 class ForeignAffairsRecipe(BasicNewsRecipe):
112 title = 'Foreign Affairs'
113 __author__ = 'Kovid Goyal'
114 language = 'en'
115 publisher = 'Council on Foreign Relations'
116 category = 'USA, Foreign Affairs'
117 description = 'The leading forum for serious discussion of American foreign policy and international affairs.'
118 encoding = 'utf-8'
119 no_stylesheets = True
120 remove_javascript = True
121 needs_subscription = 'optional'
122 remove_attributes = ['style', 'height', 'width']
123 masthead_url = 'https://upload.wikimedia.org/wikipedia/commons/thumb/9/9a/Foreign_Affairs_Logo.svg/500px-Foreign_Affairs_Logo.svg.png'
124 extra_css = '''
125 .topper__date, .topper__byline, .figure__caption, .calibre-nuked-tag-figcaption, .font-style-italic { font-size: small; }
126 .topper__subtitle { font-style: italic; color: #202020; }
127 em, blockquote { color: #202020; }
128 img {display:block; margin:0 auto;}
129 '''
130
131 INDEX = 'https://www.foreignaffairs.com/magazine'
132
133 recipe_specific_options = {'issue': {'short': 'Enter the Issue Number you want to download ', 'long': 'For example, 2024/103/1'}}
134
135 keep_only_tags = [
136 classes('topper__heading-container topper__image-container paywall-content'),
137 ]
138
139 remove_tags = [dict(name=['svg', 'meta']), classes('topper__issue article-newsletter-signup--container dfp-tag-wrapper')]
140
141 conversion_options = {'comments': description, 'tags': category, 'language': 'en', 'publisher': publisher}
142 ignore_duplicate_articles = {'title', 'url'}
143 remove_empty_feeds = True
144
145 def parse_index(self):
146 d = self.recipe_specific_options.get('issue')
147 if d and isinstance(d, str):
148 self.INDEX = 'https://www.foreignaffairs.com/issues/' + d
149
150 soup = self.index_to_soup(self.INDEX)
151 # get dates
152 date = re.split(r'\s\|\s', self.tag_to_string(soup.head.title.string))[0]
153 self.timefmt = ' [%s]' % date
154 link = soup.find('link', rel='canonical', href=True)['href']
155 year, volnum, issue_vol = link.split('/')[-3:]
156 main = soup.find('main', attrs={'id': 'content'})
157 cov = main.find('img', attrs={'srcset': lambda x: x and 'Cover.jpg' in x})
158 if cov:
159 self.cover_url = re.sub(r'_webp_issue_small_\dx', '_webp_issue_large_2x', cov['srcset'].split()[0])
160
161 cls = soup.find('link', attrs={'rel': 'shortlink'})['href']
162 node_id = re.search(r'https://www.foreignaffairs.com/node/(\d+)', cls).group(1)
163 br = self.cloned_browser
164 feeds = get_issue_data(br, self.log, node_id, year, volnum, issue_vol)
165 return feeds
166
167 def preprocess_html(self, soup):
168 for h2 in soup.findAll(**classes('topper__subtitle')):
169 h2.name = 'p'
170 for by in soup.findAll(**classes('topper__byline topper__date font-style-italic')):
171 by.name = 'div'
172 for img in soup.find_all('img', attrs={'srcset': True}):
173 img['src'] = re.sub(r'_webp_small_\dx', '_webp_large_1x', img['srcset'].split()[0])
174 return soup
175
176 def get_browser(self):
177
178 def select_form(form):
179 return form.attrs.get('id', None) == 'fa-user-login-form'
180
181 br = BasicNewsRecipe.get_browser(self)
182 if self.username is not None and self.password is not None:
183 br.open('https://www.foreignaffairs.com/user/login')
184 br.select_form(predicate=select_form)
185 br.form['name'] = self.username
186 br.form['pass'] = self.password
187 br.submit()
188 return br