| 1 |
#!/usr/bin/env python |
| 2 |
import json |
| 3 |
import re |
| 4 |
|
| 5 |
import mechanize |
| 6 |
|
| 7 |
from calibre.web.feeds.news import BasicNewsRecipe, classes |
| 8 |
|
| 9 |
|
| 10 |
def as_article(source, log): |
| 11 |
url = source['url'] |
| 12 |
title = source['title'] |
| 13 |
desc = '' |
| 14 |
if source.get('field_subtitle'): |
| 15 |
desc += source['field_subtitle'] |
| 16 |
if source.get('field_display_authors'): |
| 17 |
desc += ' by ' + source['field_display_authors'] |
| 18 |
log(title, url) |
| 19 |
return {'url': url, 'title': title, 'description': desc} |
| 20 |
|
| 21 |
|
| 22 |
def get_issue_data(br, log, node_id='1126213', year='2020', volnum='99', issue_vol='5'): |
| 23 |
headers = { |
| 24 |
'Accept': 'application/json, text/plain, */*', |
| 25 |
'Content-Type': 'application/json;charset=UTF-8', |
| 26 |
'Origin': 'https://www.foreignaffairs.com', |
| 27 |
'Referer': 'https://www.foreignaffairs.com', |
| 28 |
} |
| 29 |
|
| 30 |
def make_query(**kwds): |
| 31 |
size = kwds.pop('size', 1) |
| 32 |
is_filter = kwds.pop('filter', None) |
| 33 |
if is_filter: |
| 34 |
q = {'filter': [{'terms': {k: v}} for k, v in kwds.items()]} |
| 35 |
else: |
| 36 |
q = {'must': [{'term': {k: v}} for k, v in kwds.items()]} |
| 37 |
return { |
| 38 |
'from': 0, |
| 39 |
'post_filter': {'bool': q}, |
| 40 |
'_source': { |
| 41 |
'includes': [ |
| 42 |
'nid', |
| 43 |
'path', |
| 44 |
'title', |
| 45 |
'field_subtitle', |
| 46 |
'field_display_authors', |
| 47 |
'fa_node_type_or_subtype', |
| 48 |
'field_issue_sspecial_articles__nid', |
| 49 |
'field_issue_sspecial_header', |
| 50 |
] |
| 51 |
}, |
| 52 |
'query': {'match_all': {}}, |
| 53 |
'sort': [{'field_sequence': 'asc'}, {'fa_normalized_date': 'desc'}], |
| 54 |
'size': size, |
| 55 |
} |
| 56 |
|
| 57 |
def get_data(data): |
| 58 |
search_url = 'https://www.foreignaffairs.com/fa-search.php' |
| 59 |
req = mechanize.Request(url=search_url, data=json.dumps(data), headers=headers, method='POST') |
| 60 |
res = br.open(req) |
| 61 |
data = json.loads(res.read()) |
| 62 |
return data['hits']['hits'] |
| 63 |
|
| 64 |
feeds = [] |
| 65 |
|
| 66 |
def as_article(source): |
| 67 |
title = source['title'][0] |
| 68 |
desc = '' |
| 69 |
fs = source.get('field_subtitle') |
| 70 |
if fs: |
| 71 |
desc = fs[0] |
| 72 |
aus = source.get('field_display_authors') |
| 73 |
if aus: |
| 74 |
desc += ' By ' + aus[0] |
| 75 |
url = 'https://www.foreignaffairs.com' + source['path'][0] |
| 76 |
return {'title': title, 'description': desc, 'url': url} |
| 77 |
|
| 78 |
issue_data = get_data(make_query(fa_node_type_or_subtype='Issue', field_issue_volume=issue_vol, field_issue_year=year, field_issue_volume_number=volnum))[ |
| 79 |
0 |
| 80 |
]['_source'] |
| 81 |
|
| 82 |
if 'field_issue_sspecial_articles__nid' in issue_data: |
| 83 |
main_sec_title = issue_data['field_issue_sspecial_header'][0] |
| 84 |
main_sec_nids = issue_data['field_issue_sspecial_articles__nid'] |
| 85 |
articles_data = get_data(make_query(nid=main_sec_nids, filter=True, size=len(main_sec_nids))) |
| 86 |
articles = [] |
| 87 |
log(main_sec_title) |
| 88 |
for entry in articles_data: |
| 89 |
source = entry['_source'] |
| 90 |
articles.append(as_article(source)) |
| 91 |
log('\t', articles[-1]['title'], articles[-1]['url']) |
| 92 |
feeds.append((main_sec_title, articles)) |
| 93 |
|
| 94 |
articles_data = get_data(make_query(field_issue__nid=node_id, size=50)) |
| 95 |
ans = {} |
| 96 |
for entry in articles_data: |
| 97 |
source = entry['_source'] |
| 98 |
section = source['fa_node_type_or_subtype'][0] |
| 99 |
ans.setdefault(section, []).append(as_article(source)) |
| 100 |
for sectitle in sorted(ans): |
| 101 |
articles = ans[sectitle] |
| 102 |
log(sectitle) |
| 103 |
if articles: |
| 104 |
for art in articles: |
| 105 |
log('\t', art['title'], art['url']) |
| 106 |
feeds.append((sectitle, articles)) |
| 107 |
|
| 108 |
return feeds |
| 109 |
|
| 110 |
|
| 111 |
class ForeignAffairsRecipe(BasicNewsRecipe): |
| 112 |
title = 'Foreign Affairs' |
| 113 |
__author__ = 'Kovid Goyal' |
| 114 |
language = 'en' |
| 115 |
publisher = 'Council on Foreign Relations' |
| 116 |
category = 'USA, Foreign Affairs' |
| 117 |
description = 'The leading forum for serious discussion of American foreign policy and international affairs.' |
| 118 |
encoding = 'utf-8' |
| 119 |
no_stylesheets = True |
| 120 |
remove_javascript = True |
| 121 |
needs_subscription = 'optional' |
| 122 |
remove_attributes = ['style', 'height', 'width'] |
| 123 |
masthead_url = 'https://upload.wikimedia.org/wikipedia/commons/thumb/9/9a/Foreign_Affairs_Logo.svg/500px-Foreign_Affairs_Logo.svg.png' |
| 124 |
extra_css = ''' |
| 125 |
.topper__date, .topper__byline, .figure__caption, .calibre-nuked-tag-figcaption, .font-style-italic { font-size: small; } |
| 126 |
.topper__subtitle { font-style: italic; color: #202020; } |
| 127 |
em, blockquote { color: #202020; } |
| 128 |
img {display:block; margin:0 auto;} |
| 129 |
''' |
| 130 |
|
| 131 |
INDEX = 'https://www.foreignaffairs.com/magazine' |
| 132 |
|
| 133 |
recipe_specific_options = {'issue': {'short': 'Enter the Issue Number you want to download ', 'long': 'For example, 2024/103/1'}} |
| 134 |
|
| 135 |
keep_only_tags = [ |
| 136 |
classes('topper__heading-container topper__image-container paywall-content'), |
| 137 |
] |
| 138 |
|
| 139 |
remove_tags = [dict(name=['svg', 'meta']), classes('topper__issue article-newsletter-signup--container dfp-tag-wrapper')] |
| 140 |
|
| 141 |
conversion_options = {'comments': description, 'tags': category, 'language': 'en', 'publisher': publisher} |
| 142 |
ignore_duplicate_articles = {'title', 'url'} |
| 143 |
remove_empty_feeds = True |
| 144 |
|
| 145 |
def parse_index(self): |
| 146 |
d = self.recipe_specific_options.get('issue') |
| 147 |
if d and isinstance(d, str): |
| 148 |
self.INDEX = 'https://www.foreignaffairs.com/issues/' + d |
| 149 |
|
| 150 |
soup = self.index_to_soup(self.INDEX) |
| 151 |
# get dates |
| 152 |
date = re.split(r'\s\|\s', self.tag_to_string(soup.head.title.string))[0] |
| 153 |
self.timefmt = ' [%s]' % date |
| 154 |
link = soup.find('link', rel='canonical', href=True)['href'] |
| 155 |
year, volnum, issue_vol = link.split('/')[-3:] |
| 156 |
main = soup.find('main', attrs={'id': 'content'}) |
| 157 |
cov = main.find('img', attrs={'srcset': lambda x: x and 'Cover.jpg' in x}) |
| 158 |
if cov: |
| 159 |
self.cover_url = re.sub(r'_webp_issue_small_\dx', '_webp_issue_large_2x', cov['srcset'].split()[0]) |
| 160 |
|
| 161 |
cls = soup.find('link', attrs={'rel': 'shortlink'})['href'] |
| 162 |
node_id = re.search(r'https://www.foreignaffairs.com/node/(\d+)', cls).group(1) |
| 163 |
br = self.cloned_browser |
| 164 |
feeds = get_issue_data(br, self.log, node_id, year, volnum, issue_vol) |
| 165 |
return feeds |
| 166 |
|
| 167 |
def preprocess_html(self, soup): |
| 168 |
for h2 in soup.findAll(**classes('topper__subtitle')): |
| 169 |
h2.name = 'p' |
| 170 |
for by in soup.findAll(**classes('topper__byline topper__date font-style-italic')): |
| 171 |
by.name = 'div' |
| 172 |
for img in soup.find_all('img', attrs={'srcset': True}): |
| 173 |
img['src'] = re.sub(r'_webp_small_\dx', '_webp_large_1x', img['srcset'].split()[0]) |
| 174 |
return soup |
| 175 |
|
| 176 |
def get_browser(self): |
| 177 |
|
| 178 |
def select_form(form): |
| 179 |
return form.attrs.get('id', None) == 'fa-user-login-form' |
| 180 |
|
| 181 |
br = BasicNewsRecipe.get_browser(self) |
| 182 |
if self.username is not None and self.password is not None: |
| 183 |
br.open('https://www.foreignaffairs.com/user/login') |
| 184 |
br.select_form(predicate=select_form) |
| 185 |
br.form['name'] = self.username |
| 186 |
br.form['pass'] = self.password |
| 187 |
br.submit() |
| 188 |
return br |