| 1 |
#!/usr/bin/env python |
| 2 |
# vim:fileencoding=utf-8 |
| 3 |
import json |
| 4 |
from urllib.parse import quote, urlparse |
| 5 |
|
| 6 |
from mechanize import Request |
| 7 |
|
| 8 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 9 |
|
| 10 |
|
| 11 |
def absurl(x): |
| 12 |
if x.startswith('//'): |
| 13 |
x = 'https:' + x |
| 14 |
elif not x.startswith('http'): |
| 15 |
x = 'https://caravanmagazine.in' + x |
| 16 |
return x |
| 17 |
|
| 18 |
|
| 19 |
def safe_dict(data, *names): |
| 20 |
ans = data |
| 21 |
for x in names: |
| 22 |
ans = ans.get(x) or '' |
| 23 |
return ans |
| 24 |
|
| 25 |
|
| 26 |
def parse_body(x): |
| 27 |
if x.get('type', '') == 'paragraph': |
| 28 |
yield '<p>' |
| 29 |
for p in x.get('content', {}): |
| 30 |
yield ''.join(parse_p(p)) |
| 31 |
yield '</p>\n' |
| 32 |
elif x.get('type', '') in {'blockquote', 'pullquote'}: |
| 33 |
yield '<blockquote>' |
| 34 |
for p in x.get('content', {}): |
| 35 |
yield from parse_body(p) |
| 36 |
yield '</blockquote>' |
| 37 |
elif x.get('type', '') == 'figure': |
| 38 |
yield '<img src="{}">'.format(absurl(x['attrs']['src'].replace('=s0', '=s768-rw'))) |
| 39 |
for p in x.get('content', {}): |
| 40 |
yield from parse_body(p) |
| 41 |
elif x.get('type', '') in {'caption', 'credit'}: |
| 42 |
yield '<div class="sub">' |
| 43 |
for div in x.get('content', {}): |
| 44 |
yield ''.join(parse_p(div)) |
| 45 |
yield '</div>\n' |
| 46 |
elif x.get('type', '') != '': |
| 47 |
if 'content' in x: |
| 48 |
yield '<p>' |
| 49 |
for p in x.get('content', {}): |
| 50 |
yield from parse_body(p) |
| 51 |
yield '</p>' |
| 52 |
|
| 53 |
|
| 54 |
def parse_p(p): |
| 55 |
if p.get('type', '') == 'text': |
| 56 |
if 'marks' in p: |
| 57 |
tag = p['marks'][0]['type'] |
| 58 |
yield '<' + tag + '>' |
| 59 |
yield p['text'] |
| 60 |
yield '</' + tag + '>' |
| 61 |
else: |
| 62 |
yield p['text'] |
| 63 |
elif p.get('type', '') == 'hard_break': |
| 64 |
yield '<br>' |
| 65 |
|
| 66 |
|
| 67 |
class CaravanMagazine(BasicNewsRecipe): |
| 68 |
title = 'Caravan Magazine' |
| 69 |
__author__ = 'Kovid Goyal, Gobelinus, unkn0wn' |
| 70 |
description = ( |
| 71 |
'The Caravan has established itself as one of the country’s most respected and intellectually agile publications, ' |
| 72 |
'setting new benchmarks for the Indian and South Asian media. We publish immersive reportage, daring commentary, ' |
| 73 |
'path-breaking investigations, insightful literary criticism and more, spanning the worlds of politics, culture, ' |
| 74 |
'business, society, media, the environment and the arts.' |
| 75 |
) |
| 76 |
language = 'en_IN' |
| 77 |
timefmt = ' [%b, %Y]' |
| 78 |
encoding = 'utf-8' |
| 79 |
|
| 80 |
no_stylesheets = True |
| 81 |
|
| 82 |
remove_attributes = ['style', 'height', 'width'] |
| 83 |
ignore_duplicate_articles = {'url'} |
| 84 |
resolve_internal_links = True |
| 85 |
needs_subscription = 'optional' |
| 86 |
logged = False |
| 87 |
|
| 88 |
extra_css = ''' |
| 89 |
img {display:block; margin:0 auto;} |
| 90 |
blockquote, em {color:#202020;} |
| 91 |
.desc {font-style:italic; color:#202020;} |
| 92 |
.sub {text-align:center; font-size:small;} |
| 93 |
.cat, .auth {font-size:small; color:#404040;} |
| 94 |
''' |
| 95 |
|
| 96 |
def get_browser(self, *args, **kw): |
| 97 |
br = BasicNewsRecipe.get_browser(self, *args, **kw) |
| 98 |
if not self.username or not self.password: |
| 99 |
return br |
| 100 |
data = json.dumps({'0': {'json': {'email': self.username, 'password': self.password}}}) |
| 101 |
if not isinstance(data, bytes): |
| 102 |
data = data.encode('utf-8') |
| 103 |
rq = Request( |
| 104 |
url='https://caravanmagazine.in/api/trpc/users.login?batch=1', |
| 105 |
data=data, |
| 106 |
headers={ |
| 107 |
'Accept': 'application/json, text/plain, */*', |
| 108 |
'Origin': 'https://caravanmagazine.in', |
| 109 |
'Referer': 'https://caravanmagazine.in/', |
| 110 |
'Content-type': 'application/json;charset=UTF-8', |
| 111 |
}, |
| 112 |
method='POST', |
| 113 |
) |
| 114 |
try: |
| 115 |
res = br.open(rq).read() |
| 116 |
res = res.decode('utf-8') |
| 117 |
res = json.loads(res) |
| 118 |
self.log(safe_dict(res[0], 'result', 'data', 'json', 'message')) |
| 119 |
self.logged = True |
| 120 |
except Exception: |
| 121 |
self.log.warn('\n**Login failed, check your username and password\n') |
| 122 |
return br |
| 123 |
return br |
| 124 |
|
| 125 |
recipe_specific_options = {'date': {'short': 'The date of the edition to download (MM-YYYY format)', 'long': 'For example, 07-2024'}} |
| 126 |
|
| 127 |
def parse_index(self): |
| 128 |
self.log('\n***\nif this recipe fails, report it on: https://www.mobileread.com/forums/forumdisplay.php?f=228\n***\n') |
| 129 |
|
| 130 |
api = 'https://api.caravanmagazine.in/api/trpc/magazines.getLatestIssue' |
| 131 |
d = self.recipe_specific_options.get('date') |
| 132 |
if d and isinstance(d, str): |
| 133 |
x = d.split('-') |
| 134 |
inp = json.dumps({'0': {'json': {'month': int(x[0]), 'year': int(x[1])}}}) |
| 135 |
api = 'https://api.caravanmagazine.in/api/trpc/magazines.getForMonthAndYear?batch=1&input=' + quote(inp, safe='') |
| 136 |
|
| 137 |
raw = json.loads(self.index_to_soup(api, raw=True)) |
| 138 |
if isinstance(raw, list): |
| 139 |
data = raw[0]['result']['data']['json'] |
| 140 |
else: |
| 141 |
data = raw['result']['data']['json'] |
| 142 |
cover = safe_dict(data, 'issue', 'cover', 'data', 'url').replace('=s0', '=s768-rw') |
| 143 |
self.cover_url = absurl(cover) |
| 144 |
|
| 145 |
feeds = [] |
| 146 |
|
| 147 |
for sec in data['categories']: |
| 148 |
section = sec['name'] |
| 149 |
self.log(section) |
| 150 |
articles = [] |
| 151 |
for arts in sec['amc']: |
| 152 |
title = safe_dict(arts, 'article', 'title') |
| 153 |
desc = safe_dict(arts, 'article', 'theme', 'name') + ' | ' + safe_dict(arts, 'article', 'printTitle') |
| 154 |
names = [] |
| 155 |
for auth in arts['article']['authors']: |
| 156 |
name = safe_dict(auth, 'profile', 'name') |
| 157 |
if name != '': |
| 158 |
names.append(name) |
| 159 |
if names: |
| 160 |
desc = desc + ' | ' + ', '.join(names) |
| 161 |
url = absurl(arts['article']['slug']) |
| 162 |
self.log('\t', title, url, '\n\t', desc) |
| 163 |
articles.append({'title': title, 'description': desc, 'url': url}) |
| 164 |
if articles: |
| 165 |
feeds.append((section, articles)) |
| 166 |
return feeds |
| 167 |
|
| 168 |
def print_version(self, url): |
| 169 |
slug = urlparse(url).path |
| 170 |
inp = json.dumps({'0': {'json': {'slug': slug}}}) |
| 171 |
return 'https://api.caravanmagazine.in/api/trpc/articles.getFromCache?batch=1&input=' + quote(inp, safe='') |
| 172 |
|
| 173 |
def preprocess_raw_html(self, raw, url): |
| 174 |
cache_data = json.loads(raw)[0] |
| 175 |
art_id = cache_data['result']['data']['json']['articleId'] |
| 176 |
prim_data = cache_data['result']['data']['json']['data'] |
| 177 |
|
| 178 |
cat = desc = lede = auth = '' |
| 179 |
|
| 180 |
cat = '<div class="cat">' + safe_dict(prim_data, 'printTitle') + '</div>\n' |
| 181 |
title = '<h1>' + safe_dict(prim_data, 'title') + '</h1>\n' |
| 182 |
desc = '<p class="desc">' + safe_dict(prim_data, 'description') + '</p>\n' |
| 183 |
|
| 184 |
authors = [] |
| 185 |
for q in prim_data.get('authors', {}): |
| 186 |
authors.append(safe_dict(q, 'name')) |
| 187 |
dt = '' |
| 188 |
if prim_data.get('writtenAt', '') != '': |
| 189 |
import time |
| 190 |
from datetime import datetime, timedelta |
| 191 |
|
| 192 |
dt = datetime.fromisoformat(prim_data['writtenAt'][:-1]) + timedelta(seconds=time.timezone) |
| 193 |
dt = dt.strftime('%b %d, %Y, %I:%M %p') |
| 194 |
auth = '<p class="auth">' + ', '.join(authors) + ' | ' + dt + '</p>\n' |
| 195 |
lede = ''.join(parse_body(prim_data.get('cover', {}))) |
| 196 |
|
| 197 |
free_cont = '' |
| 198 |
for x in prim_data['data']['content']: |
| 199 |
free_cont += '\n' + ''.join(parse_body(x)) |
| 200 |
|
| 201 |
premium_cont = '' |
| 202 |
if self.logged: |
| 203 |
cont_url = 'https://api.caravanmagazine.in/api/paywall/check-article?articleId=' |
| 204 |
art_cont = json.loads(self.index_to_soup(cont_url + str(art_id), raw=True)) |
| 205 |
for x in art_cont['premiumContent']: |
| 206 |
premium_cont += '\n' + ''.join(parse_body(x)) |
| 207 |
|
| 208 |
return '<html><body><div>' + cat + title + desc + auth + lede + free_cont + premium_cont + '</div></body></html>' |