Debian Codegraph @

recipes/caravan_magazine.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 import json
4 from urllib.parse import quote, urlparse
5
6 from mechanize import Request
7
8 from calibre.web.feeds.news import BasicNewsRecipe
9
10
11 def absurl(x):
12 if x.startswith('//'):
13 x = 'https:' + x
14 elif not x.startswith('http'):
15 x = 'https://caravanmagazine.in' + x
16 return x
17
18
19 def safe_dict(data, *names):
20 ans = data
21 for x in names:
22 ans = ans.get(x) or ''
23 return ans
24
25
26 def parse_body(x):
27 if x.get('type', '') == 'paragraph':
28 yield '<p>'
29 for p in x.get('content', {}):
30 yield ''.join(parse_p(p))
31 yield '</p>\n'
32 elif x.get('type', '') in {'blockquote', 'pullquote'}:
33 yield '<blockquote>'
34 for p in x.get('content', {}):
35 yield from parse_body(p)
36 yield '</blockquote>'
37 elif x.get('type', '') == 'figure':
38 yield '<img src="{}">'.format(absurl(x['attrs']['src'].replace('=s0', '=s768-rw')))
39 for p in x.get('content', {}):
40 yield from parse_body(p)
41 elif x.get('type', '') in {'caption', 'credit'}:
42 yield '<div class="sub">'
43 for div in x.get('content', {}):
44 yield ''.join(parse_p(div))
45 yield '</div>\n'
46 elif x.get('type', '') != '':
47 if 'content' in x:
48 yield '<p>'
49 for p in x.get('content', {}):
50 yield from parse_body(p)
51 yield '</p>'
52
53
54 def parse_p(p):
55 if p.get('type', '') == 'text':
56 if 'marks' in p:
57 tag = p['marks'][0]['type']
58 yield '<' + tag + '>'
59 yield p['text']
60 yield '</' + tag + '>'
61 else:
62 yield p['text']
63 elif p.get('type', '') == 'hard_break':
64 yield '<br>'
65
66
67 class CaravanMagazine(BasicNewsRecipe):
68 title = 'Caravan Magazine'
69 __author__ = 'Kovid Goyal, Gobelinus, unkn0wn'
70 description = (
71 'The Caravan has established itself as one of the country’s most respected and intellectually agile publications, '
72 'setting new benchmarks for the Indian and South Asian media. We publish immersive reportage, daring commentary, '
73 'path-breaking investigations, insightful literary criticism and more, spanning the worlds of politics, culture, '
74 'business, society, media, the environment and the arts.'
75 )
76 language = 'en_IN'
77 timefmt = ' [%b, %Y]'
78 encoding = 'utf-8'
79
80 no_stylesheets = True
81
82 remove_attributes = ['style', 'height', 'width']
83 ignore_duplicate_articles = {'url'}
84 resolve_internal_links = True
85 needs_subscription = 'optional'
86 logged = False
87
88 extra_css = '''
89 img {display:block; margin:0 auto;}
90 blockquote, em {color:#202020;}
91 .desc {font-style:italic; color:#202020;}
92 .sub {text-align:center; font-size:small;}
93 .cat, .auth {font-size:small; color:#404040;}
94 '''
95
96 def get_browser(self, *args, **kw):
97 br = BasicNewsRecipe.get_browser(self, *args, **kw)
98 if not self.username or not self.password:
99 return br
100 data = json.dumps({'0': {'json': {'email': self.username, 'password': self.password}}})
101 if not isinstance(data, bytes):
102 data = data.encode('utf-8')
103 rq = Request(
104 url='https://caravanmagazine.in/api/trpc/users.login?batch=1',
105 data=data,
106 headers={
107 'Accept': 'application/json, text/plain, */*',
108 'Origin': 'https://caravanmagazine.in',
109 'Referer': 'https://caravanmagazine.in/',
110 'Content-type': 'application/json;charset=UTF-8',
111 },
112 method='POST',
113 )
114 try:
115 res = br.open(rq).read()
116 res = res.decode('utf-8')
117 res = json.loads(res)
118 self.log(safe_dict(res[0], 'result', 'data', 'json', 'message'))
119 self.logged = True
120 except Exception:
121 self.log.warn('\n**Login failed, check your username and password\n')
122 return br
123 return br
124
125 recipe_specific_options = {'date': {'short': 'The date of the edition to download (MM-YYYY format)', 'long': 'For example, 07-2024'}}
126
127 def parse_index(self):
128 self.log('\n***\nif this recipe fails, report it on: https://www.mobileread.com/forums/forumdisplay.php?f=228\n***\n')
129
130 api = 'https://api.caravanmagazine.in/api/trpc/magazines.getLatestIssue'
131 d = self.recipe_specific_options.get('date')
132 if d and isinstance(d, str):
133 x = d.split('-')
134 inp = json.dumps({'0': {'json': {'month': int(x[0]), 'year': int(x[1])}}})
135 api = 'https://api.caravanmagazine.in/api/trpc/magazines.getForMonthAndYear?batch=1&input=' + quote(inp, safe='')
136
137 raw = json.loads(self.index_to_soup(api, raw=True))
138 if isinstance(raw, list):
139 data = raw[0]['result']['data']['json']
140 else:
141 data = raw['result']['data']['json']
142 cover = safe_dict(data, 'issue', 'cover', 'data', 'url').replace('=s0', '=s768-rw')
143 self.cover_url = absurl(cover)
144
145 feeds = []
146
147 for sec in data['categories']:
148 section = sec['name']
149 self.log(section)
150 articles = []
151 for arts in sec['amc']:
152 title = safe_dict(arts, 'article', 'title')
153 desc = safe_dict(arts, 'article', 'theme', 'name') + ' | ' + safe_dict(arts, 'article', 'printTitle')
154 names = []
155 for auth in arts['article']['authors']:
156 name = safe_dict(auth, 'profile', 'name')
157 if name != '':
158 names.append(name)
159 if names:
160 desc = desc + ' | ' + ', '.join(names)
161 url = absurl(arts['article']['slug'])
162 self.log('\t', title, url, '\n\t', desc)
163 articles.append({'title': title, 'description': desc, 'url': url})
164 if articles:
165 feeds.append((section, articles))
166 return feeds
167
168 def print_version(self, url):
169 slug = urlparse(url).path
170 inp = json.dumps({'0': {'json': {'slug': slug}}})
171 return 'https://api.caravanmagazine.in/api/trpc/articles.getFromCache?batch=1&input=' + quote(inp, safe='')
172
173 def preprocess_raw_html(self, raw, url):
174 cache_data = json.loads(raw)[0]
175 art_id = cache_data['result']['data']['json']['articleId']
176 prim_data = cache_data['result']['data']['json']['data']
177
178 cat = desc = lede = auth = ''
179
180 cat = '<div class="cat">' + safe_dict(prim_data, 'printTitle') + '</div>\n'
181 title = '<h1>' + safe_dict(prim_data, 'title') + '</h1>\n'
182 desc = '<p class="desc">' + safe_dict(prim_data, 'description') + '</p>\n'
183
184 authors = []
185 for q in prim_data.get('authors', {}):
186 authors.append(safe_dict(q, 'name'))
187 dt = ''
188 if prim_data.get('writtenAt', '') != '':
189 import time
190 from datetime import datetime, timedelta
191
192 dt = datetime.fromisoformat(prim_data['writtenAt'][:-1]) + timedelta(seconds=time.timezone)
193 dt = dt.strftime('%b %d, %Y, %I:%M %p')
194 auth = '<p class="auth">' + ', '.join(authors) + ' | ' + dt + '</p>\n'
195 lede = ''.join(parse_body(prim_data.get('cover', {})))
196
197 free_cont = ''
198 for x in prim_data['data']['content']:
199 free_cont += '\n' + ''.join(parse_body(x))
200
201 premium_cont = ''
202 if self.logged:
203 cont_url = 'https://api.caravanmagazine.in/api/paywall/check-article?articleId='
204 art_cont = json.loads(self.index_to_soup(cont_url + str(art_id), raw=True))
205 for x in art_cont['premiumContent']:
206 premium_cont += '\n' + ''.join(parse_body(x))
207
208 return '<html><body><div>' + cat + title + desc + auth + lede + free_cont + premium_cont + '</div></body></html>'