| 1 |
#!/usr/bin/env python |
| 2 |
import datetime |
| 3 |
import json |
| 4 |
|
| 5 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 6 |
|
| 7 |
|
| 8 |
def absurl(url): |
| 9 |
if not url.startswith('http'): |
| 10 |
return 'https://thewire.in/' + url |
| 11 |
return url |
| 12 |
|
| 13 |
|
| 14 |
def json_to_html(js): |
| 15 |
data = json.loads(js)['post-detail'][0] |
| 16 |
title = f'<h1>{data["post_title"]}</h1>' |
| 17 |
exp = auth = image = sec = '' |
| 18 |
sec = f'<div style="font-size: small;">{data["categories"][0]["slug"]}</div>' |
| 19 |
if data.get('post_excerpt'): |
| 20 |
exp = f'<p style="font-style: italic;">{data["post_excerpt"]}</p>' |
| 21 |
if data.get('post_author_name'): |
| 22 |
auth = f'<p style="font-size: small;">By {", ".join(x["author_name"] for x in data["post_author_name"])} | {data["post_date"]}</p>' |
| 23 |
if data.get('featured_image'): |
| 24 |
image_url = data['featured_image'][0] |
| 25 |
image = f'<div><img src="{image_url}"><div style="font-size:small;">{data.get("featured_image_caption", "")}</div></div>' |
| 26 |
return '<html><body>' + sec + title + exp + image + auth + data['post_content'] + '</body></html>' |
| 27 |
|
| 28 |
|
| 29 |
class TheWire(BasicNewsRecipe): |
| 30 |
title = 'The Wire' |
| 31 |
__author__ = 'unkn0wn' |
| 32 |
description = 'The Wire is an Indian nonprofit news and opinion website' |
| 33 |
language = 'en_IN' |
| 34 |
masthead_url = 'https://cdn.thewire.in/wp-content/uploads/thewire-app-images/wire-logo.svg' |
| 35 |
remove_javascript = True |
| 36 |
remove_attributes = ['height', 'width'] |
| 37 |
ignore_duplicate_articles = {'url'} |
| 38 |
resolve_internal_links = True |
| 39 |
remove_empty_feeds = True |
| 40 |
extra_css = '[id^="caption"] { font-size: small;}' |
| 41 |
|
| 42 |
def get_browser(self, *args, **kw): |
| 43 |
br = BasicNewsRecipe.get_browser(self, *args, **kw) |
| 44 |
br.addheaders += [('Referer', 'https://thewire.in/')] |
| 45 |
return br |
| 46 |
|
| 47 |
def parse_index(self): |
| 48 |
raw = self.index_to_soup('https://thewirehindi.com/home_data_2.json', raw=True) |
| 49 |
dmp = json.loads(raw) |
| 50 |
feeds = [] |
| 51 |
for k, v in dmp.items(): |
| 52 |
if not isinstance(v, dict): |
| 53 |
continue |
| 54 |
if k == 'videos': |
| 55 |
continue |
| 56 |
section = k.capitalize() |
| 57 |
self.log(section) |
| 58 |
articles = [] |
| 59 |
|
| 60 |
for a, b in v.items(): |
| 61 |
if not isinstance(b, dict): |
| 62 |
continue |
| 63 |
if not b.get('post_type', '') == 'post': |
| 64 |
continue |
| 65 |
now = datetime.now(datetime.timezone.utc) # Ensure 'now' is offset-aware |
| 66 |
post_date = datetime.strptime(b['post_date'], '%Y-%m-%d %H:%M:%S').replace(tzinfo=datetime.timezone.utc) # Make 'post_date' offset-aware |
| 67 |
if (now - post_date).days > self.oldest_article: |
| 68 |
continue |
| 69 |
title = b['post_title'] |
| 70 |
desc = b['post_excerpt'] |
| 71 |
slg = b['categories'][0]['slug'] + '/' + b['post_name'] |
| 72 |
url = absurl(slg) |
| 73 |
self.log('\t', title, '\n\t', desc, '\n\t\t', url) |
| 74 |
articles.append({'title': title, 'description': desc, 'url': url}) |
| 75 |
if articles: |
| 76 |
feeds.append((section, articles)) |
| 77 |
return feeds |
| 78 |
|
| 79 |
articles_are_obfuscated = True |
| 80 |
|
| 81 |
def get_obfuscated_article(self, url): |
| 82 |
raw_ = self.index_to_soup( |
| 83 |
'https://cms.thewire.in/wp-json/thewire/v2/posts/detail/' + url.rsplit('/')[-1], |
| 84 |
raw=True, |
| 85 |
) |
| 86 |
return {'data': json_to_html(raw_), 'url': url} |