| 1 |
#!/usr/bin/env python |
| 2 |
# vim:fileencoding=utf-8 |
| 3 |
import json |
| 4 |
import re |
| 5 |
import urllib.parse |
| 6 |
from datetime import datetime, timedelta |
| 7 |
|
| 8 |
from calibre.web.feeds.news import BasicNewsRecipe, classes |
| 9 |
|
| 10 |
|
| 11 |
class LiveMint(BasicNewsRecipe): |
| 12 |
title = 'Live Mint' |
| 13 |
description = 'Financial News from India.' |
| 14 |
language = 'en_IN' |
| 15 |
__author__ = 'Krittika Goyal, revised by unkn0wn' |
| 16 |
oldest_article = 1.15 # days |
| 17 |
max_articles_per_feed = 50 |
| 18 |
encoding = 'utf-8' |
| 19 |
use_embedded_content = False |
| 20 |
no_stylesheets = True |
| 21 |
remove_attributes = ['style', 'height', 'width'] |
| 22 |
masthead_url = 'https://images.livemint.com/static/livemint-logo-v1.svg' |
| 23 |
|
| 24 |
recipe_specific_options = { |
| 25 |
'days': { |
| 26 |
'short': 'Oldest article to download from this news source. In days ', |
| 27 |
'long': 'For example, 0.5, gives you articles from the past 12 hours', |
| 28 |
'default': str(oldest_article), |
| 29 |
} |
| 30 |
} |
| 31 |
remove_empty_feeds = True |
| 32 |
resolve_internal_links = True |
| 33 |
|
| 34 |
def __init__(self, *args, **kwargs): |
| 35 |
BasicNewsRecipe.__init__(self, *args, **kwargs) |
| 36 |
d = self.recipe_specific_options.get('days') |
| 37 |
if d and isinstance(d, str): |
| 38 |
self.oldest_article = float(d) |
| 39 |
|
| 40 |
def get_cover_url(self): |
| 41 |
today = datetime.today().strftime('%d/%m/%Y') |
| 42 |
if datetime.today().weekday() in (5, 6): |
| 43 |
today = (datetime.today() - timedelta(days=datetime.today().weekday() - 4)).strftime('%d/%m/%Y') |
| 44 |
soup = self.index_to_soup( |
| 45 |
'https://epaper.livemint.com/Home/GetAllpages?editionid=1&editiondate=' + urllib.parse.quote(today, safe=''), |
| 46 |
raw=True, |
| 47 |
) |
| 48 |
for x in json.loads(soup): |
| 49 |
if 'Mint_Front_' in x['SectionName'] or 'Mint_Cover' in x['SectionName']: |
| 50 |
return x['MrImageUrl'] |
| 51 |
|
| 52 |
extra_css = ''' |
| 53 |
img {margin:0 auto;} |
| 54 |
.psTopLogoItem img, .ecologoStory { width:100; } |
| 55 |
#img-cap {font-size:small; text-align:center;} |
| 56 |
.summary, .highlights, .synopsis { |
| 57 |
font-weight:normal !important; font-style:italic; color:#202020; |
| 58 |
} |
| 59 |
em, blockquote {color:#202020;} |
| 60 |
.moreAbout, .articleInfo, .metaData, .psTopicsHeading, .topicsTag, .auth {font-size:small;} |
| 61 |
''' |
| 62 |
|
| 63 |
keep_only_tags = [ |
| 64 |
dict( |
| 65 |
name='article', |
| 66 |
attrs={'id': lambda x: x and x.startswith(('article_', 'box_'))}, |
| 67 |
), |
| 68 |
dict(attrs={'class': lambda x: x and x.startswith(('storyPage_storyBox__', 'storyBoxSchema'))}), |
| 69 |
classes('contentSec'), |
| 70 |
] |
| 71 |
|
| 72 |
remove_tags = [ |
| 73 |
dict(name=['meta', 'link', 'svg', 'button', 'iframe']), |
| 74 |
dict( |
| 75 |
attrs={ |
| 76 |
'class': lambda x: ( |
| 77 |
x |
| 78 |
and x.startswith(( |
| 79 |
'storyPage_alsoRead__', |
| 80 |
'storyPage_firstPublishDate__', |
| 81 |
'storyPage_bcrumb__', |
| 82 |
)) |
| 83 |
) |
| 84 |
} |
| 85 |
), |
| 86 |
dict(attrs={'id': ['faqSection', 'seoText', 'ellipsisId', 'gift_redeemed_box ']}), |
| 87 |
classes( |
| 88 |
'trendingSimilarHeight moreNews mobAppDownload label msgError msgOk taboolaHeight gadgetSlider ninSec' |
| 89 |
' socialHolder imgbig disclamerText disqus-comment-count openinApp2 lastAdSlot bs_logo author-widget' |
| 90 |
' datePublish sepStory premiumSlider moreStory Joinus moreAbout milestone benefitText checkCibilBtn' |
| 91 |
' double_gift_box trade' |
| 92 |
), |
| 93 |
] |
| 94 |
|
| 95 |
feeds = [ |
| 96 |
('Companies', 'https://www.livemint.com/rss/companies'), |
| 97 |
('Opinion', 'https://www.livemint.com/rss/opinion'), |
| 98 |
('Money', 'https://www.livemint.com/rss/money'), |
| 99 |
('Economy', 'https://www.livemint.com/rss/economy'), |
| 100 |
('Politics', 'https://www.livemint.com/rss/politics'), |
| 101 |
('Science', 'https://www.livemint.com/rss/science'), |
| 102 |
('Industry', 'https://www.livemint.com/rss/industry'), |
| 103 |
('Education', 'https://www.livemint.com/rss/education'), |
| 104 |
('Sports', 'https://www.livemint.com/rss/sports'), |
| 105 |
('Technology', 'https://www.livemint.com/rss/technology'), |
| 106 |
('News', 'https://www.livemint.com/rss/news'), |
| 107 |
('Mutual Funds', 'https://www.livemint.com/rss/Mutual Funds'), |
| 108 |
('Markets', 'https://www.livemint.com/rss/markets'), |
| 109 |
('AI', 'https://www.livemint.com/rss/AI'), |
| 110 |
('Insurance', 'https://www.livemint.com/rss/insurance'), |
| 111 |
('Budget', 'https://www.livemint.com/rss/budget'), |
| 112 |
('Elections', 'https://www.livemint.com/rss/elections'), |
| 113 |
] |
| 114 |
|
| 115 |
def preprocess_raw_html(self, raw, *a): |
| 116 |
# remove empty p tags |
| 117 |
raw = re.sub( |
| 118 |
r'(<p>\s*)(<[^(\/|a|i|b|em|strong)])', |
| 119 |
r'\g<2>', |
| 120 |
re.sub(r'(<p>\s* \s*<\/p>)|(<p>\s*<\/p>)|(<p\s*\S+> \s*<\/p>)', '', raw), |
| 121 |
) |
| 122 |
if '<script>var wsjFlag=true;</script>' in raw: |
| 123 |
m = re.search(r'type="application/ld\+json">[^<]+?"@type": "NewsArticle"', raw) |
| 124 |
raw1 = raw[m.start() :] |
| 125 |
raw1 = raw1.split('>', 1)[1].strip() |
| 126 |
data = json.JSONDecoder().raw_decode(raw1)[0] |
| 127 |
value = data['hasPart']['value'] |
| 128 |
body = data['articleBody'] + '</p> <p>' + re.sub(r'(([a-z]|[^A-Z])\.|\.”)([A-Z]|“[A-Z])', r'\1 <p> \3', value) |
| 129 |
body = '<div class="FirstEle"> <p>' + body + '</p> </div>' |
| 130 |
raw2 = re.sub(r'<div class="FirstEle">([^}]*)</div>', body, raw) |
| 131 |
return raw2 |
| 132 |
return raw |
| 133 |
|
| 134 |
def preprocess_html(self, soup): |
| 135 |
auth = soup.find(attrs={'class': lambda x: x and x.startswith(('storyPage_authorInfo__', 'storyPage_authorSocial__'))}) |
| 136 |
if auth: |
| 137 |
auth['class'] = 'auth' |
| 138 |
summ = soup.find(attrs={'class': lambda x: x and x.startswith('storyPage_summary__')}) |
| 139 |
if summ: |
| 140 |
summ['class'] = 'summary' |
| 141 |
summ.name = 'p' |
| 142 |
for strong in soup.findAll('strong'): |
| 143 |
if strong.find('p'): |
| 144 |
strong.name = 'div' |
| 145 |
for embed in soup.findAll('div', attrs={'class': 'embed'}): |
| 146 |
nos = embed.find('noscript') |
| 147 |
if nos: |
| 148 |
nos.name = 'span' |
| 149 |
for span in soup.findAll('figcaption'): |
| 150 |
span['id'] = 'img-cap' |
| 151 |
for auth in soup.findAll('span', attrs={'class': lambda x: x and 'articleInfo' in x.split()}): |
| 152 |
auth.name = 'div' |
| 153 |
for img in soup.findAll('img', attrs={'data-src': True}): |
| 154 |
img['src'] = img['data-src'] |
| 155 |
for span in soup.findAll('span', attrs={'class': 'exclusive'}): |
| 156 |
span.extract() |
| 157 |
for al in soup.findAll('a', attrs={'class': 'manualbacklink'}): |
| 158 |
pa = al.findParent(['p', 'h2', 'h3', 'h4']) |
| 159 |
if pa: |
| 160 |
pa.extract() |
| 161 |
wa = soup.find(**classes('autobacklink-topic')) |
| 162 |
if wa: |
| 163 |
p = wa.findParent('p') |
| 164 |
if p: |
| 165 |
p.extract() |
| 166 |
return soup |
| 167 |
|
| 168 |
def populate_article_metadata(self, article, soup, first): |
| 169 |
article.title = article.title.replace('<span class="webrupee">₹</span>', '₹') |