Debian Codegraph @

recipes/livemint.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 import json
4 import re
5 import urllib.parse
6 from datetime import datetime, timedelta
7
8 from calibre.web.feeds.news import BasicNewsRecipe, classes
9
10
11 class LiveMint(BasicNewsRecipe):
12 title = 'Live Mint'
13 description = 'Financial News from India.'
14 language = 'en_IN'
15 __author__ = 'Krittika Goyal, revised by unkn0wn'
16 oldest_article = 1.15 # days
17 max_articles_per_feed = 50
18 encoding = 'utf-8'
19 use_embedded_content = False
20 no_stylesheets = True
21 remove_attributes = ['style', 'height', 'width']
22 masthead_url = 'https://images.livemint.com/static/livemint-logo-v1.svg'
23
24 recipe_specific_options = {
25 'days': {
26 'short': 'Oldest article to download from this news source. In days ',
27 'long': 'For example, 0.5, gives you articles from the past 12 hours',
28 'default': str(oldest_article),
29 }
30 }
31 remove_empty_feeds = True
32 resolve_internal_links = True
33
34 def __init__(self, *args, **kwargs):
35 BasicNewsRecipe.__init__(self, *args, **kwargs)
36 d = self.recipe_specific_options.get('days')
37 if d and isinstance(d, str):
38 self.oldest_article = float(d)
39
40 def get_cover_url(self):
41 today = datetime.today().strftime('%d/%m/%Y')
42 if datetime.today().weekday() in (5, 6):
43 today = (datetime.today() - timedelta(days=datetime.today().weekday() - 4)).strftime('%d/%m/%Y')
44 soup = self.index_to_soup(
45 'https://epaper.livemint.com/Home/GetAllpages?editionid=1&editiondate=' + urllib.parse.quote(today, safe=''),
46 raw=True,
47 )
48 for x in json.loads(soup):
49 if 'Mint_Front_' in x['SectionName'] or 'Mint_Cover' in x['SectionName']:
50 return x['MrImageUrl']
51
52 extra_css = '''
53 img {margin:0 auto;}
54 .psTopLogoItem img, .ecologoStory { width:100; }
55 #img-cap {font-size:small; text-align:center;}
56 .summary, .highlights, .synopsis {
57 font-weight:normal !important; font-style:italic; color:#202020;
58 }
59 em, blockquote {color:#202020;}
60 .moreAbout, .articleInfo, .metaData, .psTopicsHeading, .topicsTag, .auth {font-size:small;}
61 '''
62
63 keep_only_tags = [
64 dict(
65 name='article',
66 attrs={'id': lambda x: x and x.startswith(('article_', 'box_'))},
67 ),
68 dict(attrs={'class': lambda x: x and x.startswith(('storyPage_storyBox__', 'storyBoxSchema'))}),
69 classes('contentSec'),
70 ]
71
72 remove_tags = [
73 dict(name=['meta', 'link', 'svg', 'button', 'iframe']),
74 dict(
75 attrs={
76 'class': lambda x: (
77 x
78 and x.startswith((
79 'storyPage_alsoRead__',
80 'storyPage_firstPublishDate__',
81 'storyPage_bcrumb__',
82 ))
83 )
84 }
85 ),
86 dict(attrs={'id': ['faqSection', 'seoText', 'ellipsisId', 'gift_redeemed_box ']}),
87 classes(
88 'trendingSimilarHeight moreNews mobAppDownload label msgError msgOk taboolaHeight gadgetSlider ninSec'
89 ' socialHolder imgbig disclamerText disqus-comment-count openinApp2 lastAdSlot bs_logo author-widget'
90 ' datePublish sepStory premiumSlider moreStory Joinus moreAbout milestone benefitText checkCibilBtn'
91 ' double_gift_box trade'
92 ),
93 ]
94
95 feeds = [
96 ('Companies', 'https://www.livemint.com/rss/companies'),
97 ('Opinion', 'https://www.livemint.com/rss/opinion'),
98 ('Money', 'https://www.livemint.com/rss/money'),
99 ('Economy', 'https://www.livemint.com/rss/economy'),
100 ('Politics', 'https://www.livemint.com/rss/politics'),
101 ('Science', 'https://www.livemint.com/rss/science'),
102 ('Industry', 'https://www.livemint.com/rss/industry'),
103 ('Education', 'https://www.livemint.com/rss/education'),
104 ('Sports', 'https://www.livemint.com/rss/sports'),
105 ('Technology', 'https://www.livemint.com/rss/technology'),
106 ('News', 'https://www.livemint.com/rss/news'),
107 ('Mutual Funds', 'https://www.livemint.com/rss/Mutual Funds'),
108 ('Markets', 'https://www.livemint.com/rss/markets'),
109 ('AI', 'https://www.livemint.com/rss/AI'),
110 ('Insurance', 'https://www.livemint.com/rss/insurance'),
111 ('Budget', 'https://www.livemint.com/rss/budget'),
112 ('Elections', 'https://www.livemint.com/rss/elections'),
113 ]
114
115 def preprocess_raw_html(self, raw, *a):
116 # remove empty p tags
117 raw = re.sub(
118 r'(<p>\s*)(<[^(\/|a|i|b|em|strong)])',
119 r'\g<2>',
120 re.sub(r'(<p>\s*&nbsp;\s*<\/p>)|(<p>\s*<\/p>)|(<p\s*\S+>&nbsp;\s*<\/p>)', '', raw),
121 )
122 if '<script>var wsjFlag=true;</script>' in raw:
123 m = re.search(r'type="application/ld\+json">[^<]+?"@type": "NewsArticle"', raw)
124 raw1 = raw[m.start() :]
125 raw1 = raw1.split('>', 1)[1].strip()
126 data = json.JSONDecoder().raw_decode(raw1)[0]
127 value = data['hasPart']['value']
128 body = data['articleBody'] + '</p> <p>' + re.sub(r'(([a-z]|[^A-Z])\.|\.”)([A-Z]|“[A-Z])', r'\1 <p> \3', value)
129 body = '<div class="FirstEle"> <p>' + body + '</p> </div>'
130 raw2 = re.sub(r'<div class="FirstEle">([^}]*)</div>', body, raw)
131 return raw2
132 return raw
133
134 def preprocess_html(self, soup):
135 auth = soup.find(attrs={'class': lambda x: x and x.startswith(('storyPage_authorInfo__', 'storyPage_authorSocial__'))})
136 if auth:
137 auth['class'] = 'auth'
138 summ = soup.find(attrs={'class': lambda x: x and x.startswith('storyPage_summary__')})
139 if summ:
140 summ['class'] = 'summary'
141 summ.name = 'p'
142 for strong in soup.findAll('strong'):
143 if strong.find('p'):
144 strong.name = 'div'
145 for embed in soup.findAll('div', attrs={'class': 'embed'}):
146 nos = embed.find('noscript')
147 if nos:
148 nos.name = 'span'
149 for span in soup.findAll('figcaption'):
150 span['id'] = 'img-cap'
151 for auth in soup.findAll('span', attrs={'class': lambda x: x and 'articleInfo' in x.split()}):
152 auth.name = 'div'
153 for img in soup.findAll('img', attrs={'data-src': True}):
154 img['src'] = img['data-src']
155 for span in soup.findAll('span', attrs={'class': 'exclusive'}):
156 span.extract()
157 for al in soup.findAll('a', attrs={'class': 'manualbacklink'}):
158 pa = al.findParent(['p', 'h2', 'h3', 'h4'])
159 if pa:
160 pa.extract()
161 wa = soup.find(**classes('autobacklink-topic'))
162 if wa:
163 p = wa.findParent('p')
164 if p:
165 p.extract()
166 return soup
167
168 def populate_article_metadata(self, article, soup, first):
169 article.title = article.title.replace('<span class="webrupee">₹</span>', '₹')