Debian Codegraph @

recipes/eenadu.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 import json
4 import re
5 from datetime import date, datetime, timedelta
6 from urllib.parse import quote
7
8 from calibre.utils.date import parse_date
9 from calibre.web.feeds.news import BasicNewsRecipe, classes
10
11
12 class eenadu_ts(BasicNewsRecipe):
13 title = 'ఈనాడు - తెలంగాణ'
14 __author__ = 'unkn0wn'
15 description = 'THE LARGEST CIRCULATED TELUGU DAILY'
16 language = 'te'
17 encoding = 'utf-8'
18 no_stylesheets = True
19 remove_javascript = True
20 masthead_url = 'https://dxxd96tbpm203.cloudfront.net//img/logo.png'
21 remove_attributes = ['style', 'height', 'width']
22 ignore_duplicate_articles = {'url', 'title'}
23
24 extra_css = '''
25 img {display:block; margin:0 auto;}
26 blockquote, em {color:#202020;}
27 .pub-t{font-size:small; font-style:italic;}
28 '''
29
30 keep_only_tags = [classes('bookWrapper fullstory')]
31 remove_tags = [classes('ext-link offset-tb1 sshare-c')]
32
33 resolve_internal_links = True
34 remove_empty_feeds = True
35
36 def get_browser(self, *args, **kw):
37 br = BasicNewsRecipe.get_browser(self, *args, **kw)
38 br.addheaders += [('Referer', 'https://www.eenadu.net/')]
39 return br
40
41 def get_cover_url(self):
42 today = quote(date.today().strftime('%d/%m/%Y'), safe='')
43 raw = self.index_to_soup('https://epaper.eenadu.net/Home/GetAllpages?editionid=1&editiondate=' + today, raw=True)
44 for cov in json.loads(raw):
45 if cov['NewsProPageTitle'].lower().startswith('front'):
46 return cov['HighResolution']
47
48 def parse_index(self):
49 inx = 'https://www.eenadu.net/'
50 section_list = [
51 ('తెలంగాణ', 'telangana'),
52 ('పాలిటిక్స్', 'politics'),
53 ('జాతీయం', 'india'),
54 ('సంపాదకీయం', 'editorial'),
55 ('బిజినెస్', 'business'),
56 # ('క్రైమ్', 'crime'),
57 ('అంతర్జాతీయం', 'world'),
58 ('క్రీడలు', 'sports'),
59 ('సినిమా', 'movies'),
60 # ('వసుంధర', 'women'),
61 ('హైదరాబాద్ జిల్లా వార్తలు', 'telangana/districts/hyderabad'),
62 ]
63
64 feeds = []
65
66 # For each section title, fetch the article urls
67 for section in section_list:
68 section_title = section[0]
69 section_url = section[1]
70 self.log(section_title)
71 soup = self.index_to_soup(inx + section_url)
72 articles = self.articles_from_soup(soup)
73 if articles:
74 feeds.append((section_title, articles))
75 return feeds
76
77 def articles_from_soup(self, soup):
78 div = soup.find('div', attrs={'class': ['col-left', 'district-more']})
79 ans = []
80 for link in div.findAll(attrs={'class': ['telugu_uni_body', 'thumb-content-more', 'owl-carousel']}):
81 for a in link.findAll('a', attrs={'href': True}):
82 url = a['href']
83 if not url.startswith('http'):
84 url = 'https://www.eenadu.net/' + url
85 h = a.find(['h4', 'h3', 'h2', 'h1'])
86 if h:
87 title = self.tag_to_string(h).strip()
88 if 'telugu-news' not in url:
89 continue
90 self.log('\t', title, '\n\t\t', url)
91 ans.append({'title': title, 'url': url})
92 return ans
93
94 def populate_article_metadata(self, article, soup, first):
95 desc = soup.find(attrs={'class': 'srtdes'})
96 if desc:
97 article.summary = self.tag_to_string(desc)
98 article.text_summary = article.summary
99
100 def preprocess_raw_html(self, raw, *a):
101 if '<!--Top Full Story Start -->' in raw:
102 body = re.search(r'<!--Top Full Story Start -->([^~]+?)<!--Tags Start -->', raw)
103 return '<html><body><div>' + body.group(1) + '</div></body></html>'
104 return raw
105
106 def preprocess_html(self, soup):
107 div = soup.find(**classes('pub-t'))
108 if div:
109 dt = re.search(r'\d+.+\d+', self.tag_to_string(div))
110 date = parse_date(dt.group(0) + ':00.000000').replace(tzinfo=None)
111 if (datetime.now() - date) > timedelta(1.5):
112 self.abort_article('Skipping old article')
113 else:
114 self.abort_article('may not be an artilce')
115 return soup