Debian Codegraph @

recipes/eenadu_ap.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 import re
4 from datetime import datetime, timedelta
5 from urllib.parse import quote
6
7 from calibre.utils.date import parse_date
8 from calibre.web.feeds.news import BasicNewsRecipe, classes
9
10
11 class eenadu_ap(BasicNewsRecipe):
12 title = 'ఈనాడు - ఆంధ్రప్రదేశ్'
13 __author__ = 'unkn0wn'
14 description = 'THE LARGEST CIRCULATED TELUGU DAILY'
15 language = 'te'
16 encoding = 'utf-8'
17 no_stylesheets = True
18 remove_javascript = True
19 masthead_url = 'https://dxxd96tbpm203.cloudfront.net//img/logo.png'
20 remove_attributes = ['style', 'height', 'width']
21 ignore_duplicate_articles = {'url', 'title'}
22
23 extra_css = '''
24 img {display:block; margin:0 auto;}
25 blockquote, em {color:#202020;}
26 .pub-t{font-size:small; font-style:italic;}
27 '''
28
29 keep_only_tags = [classes('bookWrapper fullstory')]
30 remove_tags = [classes('ext-link offset-tb1 sshare-c')]
31
32 resolve_internal_links = True
33 remove_empty_feeds = True
34
35 def get_browser(self, *args, **kw):
36 br = BasicNewsRecipe.get_browser(self, *args, **kw)
37 br.addheaders += [('Referer', 'https://www.eenadu.net/')]
38 return br
39
40 def get_cover_url(self):
41 import json
42 from datetime import date
43
44 today = quote(date.today().strftime('%d/%m/%Y'), safe='')
45 raw = self.index_to_soup('https://epaper.eenadu.net/Home/GetAllpages?editionid=2&editiondate=' + today, raw=True)
46 for cov in json.loads(raw):
47 if cov['NewsProPageTitle'].lower().startswith('front'):
48 return cov['HighResolution']
49
50 def parse_index(self):
51 inx = 'https://www.eenadu.net/'
52 section_list = [
53 ('ఆంధ్రప్రదేశ్', 'andhra-pradesh'),
54 ('పాలిటిక్స్', 'politics'),
55 ('జాతీయం', 'india'),
56 ('సంపాదకీయం', 'editorial'),
57 ('బిజినెస్', 'business'),
58 # ('క్రైమ్', 'crime'),
59 ('అంతర్జాతీయం', 'world'),
60 ('క్రీడలు', 'sports'),
61 ('సినిమా', 'movies'),
62 # ('వసుంధర', 'women'),
63 ('విశాఖపట్నం జిల్లా వార్తలు', 'andhra-pradesh/districts/visakhapatnam'),
64 ]
65
66 feeds = []
67
68 # For each section title, fetch the article urls
69 for section in section_list:
70 section_title = section[0]
71 section_url = section[1]
72 self.log(section_title)
73 soup = self.index_to_soup(inx + section_url)
74 articles = self.articles_from_soup(soup)
75 if articles:
76 feeds.append((section_title, articles))
77 return feeds
78
79 def articles_from_soup(self, soup):
80 div = soup.find('div', attrs={'class': ['col-left', 'district-more']})
81 ans = []
82 for link in div.findAll(attrs={'class': ['telugu_uni_body', 'thumb-content-more', 'owl-carousel']}):
83 for a in link.findAll('a', attrs={'href': True}):
84 url = a['href']
85 if not url.startswith('http'):
86 url = 'https://www.eenadu.net/' + url
87 h = a.find(['h4', 'h3', 'h2', 'h1'])
88 if h:
89 title = self.tag_to_string(h).strip()
90 if 'telugu-news' not in url:
91 continue
92 self.log('\t', title, '\n\t\t', url)
93 ans.append({'title': title, 'url': url})
94 return ans
95
96 def populate_article_metadata(self, article, soup, first):
97 desc = soup.find(attrs={'class': 'srtdes'})
98 if desc:
99 article.summary = self.tag_to_string(desc)
100 article.text_summary = article.summary
101
102 def preprocess_raw_html(self, raw, *a):
103 if '<!--Top Full Story Start -->' in raw:
104 body = re.search(r'<!--Top Full Story Start -->([^~]+?)<!--Tags Start -->', raw)
105 return '<html><body><div>' + body.group(1) + '</div></body></html>'
106 return raw
107
108 def preprocess_html(self, soup):
109 div = soup.find(**classes('pub-t'))
110 if div:
111 dt = re.search(r'\d+.+\d+', self.tag_to_string(div))
112 date = parse_date(dt.group(0) + ':00.000000').replace(tzinfo=None)
113 if (datetime.now() - date) > timedelta(1.5):
114 self.abort_article('Skipping old article')
115 else:
116 self.abort_article('may not be an artilce')
117 return soup