| 1 |
#!/usr/bin/env python |
| 2 |
# vim:fileencoding=utf-8 |
| 3 |
import json |
| 4 |
import re |
| 5 |
from datetime import date, datetime, timedelta |
| 6 |
from urllib.parse import quote |
| 7 |
|
| 8 |
from calibre.utils.date import parse_date |
| 9 |
from calibre.web.feeds.news import BasicNewsRecipe, classes |
| 10 |
|
| 11 |
|
| 12 |
class eenadu_ts(BasicNewsRecipe): |
| 13 |
title = 'ఈనాడు - తెలంగాణ' |
| 14 |
__author__ = 'unkn0wn' |
| 15 |
description = 'THE LARGEST CIRCULATED TELUGU DAILY' |
| 16 |
language = 'te' |
| 17 |
encoding = 'utf-8' |
| 18 |
no_stylesheets = True |
| 19 |
remove_javascript = True |
| 20 |
masthead_url = 'https://dxxd96tbpm203.cloudfront.net//img/logo.png' |
| 21 |
remove_attributes = ['style', 'height', 'width'] |
| 22 |
ignore_duplicate_articles = {'url', 'title'} |
| 23 |
|
| 24 |
extra_css = ''' |
| 25 |
img {display:block; margin:0 auto;} |
| 26 |
blockquote, em {color:#202020;} |
| 27 |
.pub-t{font-size:small; font-style:italic;} |
| 28 |
''' |
| 29 |
|
| 30 |
keep_only_tags = [classes('bookWrapper fullstory')] |
| 31 |
remove_tags = [classes('ext-link offset-tb1 sshare-c')] |
| 32 |
|
| 33 |
resolve_internal_links = True |
| 34 |
remove_empty_feeds = True |
| 35 |
|
| 36 |
def get_browser(self, *args, **kw): |
| 37 |
br = BasicNewsRecipe.get_browser(self, *args, **kw) |
| 38 |
br.addheaders += [('Referer', 'https://www.eenadu.net/')] |
| 39 |
return br |
| 40 |
|
| 41 |
def get_cover_url(self): |
| 42 |
today = quote(date.today().strftime('%d/%m/%Y'), safe='') |
| 43 |
raw = self.index_to_soup('https://epaper.eenadu.net/Home/GetAllpages?editionid=1&editiondate=' + today, raw=True) |
| 44 |
for cov in json.loads(raw): |
| 45 |
if cov['NewsProPageTitle'].lower().startswith('front'): |
| 46 |
return cov['HighResolution'] |
| 47 |
|
| 48 |
def parse_index(self): |
| 49 |
inx = 'https://www.eenadu.net/' |
| 50 |
section_list = [ |
| 51 |
('తెలంగాణ', 'telangana'), |
| 52 |
('పాలిటిక్స్', 'politics'), |
| 53 |
('జాతీయం', 'india'), |
| 54 |
('సంపాదకీయం', 'editorial'), |
| 55 |
('బిజినెస్', 'business'), |
| 56 |
# ('క్రైమ్', 'crime'), |
| 57 |
('అంతర్జాతీయం', 'world'), |
| 58 |
('క్రీడలు', 'sports'), |
| 59 |
('సినిమా', 'movies'), |
| 60 |
# ('వసుంధర', 'women'), |
| 61 |
('హైదరాబాద్ జిల్లా వార్తలు', 'telangana/districts/hyderabad'), |
| 62 |
] |
| 63 |
|
| 64 |
feeds = [] |
| 65 |
|
| 66 |
# For each section title, fetch the article urls |
| 67 |
for section in section_list: |
| 68 |
section_title = section[0] |
| 69 |
section_url = section[1] |
| 70 |
self.log(section_title) |
| 71 |
soup = self.index_to_soup(inx + section_url) |
| 72 |
articles = self.articles_from_soup(soup) |
| 73 |
if articles: |
| 74 |
feeds.append((section_title, articles)) |
| 75 |
return feeds |
| 76 |
|
| 77 |
def articles_from_soup(self, soup): |
| 78 |
div = soup.find('div', attrs={'class': ['col-left', 'district-more']}) |
| 79 |
ans = [] |
| 80 |
for link in div.findAll(attrs={'class': ['telugu_uni_body', 'thumb-content-more', 'owl-carousel']}): |
| 81 |
for a in link.findAll('a', attrs={'href': True}): |
| 82 |
url = a['href'] |
| 83 |
if not url.startswith('http'): |
| 84 |
url = 'https://www.eenadu.net/' + url |
| 85 |
h = a.find(['h4', 'h3', 'h2', 'h1']) |
| 86 |
if h: |
| 87 |
title = self.tag_to_string(h).strip() |
| 88 |
if 'telugu-news' not in url: |
| 89 |
continue |
| 90 |
self.log('\t', title, '\n\t\t', url) |
| 91 |
ans.append({'title': title, 'url': url}) |
| 92 |
return ans |
| 93 |
|
| 94 |
def populate_article_metadata(self, article, soup, first): |
| 95 |
desc = soup.find(attrs={'class': 'srtdes'}) |
| 96 |
if desc: |
| 97 |
article.summary = self.tag_to_string(desc) |
| 98 |
article.text_summary = article.summary |
| 99 |
|
| 100 |
def preprocess_raw_html(self, raw, *a): |
| 101 |
if '<!--Top Full Story Start -->' in raw: |
| 102 |
body = re.search(r'<!--Top Full Story Start -->([^~]+?)<!--Tags Start -->', raw) |
| 103 |
return '<html><body><div>' + body.group(1) + '</div></body></html>' |
| 104 |
return raw |
| 105 |
|
| 106 |
def preprocess_html(self, soup): |
| 107 |
div = soup.find(**classes('pub-t')) |
| 108 |
if div: |
| 109 |
dt = re.search(r'\d+.+\d+', self.tag_to_string(div)) |
| 110 |
date = parse_date(dt.group(0) + ':00.000000').replace(tzinfo=None) |
| 111 |
if (datetime.now() - date) > timedelta(1.5): |
| 112 |
self.abort_article('Skipping old article') |
| 113 |
else: |
| 114 |
self.abort_article('may not be an artilce') |
| 115 |
return soup |