| 1 |
#!/usr/bin/env python |
| 2 |
# vim:fileencoding=utf-8 |
| 3 |
import re |
| 4 |
from datetime import datetime, timedelta |
| 5 |
from urllib.parse import quote |
| 6 |
|
| 7 |
from calibre.utils.date import parse_date |
| 8 |
from calibre.web.feeds.news import BasicNewsRecipe, classes |
| 9 |
|
| 10 |
|
| 11 |
class eenadu_ap(BasicNewsRecipe): |
| 12 |
title = 'ఈనాడు - ఆంధ్రప్రదేశ్' |
| 13 |
__author__ = 'unkn0wn' |
| 14 |
description = 'THE LARGEST CIRCULATED TELUGU DAILY' |
| 15 |
language = 'te' |
| 16 |
encoding = 'utf-8' |
| 17 |
no_stylesheets = True |
| 18 |
remove_javascript = True |
| 19 |
masthead_url = 'https://dxxd96tbpm203.cloudfront.net//img/logo.png' |
| 20 |
remove_attributes = ['style', 'height', 'width'] |
| 21 |
ignore_duplicate_articles = {'url', 'title'} |
| 22 |
|
| 23 |
extra_css = ''' |
| 24 |
img {display:block; margin:0 auto;} |
| 25 |
blockquote, em {color:#202020;} |
| 26 |
.pub-t{font-size:small; font-style:italic;} |
| 27 |
''' |
| 28 |
|
| 29 |
keep_only_tags = [classes('bookWrapper fullstory')] |
| 30 |
remove_tags = [classes('ext-link offset-tb1 sshare-c')] |
| 31 |
|
| 32 |
resolve_internal_links = True |
| 33 |
remove_empty_feeds = True |
| 34 |
|
| 35 |
def get_browser(self, *args, **kw): |
| 36 |
br = BasicNewsRecipe.get_browser(self, *args, **kw) |
| 37 |
br.addheaders += [('Referer', 'https://www.eenadu.net/')] |
| 38 |
return br |
| 39 |
|
| 40 |
def get_cover_url(self): |
| 41 |
import json |
| 42 |
from datetime import date |
| 43 |
|
| 44 |
today = quote(date.today().strftime('%d/%m/%Y'), safe='') |
| 45 |
raw = self.index_to_soup('https://epaper.eenadu.net/Home/GetAllpages?editionid=2&editiondate=' + today, raw=True) |
| 46 |
for cov in json.loads(raw): |
| 47 |
if cov['NewsProPageTitle'].lower().startswith('front'): |
| 48 |
return cov['HighResolution'] |
| 49 |
|
| 50 |
def parse_index(self): |
| 51 |
inx = 'https://www.eenadu.net/' |
| 52 |
section_list = [ |
| 53 |
('ఆంధ్రప్రదేశ్', 'andhra-pradesh'), |
| 54 |
('పాలిటిక్స్', 'politics'), |
| 55 |
('జాతీయం', 'india'), |
| 56 |
('సంపాదకీయం', 'editorial'), |
| 57 |
('బిజినెస్', 'business'), |
| 58 |
# ('క్రైమ్', 'crime'), |
| 59 |
('అంతర్జాతీయం', 'world'), |
| 60 |
('క్రీడలు', 'sports'), |
| 61 |
('సినిమా', 'movies'), |
| 62 |
# ('వసుంధర', 'women'), |
| 63 |
('విశాఖపట్నం జిల్లా వార్తలు', 'andhra-pradesh/districts/visakhapatnam'), |
| 64 |
] |
| 65 |
|
| 66 |
feeds = [] |
| 67 |
|
| 68 |
# For each section title, fetch the article urls |
| 69 |
for section in section_list: |
| 70 |
section_title = section[0] |
| 71 |
section_url = section[1] |
| 72 |
self.log(section_title) |
| 73 |
soup = self.index_to_soup(inx + section_url) |
| 74 |
articles = self.articles_from_soup(soup) |
| 75 |
if articles: |
| 76 |
feeds.append((section_title, articles)) |
| 77 |
return feeds |
| 78 |
|
| 79 |
def articles_from_soup(self, soup): |
| 80 |
div = soup.find('div', attrs={'class': ['col-left', 'district-more']}) |
| 81 |
ans = [] |
| 82 |
for link in div.findAll(attrs={'class': ['telugu_uni_body', 'thumb-content-more', 'owl-carousel']}): |
| 83 |
for a in link.findAll('a', attrs={'href': True}): |
| 84 |
url = a['href'] |
| 85 |
if not url.startswith('http'): |
| 86 |
url = 'https://www.eenadu.net/' + url |
| 87 |
h = a.find(['h4', 'h3', 'h2', 'h1']) |
| 88 |
if h: |
| 89 |
title = self.tag_to_string(h).strip() |
| 90 |
if 'telugu-news' not in url: |
| 91 |
continue |
| 92 |
self.log('\t', title, '\n\t\t', url) |
| 93 |
ans.append({'title': title, 'url': url}) |
| 94 |
return ans |
| 95 |
|
| 96 |
def populate_article_metadata(self, article, soup, first): |
| 97 |
desc = soup.find(attrs={'class': 'srtdes'}) |
| 98 |
if desc: |
| 99 |
article.summary = self.tag_to_string(desc) |
| 100 |
article.text_summary = article.summary |
| 101 |
|
| 102 |
def preprocess_raw_html(self, raw, *a): |
| 103 |
if '<!--Top Full Story Start -->' in raw: |
| 104 |
body = re.search(r'<!--Top Full Story Start -->([^~]+?)<!--Tags Start -->', raw) |
| 105 |
return '<html><body><div>' + body.group(1) + '</div></body></html>' |
| 106 |
return raw |
| 107 |
|
| 108 |
def preprocess_html(self, soup): |
| 109 |
div = soup.find(**classes('pub-t')) |
| 110 |
if div: |
| 111 |
dt = re.search(r'\d+.+\d+', self.tag_to_string(div)) |
| 112 |
date = parse_date(dt.group(0) + ':00.000000').replace(tzinfo=None) |
| 113 |
if (datetime.now() - date) > timedelta(1.5): |
| 114 |
self.abort_article('Skipping old article') |
| 115 |
else: |
| 116 |
self.abort_article('may not be an artilce') |
| 117 |
return soup |