Debian Codegraph @

recipes/hindu.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 import json
4 import re
5 from collections import defaultdict
6 from datetime import date
7
8 from calibre.web.feeds.news import BasicNewsRecipe, classes
9
10
11 def absurl(url):
12 if url.startswith('/'):
13 url = 'https://www.thehindu.com' + url
14 return url
15
16
17 class TheHindu(BasicNewsRecipe):
18 title = 'The Hindu Print Edition'
19 __author__ = 'unkn0wn'
20 description = "Articles from The Hindu, Today's Paper."
21 language = 'en_IN'
22 no_stylesheets = True
23 masthead_url = 'https://www.thehindu.com/theme/images/th-online/thehindu-logo.svg'
24 remove_attributes = ['style', 'height', 'width']
25
26 extra_css = '''
27 .caption {font-size:small; text-align:center;}
28 .author, .dateLine {font-size:small;}
29 .subhead, .subhead_lead, .bold {font-weight:bold;}
30 img {display:block; margin:0 auto;}
31 .italic, .sub-title {font-style:italic; color:#202020;}
32 '''
33
34 recipe_specific_options = {
35 'location': {
36 'short': 'The name of the local edition',
37 'long': (
38 'If The Hindu is available in your local town/city, '
39 'set this to your location, for example, hyderabad\n'
40 'Available Editions: bengaluru, chennai, coimbatore, delhi, '
41 'erode, hyderabad, international, kochi, kolkata,\n'
42 'kozhikode, madurai, mangalore, mumbai, thiruvananthapuram, '
43 'tiruchirapalli, vijayawada, visakhapatnam'
44 ),
45 'default': 'chennai',
46 },
47 'date': {'short': 'The date of the edition to download (YYYY-MM-DD format)', 'long': 'For example, 2023-01-28'},
48 }
49
50 ignore_duplicate_articles = {'url'}
51
52 keep_only_tags = [classes('article-section')]
53
54 remove_tags = [classes('hide-mobile comments-shares share-page editiondetails')]
55
56 def preprocess_html(self, soup):
57 for cap in soup.findAll('p', attrs={'class': 'caption'}):
58 cap.name = 'figcaption'
59 for img in soup.findAll('img', attrs={'data-original': True}):
60 img['src'] = img['data-original']
61 for h3 in soup.findAll(**classes('sub-title')):
62 h3.name = 'p'
63 return soup
64
65 def parse_index(self):
66 self.title = 'The Hindu'
67 local_edition = 'th_chennai'
68 d = self.recipe_specific_options.get('location')
69 if d and isinstance(d, str):
70 local_edition = 'th_' + d
71
72 past_edition = self.recipe_specific_options.get('date')
73
74 dt = date.today()
75 if past_edition and isinstance(past_edition, str):
76 year, month, day = (int(x) for x in past_edition.split('-'))
77 dt = date(year, month, day)
78
79 today = dt.strftime('%Y-%m-%d')
80
81 self.log('Downloading The Hindu, ' + local_edition[3:] + ' edition, ' + today)
82 url = absurl('/todays-paper/' + today + '/' + local_edition + '/')
83 if local_edition == 'th_chennai' and today == date.today().strftime('%Y-%m-%d'):
84 url = absurl('/todays-paper/')
85
86 mag_url = None
87 if dt.weekday() == 0:
88 mag_url = url + '?supplement=' + local_edition + '-epbs'
89 if dt.weekday() == 4:
90 mag_url = url + '?supplement=' + local_edition + '-fr'
91 if dt.weekday() == 5:
92 mag_url = url + '?supplement=' + local_edition + '-mp'
93 if dt.weekday() == 6:
94 mag_url = url + '?supplement=' + local_edition + '-sm'
95
96 raw = self.index_to_soup(url, raw=True)
97 soup = self.index_to_soup(raw)
98 ans = self.hindu_parse_index(soup)
99 if not ans:
100 raise ValueError('The Hindu Newspaper is not published Today.')
101 cover = soup.find(attrs={'class': 'hindu-ad'})
102 if cover:
103 self.cover_url = cover.img['src']
104 if mag_url:
105 self.log('\nFetching Magazine')
106 soup = self.index_to_soup(mag_url)
107 ans2 = self.hindu_parse_index(soup)
108 if ans2:
109 return ans + ans2
110 self.log('\nMagazine not Found')
111 return ans
112 return ans
113
114 def hindu_parse_index(self, soup):
115 for script in soup.findAll('script'):
116 if not self.tag_to_string(script).__contains__('grouped_articles = {"'):
117 continue
118 if script is not None:
119 art = re.search(r'grouped_articles = ({\".*)', self.tag_to_string(script))
120 data = json.JSONDecoder().raw_decode(art.group(1))[0]
121
122 feeds_dict = defaultdict(list)
123
124 a = json.dumps(data)
125 for sec in json.loads(a):
126 for item in data[sec]:
127 section = sec.replace('TH_', '')
128 title = item['articleheadline']
129 url = absurl(item['href'])
130 desc = 'Page no.' + item['pageno'] + ' | ' + item['teaser_text'] or ''
131 self.log(' ', title, '\n\t', url)
132 feeds_dict[section].append({'title': title, 'url': url, 'description': desc})
133 return list(feeds_dict.items())
134 else:
135 return []