Debian Codegraph @

recipes/history_today.recipe

raw
1 import re
2 from collections import OrderedDict
3
4 from calibre.web.feeds.news import BasicNewsRecipe
5
6
7 class HistoryToday(BasicNewsRecipe):
8 title = 'History Today'
9 __author__ = 'Rick Shang'
10
11 description = 'UK-based magazine, publishing articles and book reviews covering all types and periods of history.'
12 language = 'en'
13 category = 'news'
14 encoding = 'UTF-8'
15
16 remove_tags = [
17 dict(name='div', attrs={'class': ['print-logo', 'print-site_name', 'print-breadcrumb']}),
18 dict(name='div', attrs={'id': ['ht-tools', 'ht-tools2', 'ht-tags']}),
19 ]
20 no_javascript = True
21 no_stylesheets = True
22
23 needs_subscription = True
24
25 def get_browser(self):
26 br = BasicNewsRecipe.get_browser(self)
27 if self.username is not None and self.password is not None:
28 br.open('https://www.historytoday.com/user/login')
29 br.select_form(nr=1)
30 br['name'] = self.username
31 br['pass'] = self.password
32 res = br.submit()
33 raw = res.read()
34 if b'Session limit exceeded' in raw:
35 br.select_form(nr=1)
36 control = br.find_control('sid').items[1]
37 sid = []
38 br['sid'] = sid.join(control)
39 br.submit()
40 return br
41
42 def parse_index(self):
43
44 # Find date
45 soup0 = self.index_to_soup('https://www.historytoday.com/')
46 dates = self.tag_to_string(soup0.find('div', attrs={'id': 'block-block-226'}).span)
47 self.timefmt = ' [%s]' % dates
48
49 # Go to issue
50 soup = self.index_to_soup('https://www.historytoday.com/contents')
51 cover = soup.find('div', attrs={'id': 'content-area'}).find('img', attrs={'src': re.compile(r'.*cover.*')})['src']
52 self.cover_url = cover
53 self.log(self.cover_url)
54
55 # Go to the main body
56
57 div = soup.find('div', attrs={'class': 'region region-content-bottom'})
58
59 feeds = OrderedDict()
60 section_title = ''
61 for section in div.findAll('div', attrs={'id': re.compile(r'block\-views\-contents.*')}):
62 section_title = self.tag_to_string(section.find('h2', attrs={'class': 'title'}))
63 sectionbody = section.find('div', attrs={'class': 'view-content'})
64 for article in sectionbody.findAll('div', attrs={'class': re.compile(r'views\-row.*')}):
65 articles = []
66 subarticle = []
67 subarticle = article.findAll('div')
68 if len(subarticle) < 2:
69 continue
70 title = self.tag_to_string(subarticle[0])
71 originalurl = 'https://www.historytoday.com' + subarticle[0].span.a['href'].strip()
72 originalpage = self.index_to_soup(originalurl)
73 printurl = originalpage.find('div', attrs={'id': 'ht-tools'}).a['href'].strip()
74 url = 'https://www.historytoday.com' + printurl
75 desc = self.tag_to_string(subarticle[1])
76 articles.append({'title': title, 'url': url, 'description': desc, 'date': ''})
77
78 if articles:
79 if section_title not in feeds:
80 feeds[section_title] = []
81 feeds[section_title] += articles
82
83 ans = list(feeds.items())
84 return ans
85
86 def cleanup(self):
87 self.browser.open('https://www.historytoday.com/logout')