Debian Codegraph @

recipes/thenewcriterion.recipe

raw
1 # -*- mode: python -*-
2 # -*- coding: utf-8 -*-
3 # vi: set fenc=utf-8 ft=python :
4 # kate: encoding utf-8; syntax python;
5
6 __license__ = 'GPL v3'
7 __copyright__ = '2019, Darko Miletic <darko.miletic at gmail.com>'
8 '''
9 www.newcriterion.com
10 '''
11
12 from calibre import strftime
13 from calibre.ptempfile import PersistentTemporaryFile
14 from calibre.web.feeds.news import BasicNewsRecipe
15
16
17 class TheNewCriterion(BasicNewsRecipe):
18 title = 'The New Criterion'
19 __author__ = 'Darko Miletic'
20 description = 'On the front lines of the battle for culture'
21 publisher = 'The Foundation for Cultural Review'
22 category = 'art, politics, USA, world'
23 oldest_article = 40
24 no_stylesheets = True
25 encoding = 'utf8'
26 use_embedded_content = False
27 language = 'en'
28 remove_empty_feeds = True
29 publication_type = 'magazine'
30 needs_subscription = 'optional'
31 delay = 1
32 simultaneous_downloads = 1
33 timeout = 8
34 ignore_duplicate_articles = {'url'}
35 articles_are_obfuscated = True
36 temp_files = []
37 fetch_retries = 10
38 auto_cleanup = True
39 masthead_url = 'https://www.newcriterion.com/themes/thenewcriterion/assets/img/horizontal-logo.svg'
40 extra_css = '''
41 body{font-family: Galliard, serif}
42 '''
43
44 conversion_options = {'comment': description, 'tags': category, 'publisher': publisher, 'language': language}
45
46 def get_browser(self):
47 br = BasicNewsRecipe.get_browser(self)
48 if self.username is not None and self.password is not None:
49 br.set_cookie('wordpress_test_cookie', 'WP+Cookie+check', '.newcriterion.com')
50 br.open('https://newcriterion.com/wp-login.php')
51 br.select_form(name='loginform')
52 br['log'] = self.username
53 br['pwd'] = self.password
54 br.submit()
55 return br
56
57 def parse_index(self):
58 months = ('january', 'february', 'march', 'april', 'may', 'june', 'july', 'august', 'september', 'october', 'november', 'december')
59 slug = months[int(strftime('%m')) - 1] + '-' + strftime('%Y')
60 currentIssue_url = 'https://newcriterion.com/issues/' + slug + '/'
61 self.log(currentIssue_url)
62 soup1 = self.index_to_soup(currentIssue_url)
63 date = strftime(' %B %Y')
64 articles = []
65 subset = soup1.find('div', attrs={'class': 'issue-layout'})
66 if subset is None:
67 return []
68 for art in subset.findAll('article', attrs={'class': 'article-display'}):
69 h2 = art.find('h2')
70 if h2 is None:
71 continue
72 a = h2.find('a', href=True)
73 if a is None or '/article/' not in a['href']:
74 continue
75 title = self.tag_to_string(a).strip()
76 excerpt = art.find('p', attrs={'class': 'post-excerpt'})
77 description = self.tag_to_string(excerpt).strip() if excerpt else ''
78 articles.append({
79 'title': title,
80 'date': date,
81 'url': a['href'],
82 'description': description,
83 })
84 return [(self.title, articles)]
85
86 def get_obfuscated_article(self, url):
87 result = None
88 count = 0
89 while count < self.fetch_retries:
90 try:
91 response = self.browser.open(url, timeout=self.timeout)
92 html = response.read()
93 count = self.fetch_retries
94 tfile = PersistentTemporaryFile('_fa.html')
95 tfile.write(html)
96 tfile.close()
97 self.temp_files.append(tfile)
98 result = tfile.name
99 except Exception:
100 print('Retrying download...')
101 count += 1
102 return result