Debian Codegraph @

recipes/journalofaccountancy.recipe

raw
1 #!/usr/bin/python2
2 # vim:fileencoding=utf-8
3 # License: GPLv3 Copyright: 2020, Jose Ortiz <jlortiz84 at gmail.com>
4
5 from __future__ import absolute_import, division, print_function, unicode_literals
6
7 import json
8 from contextlib import closing
9 from time import sleep
10
11 from mechanize import Request
12
13 from calibre.web.feeds.news import BasicNewsRecipe
14
15
16 def absolutize(url):
17 if url.startswith('/'):
18 url = ('https://www.journalofaccountancy.com' + url).partition('#')[0]
19 return url
20
21
22 def classes(classes):
23 q = frozenset(classes.split(' '))
24 return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)})
25
26
27 class JournalOfAccountancy(BasicNewsRecipe):
28 __author__ = 'Jose Ortiz'
29 language = 'en'
30 title = 'Journal of Accountancy'
31 description = (
32 'A monthly journal of tax, financial reporting, auditing and other'
33 ' topics of accountancy from American Institute of Certified Public'
34 ' Accountants (AICPA).'
35 )
36 publication_type = 'magazine'
37 masthead_url = 'http://developmentprofits.com/images/JournalOfAccountancy.jpg'
38 no_stylesheets = True
39 remove_javascript = True
40
41 conversion_options = {'comments': description, 'tags': 'News, Accountancy', 'publisher': 'American Institute of Certified Public Accountants (AICPA)'}
42
43 keep_only_tags = [classes('contentSectionArticlePage')]
44
45 def parse_index(self):
46 # ISSUES ######################
47 issues_url = 'https://www.journalofaccountancy.com/issues.html'
48 with closing(self.browser.open(issues_url)):
49 pass
50 ###############################
51
52 common_headers = {
53 'X-Requested-With': 'XMLHttpRequest',
54 'Accept': 'application/json, text/javascript, */*; q=0.01',
55 'DNT': '1',
56 'Pragma': 'no-cache',
57 'Cache-Control': 'no-cache',
58 }
59
60 URL_TEMPLATE = 'https://www.journalofaccountancy.com/content/jofa-home/issues/jcr:content/main-content-section/issuelibrary.%s.service'
61
62 # INIT #################################################
63 init_url = URL_TEMPLATE % 'init'
64 init_headers = {'Referer': issues_url}
65 init_headers.update(common_headers)
66
67 self.log('\nINIT URL at ', init_url)
68 with closing(self.browser.open(Request(init_url, None, init_headers))) as r:
69 issue_path = json.loads(r.read())[0]['page']['path']
70 ########################################################
71
72 # FILTER ###############################
73 filter_url = URL_TEMPLATE % ('filter.' + issue_path.split('/')[-2])
74 filter_headers = {'issues': issue_path}
75 filter_headers.update(init_headers)
76
77 self.log('\nFILTER URL at ', filter_url)
78 with closing(self.browser.open(Request(filter_url, None, filter_headers))) as r:
79 issue_data = json.loads(r.read())[0]
80 ########################################
81
82 self.cover_url = absolutize(issue_data['issueCover']['src'])
83 self.log('cover_url at ', self.cover_url)
84 self.timefmt = ' ' + issue_data['issueName']
85
86 # INDEX ####################################
87 index_url = absolutize(issue_path + '.html')
88 self.log('INDEX URL at ', index_url)
89 self.log('3 second pause')
90 sleep(3) # mimicking human user behavior
91 with closing(self.browser.open(index_url)):
92 pass
93 ############################################
94
95 service_headers = {'Referer': index_url}
96 service_headers.update(common_headers)
97
98 def get_data(service):
99 service_url = 'https://www.journalofaccountancy.com' + issue_path + '/jcr:content/main-content-section/' + service + '.en.service'
100 self.log('\nSERIVICE URL at ', service_url)
101 req = Request(service_url, None, service_headers)
102 with closing(self.browser.open(req)) as r:
103 return json.loads(r.read())
104
105 def make_topic(category, articles):
106 topic = (category, [])
107 self.log(topic[0])
108 for article in articles:
109 title = article['articleTitle']
110 url = absolutize(article['page']['path'] + '.html')
111 desc = article.get('articleAbstract')
112 self.log('\t', title, ' at ', url)
113 topic[1].append({'title': title, 'url': url, 'description': desc})
114 return topic
115
116 ans = [make_topic('SPOTLIGHT', get_data('issuelanding/articles1')), make_topic('FEATURES', get_data('issuelanding/articles2'))]
117
118 for category, articles in get_data('articletypelist').items():
119 ans.append(make_topic(category, articles))
120
121 return ans