| 1 |
#!/usr/bin/python2 |
| 2 |
# vim:fileencoding=utf-8 |
| 3 |
# License: GPLv3 Copyright: 2020, Jose Ortiz <jlortiz84 at gmail.com> |
| 4 |
|
| 5 |
from __future__ import absolute_import, division, print_function, unicode_literals |
| 6 |
|
| 7 |
import json |
| 8 |
from contextlib import closing |
| 9 |
from time import sleep |
| 10 |
|
| 11 |
from mechanize import Request |
| 12 |
|
| 13 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 14 |
|
| 15 |
|
| 16 |
def absolutize(url): |
| 17 |
if url.startswith('/'): |
| 18 |
url = ('https://www.journalofaccountancy.com' + url).partition('#')[0] |
| 19 |
return url |
| 20 |
|
| 21 |
|
| 22 |
def classes(classes): |
| 23 |
q = frozenset(classes.split(' ')) |
| 24 |
return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)}) |
| 25 |
|
| 26 |
|
| 27 |
class JournalOfAccountancy(BasicNewsRecipe): |
| 28 |
__author__ = 'Jose Ortiz' |
| 29 |
language = 'en' |
| 30 |
title = 'Journal of Accountancy' |
| 31 |
description = ( |
| 32 |
'A monthly journal of tax, financial reporting, auditing and other' |
| 33 |
' topics of accountancy from American Institute of Certified Public' |
| 34 |
' Accountants (AICPA).' |
| 35 |
) |
| 36 |
publication_type = 'magazine' |
| 37 |
masthead_url = 'http://developmentprofits.com/images/JournalOfAccountancy.jpg' |
| 38 |
no_stylesheets = True |
| 39 |
remove_javascript = True |
| 40 |
|
| 41 |
conversion_options = {'comments': description, 'tags': 'News, Accountancy', 'publisher': 'American Institute of Certified Public Accountants (AICPA)'} |
| 42 |
|
| 43 |
keep_only_tags = [classes('contentSectionArticlePage')] |
| 44 |
|
| 45 |
def parse_index(self): |
| 46 |
# ISSUES ###################### |
| 47 |
issues_url = 'https://www.journalofaccountancy.com/issues.html' |
| 48 |
with closing(self.browser.open(issues_url)): |
| 49 |
pass |
| 50 |
############################### |
| 51 |
|
| 52 |
common_headers = { |
| 53 |
'X-Requested-With': 'XMLHttpRequest', |
| 54 |
'Accept': 'application/json, text/javascript, */*; q=0.01', |
| 55 |
'DNT': '1', |
| 56 |
'Pragma': 'no-cache', |
| 57 |
'Cache-Control': 'no-cache', |
| 58 |
} |
| 59 |
|
| 60 |
URL_TEMPLATE = 'https://www.journalofaccountancy.com/content/jofa-home/issues/jcr:content/main-content-section/issuelibrary.%s.service' |
| 61 |
|
| 62 |
# INIT ################################################# |
| 63 |
init_url = URL_TEMPLATE % 'init' |
| 64 |
init_headers = {'Referer': issues_url} |
| 65 |
init_headers.update(common_headers) |
| 66 |
|
| 67 |
self.log('\nINIT URL at ', init_url) |
| 68 |
with closing(self.browser.open(Request(init_url, None, init_headers))) as r: |
| 69 |
issue_path = json.loads(r.read())[0]['page']['path'] |
| 70 |
######################################################## |
| 71 |
|
| 72 |
# FILTER ############################### |
| 73 |
filter_url = URL_TEMPLATE % ('filter.' + issue_path.split('/')[-2]) |
| 74 |
filter_headers = {'issues': issue_path} |
| 75 |
filter_headers.update(init_headers) |
| 76 |
|
| 77 |
self.log('\nFILTER URL at ', filter_url) |
| 78 |
with closing(self.browser.open(Request(filter_url, None, filter_headers))) as r: |
| 79 |
issue_data = json.loads(r.read())[0] |
| 80 |
######################################## |
| 81 |
|
| 82 |
self.cover_url = absolutize(issue_data['issueCover']['src']) |
| 83 |
self.log('cover_url at ', self.cover_url) |
| 84 |
self.timefmt = ' ' + issue_data['issueName'] |
| 85 |
|
| 86 |
# INDEX #################################### |
| 87 |
index_url = absolutize(issue_path + '.html') |
| 88 |
self.log('INDEX URL at ', index_url) |
| 89 |
self.log('3 second pause') |
| 90 |
sleep(3) # mimicking human user behavior |
| 91 |
with closing(self.browser.open(index_url)): |
| 92 |
pass |
| 93 |
############################################ |
| 94 |
|
| 95 |
service_headers = {'Referer': index_url} |
| 96 |
service_headers.update(common_headers) |
| 97 |
|
| 98 |
def get_data(service): |
| 99 |
service_url = 'https://www.journalofaccountancy.com' + issue_path + '/jcr:content/main-content-section/' + service + '.en.service' |
| 100 |
self.log('\nSERIVICE URL at ', service_url) |
| 101 |
req = Request(service_url, None, service_headers) |
| 102 |
with closing(self.browser.open(req)) as r: |
| 103 |
return json.loads(r.read()) |
| 104 |
|
| 105 |
def make_topic(category, articles): |
| 106 |
topic = (category, []) |
| 107 |
self.log(topic[0]) |
| 108 |
for article in articles: |
| 109 |
title = article['articleTitle'] |
| 110 |
url = absolutize(article['page']['path'] + '.html') |
| 111 |
desc = article.get('articleAbstract') |
| 112 |
self.log('\t', title, ' at ', url) |
| 113 |
topic[1].append({'title': title, 'url': url, 'description': desc}) |
| 114 |
return topic |
| 115 |
|
| 116 |
ans = [make_topic('SPOTLIGHT', get_data('issuelanding/articles1')), make_topic('FEATURES', get_data('issuelanding/articles2'))] |
| 117 |
|
| 118 |
for category, articles in get_data('articletypelist').items(): |
| 119 |
ans.append(make_topic(category, articles)) |
| 120 |
|
| 121 |
return ans |