Debian Codegraph @

recipes/oc_register.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 from __future__ import absolute_import, division, print_function, unicode_literals
4
5 import json
6 import time
7 from pprint import pprint
8
9 from calibre.web.feeds.news import BasicNewsRecipe
10
11
12 class OrangeCountyRegister(BasicNewsRecipe):
13 title = 'Orange County Register'
14 __author__ = 'TechnoCat'
15 description = 'The O.C. Register\nRecipe: Nov 2016'
16 cover_url = 'http://www.realclearpolitics.com/dev/mt-static/images/logo.gif'
17 custom_title = 'OC Register - ' + time.strftime('%d %b %Y')
18 auto_cleanup = True
19 extra_css = 'div.metaAuthor { display:none;}\n'
20 encoding = 'utf8'
21 language = 'en_US'
22 needs_subscription = False
23 no_stylesheets = True
24 oldest_article = 7
25 remove_javascript = True
26 remove_tags = [dict(name='img', attrs={})]
27 # Don't go down
28 recursions = 0
29 max_articles_per_feed = 400
30 debugMessages = False
31
32 feeds = [
33 ('News', 'https://www.ocregister.com/news/'),
34 ('Opinion', 'https://www.ocregister.com/opinion/'),
35 ('Politics', 'https://www.ocregister.com/news/politics/'),
36 ('Business', 'https://www.ocregister.com/business/'),
37 ]
38
39 def parsePage(self, index):
40 if self.debugMessages is True:
41 print('\n\nStarting ' + self.feeds[index][0])
42 articleList = []
43 soup = self.index_to_soup(self.feeds[index][1])
44 # Have this index page now.
45 # look for a.article-title
46 # If any, the description is <div class="excerpt">
47 for newsentry in soup.findAll('a', {'class': 'article-title'}):
48 print('Next up:')
49 print(newsentry)
50 title = newsentry['title']
51 url = newsentry['href']
52 print('Title: ')
53 print(title)
54 print('URL')
55 print(url)
56 pubdate = time.strftime('%a, %d %b')
57 articleList.append(dict(title=title, url=url, date=pubdate, description=title, content=''))
58 return articleList
59
60 def extract_readable_article(self, html, url):
61 cleanedHTML = super(OrangeCountyRegister, self).extract_readable_article(html, url)
62 print('Processing html for author')
63 # Find the attribs...
64 attribDict = self.htmlToAttribsDict(html)
65 print('dict is type...')
66 print(type(attribDict))
67 author = attribDict.get('Byline')
68 if author is not None:
69 # add author code after <body>
70 print('Adding author in meta')
71 print(author)
72 cleanedHTML = cleanedHTML.replace('<body>', '<body>\n<div class="metaAuthor" value="' + author + '"></div>\n')
73 else:
74 print('no author found')
75 print(html)
76 # pubDate = attribDict.get('Publish Hour of Day')
77 return cleanedHTML
78
79 def loadURL(self, url):
80 return self.index_to_soup(url, raw=True)
81
82 def htmlToAttribsDict(self, rawHTML):
83 tokenStart = 'dataLayer.push({'
84 tokenEnd = '});'
85 print('1')
86 startJSON = rawHTML.find(tokenStart)
87 if startJSON < 0:
88 return
89 JSONBeginning = rawHTML[startJSON + len(tokenStart) - 1 :]
90 endJSON = JSONBeginning.find(tokenEnd)
91 if endJSON < 0:
92 return
93 JSON = JSONBeginning[: endJSON + 1]
94 JSONQuoted = JSON.replace("'", '"')
95 try:
96 metadata = json.loads(JSONQuoted)
97 pprint(metadata)
98 return metadata
99 except ValueError:
100 print('Could not decode JSON:')
101 print(JSONQuoted)
102 return None
103
104 # calibre.web.feeds.news.BasicNewsRecipe.parse_index() fetches the list of articles.
105 # returns a list of tuple ('feed title', list of articles)
106 # {
107 # 'title' : article title,
108 # 'url' : URL of print version,
109 # 'date' : The publication date of the article as a string,
110 # 'description' : A summary of the article
111 # 'content' : The full article (can be an empty string). This is used by FullContentProfile
112 # }
113 # this is used instead of BasicNewsRecipe.parse_feeds().
114 def parse_index(self):
115 # Parse the page into Python Soup
116 # articleList = []
117 ans = []
118 feedsCount = len(self.feeds)
119 for x in range(feedsCount - 1): # should be ,4
120 feedarticles = self.parsePage(x)
121 if feedarticles is not None:
122 ans.append((self.feeds[x][0], feedarticles))
123 if self.debugMessages is True:
124 print(ans)
125 return ans