| 1 |
#!/usr/bin/env python |
| 2 |
# vim:fileencoding=utf-8 |
| 3 |
from __future__ import absolute_import, division, print_function, unicode_literals |
| 4 |
|
| 5 |
import json |
| 6 |
import time |
| 7 |
from pprint import pprint |
| 8 |
|
| 9 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 10 |
|
| 11 |
|
| 12 |
class OrangeCountyRegister(BasicNewsRecipe): |
| 13 |
title = 'Orange County Register' |
| 14 |
__author__ = 'TechnoCat' |
| 15 |
description = 'The O.C. Register\nRecipe: Nov 2016' |
| 16 |
cover_url = 'http://www.realclearpolitics.com/dev/mt-static/images/logo.gif' |
| 17 |
custom_title = 'OC Register - ' + time.strftime('%d %b %Y') |
| 18 |
auto_cleanup = True |
| 19 |
extra_css = 'div.metaAuthor { display:none;}\n' |
| 20 |
encoding = 'utf8' |
| 21 |
language = 'en_US' |
| 22 |
needs_subscription = False |
| 23 |
no_stylesheets = True |
| 24 |
oldest_article = 7 |
| 25 |
remove_javascript = True |
| 26 |
remove_tags = [dict(name='img', attrs={})] |
| 27 |
# Don't go down |
| 28 |
recursions = 0 |
| 29 |
max_articles_per_feed = 400 |
| 30 |
debugMessages = False |
| 31 |
|
| 32 |
feeds = [ |
| 33 |
('News', 'https://www.ocregister.com/news/'), |
| 34 |
('Opinion', 'https://www.ocregister.com/opinion/'), |
| 35 |
('Politics', 'https://www.ocregister.com/news/politics/'), |
| 36 |
('Business', 'https://www.ocregister.com/business/'), |
| 37 |
] |
| 38 |
|
| 39 |
def parsePage(self, index): |
| 40 |
if self.debugMessages is True: |
| 41 |
print('\n\nStarting ' + self.feeds[index][0]) |
| 42 |
articleList = [] |
| 43 |
soup = self.index_to_soup(self.feeds[index][1]) |
| 44 |
# Have this index page now. |
| 45 |
# look for a.article-title |
| 46 |
# If any, the description is <div class="excerpt"> |
| 47 |
for newsentry in soup.findAll('a', {'class': 'article-title'}): |
| 48 |
print('Next up:') |
| 49 |
print(newsentry) |
| 50 |
title = newsentry['title'] |
| 51 |
url = newsentry['href'] |
| 52 |
print('Title: ') |
| 53 |
print(title) |
| 54 |
print('URL') |
| 55 |
print(url) |
| 56 |
pubdate = time.strftime('%a, %d %b') |
| 57 |
articleList.append(dict(title=title, url=url, date=pubdate, description=title, content='')) |
| 58 |
return articleList |
| 59 |
|
| 60 |
def extract_readable_article(self, html, url): |
| 61 |
cleanedHTML = super(OrangeCountyRegister, self).extract_readable_article(html, url) |
| 62 |
print('Processing html for author') |
| 63 |
# Find the attribs... |
| 64 |
attribDict = self.htmlToAttribsDict(html) |
| 65 |
print('dict is type...') |
| 66 |
print(type(attribDict)) |
| 67 |
author = attribDict.get('Byline') |
| 68 |
if author is not None: |
| 69 |
# add author code after <body> |
| 70 |
print('Adding author in meta') |
| 71 |
print(author) |
| 72 |
cleanedHTML = cleanedHTML.replace('<body>', '<body>\n<div class="metaAuthor" value="' + author + '"></div>\n') |
| 73 |
else: |
| 74 |
print('no author found') |
| 75 |
print(html) |
| 76 |
# pubDate = attribDict.get('Publish Hour of Day') |
| 77 |
return cleanedHTML |
| 78 |
|
| 79 |
def loadURL(self, url): |
| 80 |
return self.index_to_soup(url, raw=True) |
| 81 |
|
| 82 |
def htmlToAttribsDict(self, rawHTML): |
| 83 |
tokenStart = 'dataLayer.push({' |
| 84 |
tokenEnd = '});' |
| 85 |
print('1') |
| 86 |
startJSON = rawHTML.find(tokenStart) |
| 87 |
if startJSON < 0: |
| 88 |
return |
| 89 |
JSONBeginning = rawHTML[startJSON + len(tokenStart) - 1 :] |
| 90 |
endJSON = JSONBeginning.find(tokenEnd) |
| 91 |
if endJSON < 0: |
| 92 |
return |
| 93 |
JSON = JSONBeginning[: endJSON + 1] |
| 94 |
JSONQuoted = JSON.replace("'", '"') |
| 95 |
try: |
| 96 |
metadata = json.loads(JSONQuoted) |
| 97 |
pprint(metadata) |
| 98 |
return metadata |
| 99 |
except ValueError: |
| 100 |
print('Could not decode JSON:') |
| 101 |
print(JSONQuoted) |
| 102 |
return None |
| 103 |
|
| 104 |
# calibre.web.feeds.news.BasicNewsRecipe.parse_index() fetches the list of articles. |
| 105 |
# returns a list of tuple ('feed title', list of articles) |
| 106 |
# { |
| 107 |
# 'title' : article title, |
| 108 |
# 'url' : URL of print version, |
| 109 |
# 'date' : The publication date of the article as a string, |
| 110 |
# 'description' : A summary of the article |
| 111 |
# 'content' : The full article (can be an empty string). This is used by FullContentProfile |
| 112 |
# } |
| 113 |
# this is used instead of BasicNewsRecipe.parse_feeds(). |
| 114 |
def parse_index(self): |
| 115 |
# Parse the page into Python Soup |
| 116 |
# articleList = [] |
| 117 |
ans = [] |
| 118 |
feedsCount = len(self.feeds) |
| 119 |
for x in range(feedsCount - 1): # should be ,4 |
| 120 |
feedarticles = self.parsePage(x) |
| 121 |
if feedarticles is not None: |
| 122 |
ans.append((self.feeds[x][0], feedarticles)) |
| 123 |
if self.debugMessages is True: |
| 124 |
print(ans) |
| 125 |
return ans |