| 1 |
from __future__ import print_function |
| 2 |
|
| 3 |
# Test with "\Program Files\Calibre2\ebook-convert.exe" RealClear.recipe |
| 4 |
# .epub --test -vv --debug-pipeline debug |
| 5 |
import re |
| 6 |
import time |
| 7 |
|
| 8 |
from calibre.ebooks.BeautifulSoup import NavigableString |
| 9 |
from calibre.web.feeds.recipes import BasicNewsRecipe |
| 10 |
|
| 11 |
try: |
| 12 |
from urllib.parse import urlparse |
| 13 |
except ImportError: |
| 14 |
from urlparse import urlparse |
| 15 |
|
| 16 |
|
| 17 |
class RealClear(BasicNewsRecipe): |
| 18 |
title = 'Real Clear' |
| 19 |
__author__ = 'TMcN' |
| 20 |
description = 'Real Clear Politics/Science/etc... aggregation of news\n' |
| 21 |
cover_url = 'http://www.realclearpolitics.com/dev/mt-static/images/logo.gif' |
| 22 |
custom_title = 'Real Clear - ' + time.strftime('%d %b %Y') |
| 23 |
auto_cleanup = True |
| 24 |
encoding = 'utf8' |
| 25 |
language = 'en' |
| 26 |
needs_subscription = False |
| 27 |
no_stylesheets = True |
| 28 |
oldest_article = 7 |
| 29 |
remove_javascript = True |
| 30 |
remove_tags = [dict(name='img', attrs={})] |
| 31 |
# Don't go down |
| 32 |
recursions = 0 |
| 33 |
max_articles_per_feed = 400 |
| 34 |
debugMessages = True |
| 35 |
|
| 36 |
# Numeric parameter is type, controls whether we look for |
| 37 |
feedsets = [ |
| 38 |
['Politics', 'http://www.realclearpolitics.com/index.xml', 0], |
| 39 |
['Policy', 'http://www.realclearpolicy.com/index.xml', 0], |
| 40 |
['Science', 'http://www.realclearscience.com/index.xml', 0], |
| 41 |
['Tech', 'http://www.realcleartechnology.com/index.xml', 0], |
| 42 |
# The feedburner is essentially the same as the top feed, politics. |
| 43 |
# ["Politics Burner", "http://feeds.feedburner.com/realclearpolitics/qlMj", 1], |
| 44 |
# ["Commentary", "http://feeds.feedburner.com/Realclearpolitics-Articles", 1], |
| 45 |
['Markets Home', 'http://www.realclearmarkets.com/index.xml', 0], |
| 46 |
['Markets', 'http://www.realclearmarkets.com/articles/index.xml', 0], |
| 47 |
['World', 'http://www.realclearworld.com/index.xml', 0], |
| 48 |
['World Blog', 'http://www.realclearworld.com/blog/index.xml', 2], |
| 49 |
] |
| 50 |
# Hints to extractPrintURL. |
| 51 |
# First column is the URL snippet. Then the string to search for as text, |
| 52 |
# and the attributes to look for above it. Start with attributes and |
| 53 |
# drill down. |
| 54 |
phUrlSnip, phLinkText, phMainSearch, phHrefSearch = range(4) |
| 55 |
|
| 56 |
printhints = [ |
| 57 |
['realclear', '', '', 'printpage'], |
| 58 |
['billoreilly.com', 'Print this entry', 'a', ''], |
| 59 |
['billoreilly.com', 'Print This Article', 'a', ''], |
| 60 |
['politico.com', 'Print', 'a', 'share-print'], |
| 61 |
['nationalreview.com', '>Print<', 'a', ''], |
| 62 |
['reason.com', '', 'a', 'printer'], |
| 63 |
# The following are not supported due to JavaScripting, and would require obfuscated_article to handle |
| 64 |
# forbes, |
| 65 |
# usatoday - just prints with all current crap anyhow |
| 66 |
] |
| 67 |
# The print link isn't obvious, and only the end is needed (the -full append.) SO maybe try that first?s |
| 68 |
# http://www.realclearpolitics.com/printpage/?url=http://www.realclearpolitics.com/articles/2012/01/24/in_speech_obama_to_call_for_fairness_--_and_four_more_years_112879-full.html |
| 69 |
# Single page articles don't have a _full; e.g. http://www.realclearpolitics.com/articles/2012/01/25/obamas_green_robber_barons_112897.html |
| 70 |
# Use the FULL PRINTPAGE URL; it formats it better too! |
| 71 |
# |
| 72 |
# NYT - try single page... |
| 73 |
# Need special code - is it one page or several? Which URL? |
| 74 |
# from http://www.nytimes.com/2012/01/22/business/apple-america-and-a-squeezed-middle-class.html?_r=1 |
| 75 |
# to http://www.nytimes.com/2012/01/22/business/apple-america-and-a-squeezed-middle-class.html?_r=1&pagewanted=all |
| 76 |
# which is at link rel="canonical" and at <meta property="og:url" |
| 77 |
# or look for "Single Page" |
| 78 |
|
| 79 |
# Returns the best-guess print url. |
| 80 |
# The second parameter (pageURL) is returned if nothing is found. |
| 81 |
def extractPrintURL(self, pageURL): |
| 82 |
tagURL = pageURL |
| 83 |
baseParse = urlparse(pageURL) |
| 84 |
baseURL = baseParse[0] + '://' + baseParse[1] |
| 85 |
hintsCount = len(self.printhints) |
| 86 |
for x in range(hintsCount): |
| 87 |
if pageURL.find(self.printhints[x][0]) == -1: |
| 88 |
continue |
| 89 |
print('Trying ' + self.printhints[x][0]) |
| 90 |
# Only retrieve the soup if we have a match to check for the |
| 91 |
# printed article with. |
| 92 |
soup = self.index_to_soup(pageURL) |
| 93 |
if soup is None: |
| 94 |
return pageURL |
| 95 |
if len(self.printhints[x][self.phHrefSearch]) > 0 and len(self.printhints[x][self.phLinkText]) == 0: |
| 96 |
# e.g. RealClear |
| 97 |
if self.debugMessages is True: |
| 98 |
print('Search by href: ' + self.printhints[x][self.phHrefSearch]) |
| 99 |
printFind = soup.find(href=re.compile(self.printhints[x][self.phHrefSearch])) |
| 100 |
elif len(self.printhints[x][3]) > 0 and len(self.printhints[x][1]) == 0: |
| 101 |
if self.debugMessages is True: |
| 102 |
print('Search 1: ' + self.printhints[x][2] + ' Attributes: ') |
| 103 |
print(self.printhints[x][3]) |
| 104 |
printFind = soup.find(self.printhints[x][2], attrs=self.printhints[x][3]) |
| 105 |
elif len(self.printhints[x][3]) > 0: |
| 106 |
if self.debugMessages is True: |
| 107 |
print('search2') |
| 108 |
printFind = soup.find(self.printhints[x][2], attrs=self.printhints[x][3], text=self.printhints[x][1]) |
| 109 |
else: |
| 110 |
if self.debugMessages is True: |
| 111 |
print('Default Search: ' + self.printhints[x][2] + ' Text: ' + self.printhints[x][1]) |
| 112 |
printFind = soup.find(self.printhints[x][2], text=self.printhints[x][1]) |
| 113 |
if printFind is None: |
| 114 |
if self.debugMessages is True: |
| 115 |
print('Not Found') |
| 116 |
# print(soup) |
| 117 |
print('end soup\n\n') |
| 118 |
continue |
| 119 |
|
| 120 |
print(printFind) |
| 121 |
if isinstance(printFind, NavigableString) is False: |
| 122 |
if printFind['href'] is not None: |
| 123 |
print('Check ' + printFind['href'] + ' for base of ' + baseURL) |
| 124 |
if printFind['href'].find('http') != 0: |
| 125 |
return baseURL + printFind['href'] |
| 126 |
return printFind['href'] |
| 127 |
tag = printFind.parent |
| 128 |
print(tag) |
| 129 |
if tag.get('href', None) is None: |
| 130 |
if self.debugMessages is True: |
| 131 |
print('Not in parent, trying skip-up') |
| 132 |
if tag.parent['href'] is None: |
| 133 |
if self.debugMessages is True: |
| 134 |
print('Not in skip either, aborting') |
| 135 |
continue |
| 136 |
return tag.parent['href'] |
| 137 |
return tag['href'] |
| 138 |
return tagURL |
| 139 |
|
| 140 |
def get_browser(self): |
| 141 |
if self.debugMessages is True: |
| 142 |
print('In get_browser') |
| 143 |
br = BasicNewsRecipe.get_browser(self) |
| 144 |
return br |
| 145 |
|
| 146 |
def parseRSS(self, index): |
| 147 |
if self.debugMessages is True: |
| 148 |
print('\n\nStarting ' + self.feedsets[index][0]) |
| 149 |
articleList = [] |
| 150 |
soup = self.index_to_soup(self.feedsets[index][1]) |
| 151 |
for div in soup.findAll('item'): |
| 152 |
title = div.find('title').contents[0] |
| 153 |
urlEl = div.find('originalLink') |
| 154 |
if urlEl is None or len(urlEl.contents) == 0: |
| 155 |
urlEl = div.find('originallink') |
| 156 |
if urlEl is None or len(urlEl.contents) == 0: |
| 157 |
urlEl = div.find('link') |
| 158 |
if urlEl is None or len(urlEl.contents) == 0: |
| 159 |
urlEl = div.find('guid') |
| 160 |
if urlEl is None or title is None or len(urlEl.contents) == 0: |
| 161 |
print('Error in feed ' + self.feedsets[index][0]) |
| 162 |
print(div) |
| 163 |
continue |
| 164 |
print(title) |
| 165 |
print(urlEl) |
| 166 |
url = urlEl.contents[0].encode('utf-8') |
| 167 |
description = div.find('description') |
| 168 |
if description is not None and description.contents is not None and len(description.contents) > 0: |
| 169 |
description = description.contents[0] |
| 170 |
else: |
| 171 |
description = 'None' |
| 172 |
pubDateEl = div.find('pubDate') |
| 173 |
if pubDateEl is None: |
| 174 |
pubDateEl = div.find('pubdate') |
| 175 |
if pubDateEl is None: |
| 176 |
pubDate = time.strftime('%a, %d %b') |
| 177 |
else: |
| 178 |
pubDate = pubDateEl.contents[0] |
| 179 |
if self.debugMessages is True: |
| 180 |
print('Article') |
| 181 |
print(title) |
| 182 |
print(description) |
| 183 |
print(pubDate) |
| 184 |
print(url) |
| 185 |
try: |
| 186 |
url = self.extractPrintURL(url) |
| 187 |
except Exception: |
| 188 |
self.log.exception('Failed to extract print URL for %s' % url) |
| 189 |
print(url) |
| 190 |
# url +=re.sub(r'\?.*', '', div['href']) |
| 191 |
pubdate = time.strftime('%a, %d %b') |
| 192 |
articleList.append(dict(title=title, url=url, date=pubdate, description=description, content='')) |
| 193 |
return articleList |
| 194 |
|
| 195 |
# calibre.web.feeds.news.BasicNewsRecipe.parse_index() fetches the list of articles. |
| 196 |
# returns a list of tuple ('feed title', list of articles) |
| 197 |
# { |
| 198 |
# 'title' : article title, |
| 199 |
# 'url' : URL of print version, |
| 200 |
# 'date' : The publication date of the article as a string, |
| 201 |
# 'description' : A summary of the article |
| 202 |
# 'content' : The full article (can be an empty string). This is used by FullContentProfile |
| 203 |
# } |
| 204 |
# this is used instead of BasicNewsRecipe.parse_feeds(). |
| 205 |
def parse_index(self): |
| 206 |
# Parse the page into Python Soup |
| 207 |
|
| 208 |
# articleList = [] |
| 209 |
ans = [] |
| 210 |
feedsCount = len(self.feedsets) |
| 211 |
for x in range(feedsCount): # should be ,4 |
| 212 |
feedarticles = self.parseRSS(x) |
| 213 |
if feedarticles is not None: |
| 214 |
ans.append((self.feedsets[x][0], feedarticles)) |
| 215 |
if self.debugMessages is True: |
| 216 |
print(ans) |
| 217 |
return ans |