Debian Codegraph @

recipes/real_clear.recipe

raw
1 from __future__ import print_function
2
3 # Test with "\Program Files\Calibre2\ebook-convert.exe" RealClear.recipe
4 # .epub --test -vv --debug-pipeline debug
5 import re
6 import time
7
8 from calibre.ebooks.BeautifulSoup import NavigableString
9 from calibre.web.feeds.recipes import BasicNewsRecipe
10
11 try:
12 from urllib.parse import urlparse
13 except ImportError:
14 from urlparse import urlparse
15
16
17 class RealClear(BasicNewsRecipe):
18 title = 'Real Clear'
19 __author__ = 'TMcN'
20 description = 'Real Clear Politics/Science/etc... aggregation of news\n'
21 cover_url = 'http://www.realclearpolitics.com/dev/mt-static/images/logo.gif'
22 custom_title = 'Real Clear - ' + time.strftime('%d %b %Y')
23 auto_cleanup = True
24 encoding = 'utf8'
25 language = 'en'
26 needs_subscription = False
27 no_stylesheets = True
28 oldest_article = 7
29 remove_javascript = True
30 remove_tags = [dict(name='img', attrs={})]
31 # Don't go down
32 recursions = 0
33 max_articles_per_feed = 400
34 debugMessages = True
35
36 # Numeric parameter is type, controls whether we look for
37 feedsets = [
38 ['Politics', 'http://www.realclearpolitics.com/index.xml', 0],
39 ['Policy', 'http://www.realclearpolicy.com/index.xml', 0],
40 ['Science', 'http://www.realclearscience.com/index.xml', 0],
41 ['Tech', 'http://www.realcleartechnology.com/index.xml', 0],
42 # The feedburner is essentially the same as the top feed, politics.
43 # ["Politics Burner", "http://feeds.feedburner.com/realclearpolitics/qlMj", 1],
44 # ["Commentary", "http://feeds.feedburner.com/Realclearpolitics-Articles", 1],
45 ['Markets Home', 'http://www.realclearmarkets.com/index.xml', 0],
46 ['Markets', 'http://www.realclearmarkets.com/articles/index.xml', 0],
47 ['World', 'http://www.realclearworld.com/index.xml', 0],
48 ['World Blog', 'http://www.realclearworld.com/blog/index.xml', 2],
49 ]
50 # Hints to extractPrintURL.
51 # First column is the URL snippet. Then the string to search for as text,
52 # and the attributes to look for above it. Start with attributes and
53 # drill down.
54 phUrlSnip, phLinkText, phMainSearch, phHrefSearch = range(4)
55
56 printhints = [
57 ['realclear', '', '', 'printpage'],
58 ['billoreilly.com', 'Print this entry', 'a', ''],
59 ['billoreilly.com', 'Print This Article', 'a', ''],
60 ['politico.com', 'Print', 'a', 'share-print'],
61 ['nationalreview.com', '>Print<', 'a', ''],
62 ['reason.com', '', 'a', 'printer'],
63 # The following are not supported due to JavaScripting, and would require obfuscated_article to handle
64 # forbes,
65 # usatoday - just prints with all current crap anyhow
66 ]
67 # The print link isn't obvious, and only the end is needed (the -full append.) SO maybe try that first?s
68 # http://www.realclearpolitics.com/printpage/?url=http://www.realclearpolitics.com/articles/2012/01/24/in_speech_obama_to_call_for_fairness_--_and_four_more_years_112879-full.html
69 # Single page articles don't have a _full; e.g. http://www.realclearpolitics.com/articles/2012/01/25/obamas_green_robber_barons_112897.html
70 # Use the FULL PRINTPAGE URL; it formats it better too!
71 #
72 # NYT - try single page...
73 # Need special code - is it one page or several? Which URL?
74 # from http://www.nytimes.com/2012/01/22/business/apple-america-and-a-squeezed-middle-class.html?_r=1
75 # to http://www.nytimes.com/2012/01/22/business/apple-america-and-a-squeezed-middle-class.html?_r=1&pagewanted=all
76 # which is at link rel="canonical" and at <meta property="og:url"
77 # or look for "Single Page"
78
79 # Returns the best-guess print url.
80 # The second parameter (pageURL) is returned if nothing is found.
81 def extractPrintURL(self, pageURL):
82 tagURL = pageURL
83 baseParse = urlparse(pageURL)
84 baseURL = baseParse[0] + '://' + baseParse[1]
85 hintsCount = len(self.printhints)
86 for x in range(hintsCount):
87 if pageURL.find(self.printhints[x][0]) == -1:
88 continue
89 print('Trying ' + self.printhints[x][0])
90 # Only retrieve the soup if we have a match to check for the
91 # printed article with.
92 soup = self.index_to_soup(pageURL)
93 if soup is None:
94 return pageURL
95 if len(self.printhints[x][self.phHrefSearch]) > 0 and len(self.printhints[x][self.phLinkText]) == 0:
96 # e.g. RealClear
97 if self.debugMessages is True:
98 print('Search by href: ' + self.printhints[x][self.phHrefSearch])
99 printFind = soup.find(href=re.compile(self.printhints[x][self.phHrefSearch]))
100 elif len(self.printhints[x][3]) > 0 and len(self.printhints[x][1]) == 0:
101 if self.debugMessages is True:
102 print('Search 1: ' + self.printhints[x][2] + ' Attributes: ')
103 print(self.printhints[x][3])
104 printFind = soup.find(self.printhints[x][2], attrs=self.printhints[x][3])
105 elif len(self.printhints[x][3]) > 0:
106 if self.debugMessages is True:
107 print('search2')
108 printFind = soup.find(self.printhints[x][2], attrs=self.printhints[x][3], text=self.printhints[x][1])
109 else:
110 if self.debugMessages is True:
111 print('Default Search: ' + self.printhints[x][2] + ' Text: ' + self.printhints[x][1])
112 printFind = soup.find(self.printhints[x][2], text=self.printhints[x][1])
113 if printFind is None:
114 if self.debugMessages is True:
115 print('Not Found')
116 # print(soup)
117 print('end soup\n\n')
118 continue
119
120 print(printFind)
121 if isinstance(printFind, NavigableString) is False:
122 if printFind['href'] is not None:
123 print('Check ' + printFind['href'] + ' for base of ' + baseURL)
124 if printFind['href'].find('http') != 0:
125 return baseURL + printFind['href']
126 return printFind['href']
127 tag = printFind.parent
128 print(tag)
129 if tag.get('href', None) is None:
130 if self.debugMessages is True:
131 print('Not in parent, trying skip-up')
132 if tag.parent['href'] is None:
133 if self.debugMessages is True:
134 print('Not in skip either, aborting')
135 continue
136 return tag.parent['href']
137 return tag['href']
138 return tagURL
139
140 def get_browser(self):
141 if self.debugMessages is True:
142 print('In get_browser')
143 br = BasicNewsRecipe.get_browser(self)
144 return br
145
146 def parseRSS(self, index):
147 if self.debugMessages is True:
148 print('\n\nStarting ' + self.feedsets[index][0])
149 articleList = []
150 soup = self.index_to_soup(self.feedsets[index][1])
151 for div in soup.findAll('item'):
152 title = div.find('title').contents[0]
153 urlEl = div.find('originalLink')
154 if urlEl is None or len(urlEl.contents) == 0:
155 urlEl = div.find('originallink')
156 if urlEl is None or len(urlEl.contents) == 0:
157 urlEl = div.find('link')
158 if urlEl is None or len(urlEl.contents) == 0:
159 urlEl = div.find('guid')
160 if urlEl is None or title is None or len(urlEl.contents) == 0:
161 print('Error in feed ' + self.feedsets[index][0])
162 print(div)
163 continue
164 print(title)
165 print(urlEl)
166 url = urlEl.contents[0].encode('utf-8')
167 description = div.find('description')
168 if description is not None and description.contents is not None and len(description.contents) > 0:
169 description = description.contents[0]
170 else:
171 description = 'None'
172 pubDateEl = div.find('pubDate')
173 if pubDateEl is None:
174 pubDateEl = div.find('pubdate')
175 if pubDateEl is None:
176 pubDate = time.strftime('%a, %d %b')
177 else:
178 pubDate = pubDateEl.contents[0]
179 if self.debugMessages is True:
180 print('Article')
181 print(title)
182 print(description)
183 print(pubDate)
184 print(url)
185 try:
186 url = self.extractPrintURL(url)
187 except Exception:
188 self.log.exception('Failed to extract print URL for %s' % url)
189 print(url)
190 # url +=re.sub(r'\?.*', '', div['href'])
191 pubdate = time.strftime('%a, %d %b')
192 articleList.append(dict(title=title, url=url, date=pubdate, description=description, content=''))
193 return articleList
194
195 # calibre.web.feeds.news.BasicNewsRecipe.parse_index() fetches the list of articles.
196 # returns a list of tuple ('feed title', list of articles)
197 # {
198 # 'title' : article title,
199 # 'url' : URL of print version,
200 # 'date' : The publication date of the article as a string,
201 # 'description' : A summary of the article
202 # 'content' : The full article (can be an empty string). This is used by FullContentProfile
203 # }
204 # this is used instead of BasicNewsRecipe.parse_feeds().
205 def parse_index(self):
206 # Parse the page into Python Soup
207
208 # articleList = []
209 ans = []
210 feedsCount = len(self.feedsets)
211 for x in range(feedsCount): # should be ,4
212 feedarticles = self.parseRSS(x)
213 if feedarticles is not None:
214 ans.append((self.feedsets[x][0], feedarticles))
215 if self.debugMessages is True:
216 print(ans)
217 return ans