| 1 |
#!/usr/bin/env python |
| 2 |
|
| 3 |
__license__ = 'GPL v3' |
| 4 |
''' |
| 5 |
Hacker News |
| 6 |
''' |
| 7 |
from calibre.ptempfile import PersistentTemporaryFile |
| 8 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 9 |
|
| 10 |
try: |
| 11 |
from urllib.parse import urlparse |
| 12 |
except ImportError: |
| 13 |
from urlparse import urlparse |
| 14 |
import re |
| 15 |
|
| 16 |
HN_URL_RE = re.compile(r'https://news\.ycombinator\.com/item\?id=\d+') |
| 17 |
HN_COMMENTS_URL_RE = re.compile(r'Comments URL:\s*<a [^>]*href="(https://news\.ycombinator\.com/item\?id=\d+)"') |
| 18 |
|
| 19 |
|
| 20 |
class HNWithCommentsLink(BasicNewsRecipe): |
| 21 |
title = 'HN With Actual Comments' |
| 22 |
__author__ = 'Tom Scholl & David Kerschner' |
| 23 |
description = 'Hacker News, run by Y Combinator. Anything that good hackers would find interesting, with a focus on programming and startups.' |
| 24 |
publisher = 'Y Combinator' |
| 25 |
category = 'news, programming, it, technology' |
| 26 |
delay = 1 |
| 27 |
max_articles_per_feed = 20 |
| 28 |
oldest_article = 3 |
| 29 |
use_embedded_content = False |
| 30 |
no_stylesheets = True |
| 31 |
encoding = 'utf-8' |
| 32 |
language = 'en' |
| 33 |
requires_version = (0, 8, 16) |
| 34 |
|
| 35 |
feeds = [('Hacker News Frontpage', 'https://hnrss.org/frontpage'), ('Ask Hacker News', 'https://hnrss.org/ask')] |
| 36 |
|
| 37 |
temp_files = [] |
| 38 |
articles_are_obfuscated = True |
| 39 |
|
| 40 |
def _find_hn_url(self, article): |
| 41 |
# hnrss.org sets <guid> to the HN thread URL, so article.id is normally |
| 42 |
# that URL. Fall back to scraping the description HTML for |
| 43 |
# "Comments URL: <a href='...'>" if the id isn't populated. |
| 44 |
aid = getattr(article, 'id', None) |
| 45 |
if aid and isinstance(aid, str): |
| 46 |
m = HN_URL_RE.match(aid) |
| 47 |
if m: |
| 48 |
return m.group(0) |
| 49 |
for attr in ('summary', 'text_summary', 'content'): |
| 50 |
text = getattr(article, attr, None) |
| 51 |
if text: |
| 52 |
m = HN_COMMENTS_URL_RE.search(text) |
| 53 |
if m: |
| 54 |
return m.group(1) |
| 55 |
m = HN_URL_RE.search(text) |
| 56 |
if m: |
| 57 |
return m.group(0) |
| 58 |
return None |
| 59 |
|
| 60 |
def parse_feeds(self): |
| 61 |
feeds = super().parse_feeds() |
| 62 |
# Build a map from article URL to HN thread URL so that when the article |
| 63 |
# <link> points to an external site we can still fetch the HN comments. |
| 64 |
self.url_to_comments = {} |
| 65 |
for feed in feeds: |
| 66 |
for article in feed.articles: |
| 67 |
hn_url = self._find_hn_url(article) |
| 68 |
if hn_url and article.url != hn_url: |
| 69 |
self.url_to_comments[article.url] = hn_url |
| 70 |
if feeds: |
| 71 |
self.hn_articles = feeds[0].articles |
| 72 |
return feeds |
| 73 |
|
| 74 |
def get_readable_content(self, url): |
| 75 |
self.log('get_readable_content(' + url + ')') |
| 76 |
br = self.get_browser() |
| 77 |
f = br.open(url) |
| 78 |
html = f.read() |
| 79 |
f.close() |
| 80 |
return self.extract_readable_article(html, url) |
| 81 |
|
| 82 |
def _extract_comments_from_soup(self, main): |
| 83 |
comments = '' |
| 84 |
for td in main.findAll('td', 'default'): |
| 85 |
comhead = td.find('span', 'comhead') |
| 86 |
if comhead: |
| 87 |
com_title = '<h4>' + self.tag_to_string(comhead).replace(' | link', '') + '</h4>' |
| 88 |
comhead.parent.extract() |
| 89 |
br = td.find('br') |
| 90 |
if br: |
| 91 |
br.extract() |
| 92 |
reply = td.find('a', attrs={'href': re.compile(r'^reply?')}) |
| 93 |
if reply: |
| 94 |
reply.parent.extract() |
| 95 |
td.name = 'div' |
| 96 |
indent_width = (int(td.parent.find('td').img['width']) * 2) / 3 |
| 97 |
td['style'] = 'padding-left: ' + str(indent_width) + 'px' |
| 98 |
comments = comments + com_title + td.prettify() |
| 99 |
return comments |
| 100 |
|
| 101 |
def get_hn_comments_html(self, url): |
| 102 |
# Comments-only fragment, used when the article's main content comes |
| 103 |
# from an external site and we want to append the HN discussion. |
| 104 |
self.log('get_hn_comments_html(' + url + ')') |
| 105 |
soup = self.index_to_soup(url) |
| 106 |
main = soup.find('tr').findNextSiblings('tr', limit=2)[1].td |
| 107 |
subtext = self.tag_to_string(main.find('td', 'subtext')) |
| 108 |
comments = self._extract_comments_from_soup(main) |
| 109 |
return '<hr/><h2>Hacker News Discussion</h2>' + '<p><em>' + subtext + '</em></p>' + comments |
| 110 |
|
| 111 |
def get_hn_content(self, url): |
| 112 |
self.log('get_hn_content(' + url + ')') |
| 113 |
soup = self.index_to_soup(url) |
| 114 |
main = soup.find('tr').findNextSiblings('tr', limit=2)[1].td |
| 115 |
|
| 116 |
title_element = main.select('td.title .titleline a')[0] |
| 117 |
title = self.tag_to_string(title_element) |
| 118 |
link = title_element['href'] |
| 119 |
if link.startswith('item?'): |
| 120 |
link = 'https://news.ycombinator.com/' + link |
| 121 |
readable_link = link.rpartition('http://')[2].rpartition('https://')[2] |
| 122 |
|
| 123 |
subtext = self.tag_to_string(main.find('td', 'subtext')) |
| 124 |
|
| 125 |
title_content_td = main.find('td', 'title').findParent('tr').findNextSiblings('tr', limit=3)[2].findAll('td', limit=2)[1] |
| 126 |
title_content = '' |
| 127 |
if not title_content_td.find('form'): |
| 128 |
title_content_td.name = 'div' |
| 129 |
title_content = title_content_td.prettify() |
| 130 |
|
| 131 |
comments = self._extract_comments_from_soup(main) |
| 132 |
body = ( |
| 133 |
'<h3>' |
| 134 |
+ title |
| 135 |
+ '</h3><p><a href="' |
| 136 |
+ link |
| 137 |
+ '">' |
| 138 |
+ readable_link |
| 139 |
+ '</a><br/><strong>' |
| 140 |
+ subtext |
| 141 |
+ '</strong></p>' |
| 142 |
+ title_content |
| 143 |
+ '<hr/><h3>Comments</h3>' |
| 144 |
+ comments |
| 145 |
) |
| 146 |
return '<html><title>' + title + '</title><body>' + body + '</body></html>' |
| 147 |
|
| 148 |
def get_obfuscated_article(self, url): |
| 149 |
self.log('get_obfuscated_article with url=' + url) |
| 150 |
if url.startswith('https://news.ycombinator.com'): |
| 151 |
content = self.get_hn_content(url) |
| 152 |
else: |
| 153 |
is_image = False |
| 154 |
for ext in ['.jpg', '.png', '.svg', '.gif', '.jpeg', '.tiff', '.bmp']: |
| 155 |
if url.endswith(ext): |
| 156 |
is_image = True |
| 157 |
break |
| 158 |
|
| 159 |
if is_image: |
| 160 |
self.log('using image_content (' + url + ')') |
| 161 |
content = '<html><body><img src="' + url + '"></body></html>' |
| 162 |
else: |
| 163 |
content = self.get_readable_content(url) |
| 164 |
|
| 165 |
hn_url = self.url_to_comments.get(url) |
| 166 |
if hn_url: |
| 167 |
comments_html = self.get_hn_comments_html(hn_url) |
| 168 |
if isinstance(content, bytes): |
| 169 |
content = content.decode('utf-8', 'replace') |
| 170 |
if '</body>' in content: |
| 171 |
content = content.replace('</body>', comments_html + '</body>', 1) |
| 172 |
else: |
| 173 |
content = content + comments_html |
| 174 |
|
| 175 |
if not isinstance(content, bytes): |
| 176 |
content = content.encode('utf-8') |
| 177 |
self.temp_files.append(PersistentTemporaryFile('_fa.html')) |
| 178 |
self.temp_files[-1].write(content) |
| 179 |
self.temp_files[-1].close() |
| 180 |
return self.temp_files[-1].name |
| 181 |
|
| 182 |
def is_link_wanted(self, url, tag): |
| 183 |
if url.endswith('.pdf'): |
| 184 |
return False |
| 185 |
return True |
| 186 |
|
| 187 |
def prettyify_url(self, url): |
| 188 |
return urlparse(url).hostname |
| 189 |
|
| 190 |
def populate_article_metadata(self, article, soup, first): |
| 191 |
article.text_summary = self.prettyify_url(article.url) |
| 192 |
article.summary = article.text_summary |