#!/usr/bin/env python __license__ = 'GPL v3' ''' Hacker News ''' from calibre.ptempfile import PersistentTemporaryFile from calibre.web.feeds.news import BasicNewsRecipe try: from urllib.parse import urlparse except ImportError: from urlparse import urlparse import re HN_URL_RE = re.compile(r'https://news\.ycombinator\.com/item\?id=\d+') HN_COMMENTS_URL_RE = re.compile(r'Comments URL:\s*]*href="(https://news\.ycombinator\.com/item\?id=\d+)"') class HNWithCommentsLink(BasicNewsRecipe): title = 'HN With Actual Comments' __author__ = 'Tom Scholl & David Kerschner' description = 'Hacker News, run by Y Combinator. Anything that good hackers would find interesting, with a focus on programming and startups.' publisher = 'Y Combinator' category = 'news, programming, it, technology' delay = 1 max_articles_per_feed = 20 oldest_article = 3 use_embedded_content = False no_stylesheets = True encoding = 'utf-8' language = 'en' requires_version = (0, 8, 16) feeds = [('Hacker News Frontpage', 'https://hnrss.org/frontpage'), ('Ask Hacker News', 'https://hnrss.org/ask')] temp_files = [] articles_are_obfuscated = True def _find_hn_url(self, article): # hnrss.org sets to the HN thread URL, so article.id is normally # that URL. Fall back to scraping the description HTML for # "Comments URL: " if the id isn't populated. aid = getattr(article, 'id', None) if aid and isinstance(aid, str): m = HN_URL_RE.match(aid) if m: return m.group(0) for attr in ('summary', 'text_summary', 'content'): text = getattr(article, attr, None) if text: m = HN_COMMENTS_URL_RE.search(text) if m: return m.group(1) m = HN_URL_RE.search(text) if m: return m.group(0) return None def parse_feeds(self): feeds = super().parse_feeds() # Build a map from article URL to HN thread URL so that when the article # points to an external site we can still fetch the HN comments. self.url_to_comments = {} for feed in feeds: for article in feed.articles: hn_url = self._find_hn_url(article) if hn_url and article.url != hn_url: self.url_to_comments[article.url] = hn_url if feeds: self.hn_articles = feeds[0].articles return feeds def get_readable_content(self, url): self.log('get_readable_content(' + url + ')') br = self.get_browser() f = br.open(url) html = f.read() f.close() return self.extract_readable_article(html, url) def _extract_comments_from_soup(self, main): comments = '' for td in main.findAll('td', 'default'): comhead = td.find('span', 'comhead') if comhead: com_title = '

' + self.tag_to_string(comhead).replace(' | link', '') + '

' comhead.parent.extract() br = td.find('br') if br: br.extract() reply = td.find('a', attrs={'href': re.compile(r'^reply?')}) if reply: reply.parent.extract() td.name = 'div' indent_width = (int(td.parent.find('td').img['width']) * 2) / 3 td['style'] = 'padding-left: ' + str(indent_width) + 'px' comments = comments + com_title + td.prettify() return comments def get_hn_comments_html(self, url): # Comments-only fragment, used when the article's main content comes # from an external site and we want to append the HN discussion. self.log('get_hn_comments_html(' + url + ')') soup = self.index_to_soup(url) main = soup.find('tr').findNextSiblings('tr', limit=2)[1].td subtext = self.tag_to_string(main.find('td', 'subtext')) comments = self._extract_comments_from_soup(main) return '

Hacker News Discussion

' + '

' + subtext + '

' + comments def get_hn_content(self, url): self.log('get_hn_content(' + url + ')') soup = self.index_to_soup(url) main = soup.find('tr').findNextSiblings('tr', limit=2)[1].td title_element = main.select('td.title .titleline a')[0] title = self.tag_to_string(title_element) link = title_element['href'] if link.startswith('item?'): link = 'https://news.ycombinator.com/' + link readable_link = link.rpartition('http://')[2].rpartition('https://')[2] subtext = self.tag_to_string(main.find('td', 'subtext')) title_content_td = main.find('td', 'title').findParent('tr').findNextSiblings('tr', limit=3)[2].findAll('td', limit=2)[1] title_content = '' if not title_content_td.find('form'): title_content_td.name = 'div' title_content = title_content_td.prettify() comments = self._extract_comments_from_soup(main) body = ( '

' + title + '

' + readable_link + '
' + subtext + '

' + title_content + '

Comments

' + comments ) return '' + title + '' + body + '' def get_obfuscated_article(self, url): self.log('get_obfuscated_article with url=' + url) if url.startswith('https://news.ycombinator.com'): content = self.get_hn_content(url) else: is_image = False for ext in ['.jpg', '.png', '.svg', '.gif', '.jpeg', '.tiff', '.bmp']: if url.endswith(ext): is_image = True break if is_image: self.log('using image_content (' + url + ')') content = '' else: content = self.get_readable_content(url) hn_url = self.url_to_comments.get(url) if hn_url: comments_html = self.get_hn_comments_html(hn_url) if isinstance(content, bytes): content = content.decode('utf-8', 'replace') if '' in content: content = content.replace('', comments_html + '', 1) else: content = content + comments_html if not isinstance(content, bytes): content = content.encode('utf-8') self.temp_files.append(PersistentTemporaryFile('_fa.html')) self.temp_files[-1].write(content) self.temp_files[-1].close() return self.temp_files[-1].name def is_link_wanted(self, url, tag): if url.endswith('.pdf'): return False return True def prettyify_url(self, url): return urlparse(url).hostname def populate_article_metadata(self, article, soup, first): article.text_summary = self.prettyify_url(article.url) article.summary = article.text_summary