Debian Codegraph @

recipes/hackernews.recipe

raw
1 #!/usr/bin/env python
2
3 __license__ = 'GPL v3'
4 '''
5 Hacker News
6 '''
7 from calibre.ptempfile import PersistentTemporaryFile
8 from calibre.web.feeds.news import BasicNewsRecipe
9
10 try:
11 from urllib.parse import urlparse
12 except ImportError:
13 from urlparse import urlparse
14 import re
15
16 HN_URL_RE = re.compile(r'https://news\.ycombinator\.com/item\?id=\d+')
17 HN_COMMENTS_URL_RE = re.compile(r'Comments URL:\s*<a [^>]*href="(https://news\.ycombinator\.com/item\?id=\d+)"')
18
19
20 class HNWithCommentsLink(BasicNewsRecipe):
21 title = 'HN With Actual Comments'
22 __author__ = 'Tom Scholl & David Kerschner'
23 description = 'Hacker News, run by Y Combinator. Anything that good hackers would find interesting, with a focus on programming and startups.'
24 publisher = 'Y Combinator'
25 category = 'news, programming, it, technology'
26 delay = 1
27 max_articles_per_feed = 20
28 oldest_article = 3
29 use_embedded_content = False
30 no_stylesheets = True
31 encoding = 'utf-8'
32 language = 'en'
33 requires_version = (0, 8, 16)
34
35 feeds = [('Hacker News Frontpage', 'https://hnrss.org/frontpage'), ('Ask Hacker News', 'https://hnrss.org/ask')]
36
37 temp_files = []
38 articles_are_obfuscated = True
39
40 def _find_hn_url(self, article):
41 # hnrss.org sets <guid> to the HN thread URL, so article.id is normally
42 # that URL. Fall back to scraping the description HTML for
43 # "Comments URL: <a href='...'>" if the id isn't populated.
44 aid = getattr(article, 'id', None)
45 if aid and isinstance(aid, str):
46 m = HN_URL_RE.match(aid)
47 if m:
48 return m.group(0)
49 for attr in ('summary', 'text_summary', 'content'):
50 text = getattr(article, attr, None)
51 if text:
52 m = HN_COMMENTS_URL_RE.search(text)
53 if m:
54 return m.group(1)
55 m = HN_URL_RE.search(text)
56 if m:
57 return m.group(0)
58 return None
59
60 def parse_feeds(self):
61 feeds = super().parse_feeds()
62 # Build a map from article URL to HN thread URL so that when the article
63 # <link> points to an external site we can still fetch the HN comments.
64 self.url_to_comments = {}
65 for feed in feeds:
66 for article in feed.articles:
67 hn_url = self._find_hn_url(article)
68 if hn_url and article.url != hn_url:
69 self.url_to_comments[article.url] = hn_url
70 if feeds:
71 self.hn_articles = feeds[0].articles
72 return feeds
73
74 def get_readable_content(self, url):
75 self.log('get_readable_content(' + url + ')')
76 br = self.get_browser()
77 f = br.open(url)
78 html = f.read()
79 f.close()
80 return self.extract_readable_article(html, url)
81
82 def _extract_comments_from_soup(self, main):
83 comments = ''
84 for td in main.findAll('td', 'default'):
85 comhead = td.find('span', 'comhead')
86 if comhead:
87 com_title = '<h4>' + self.tag_to_string(comhead).replace(' | link', '') + '</h4>'
88 comhead.parent.extract()
89 br = td.find('br')
90 if br:
91 br.extract()
92 reply = td.find('a', attrs={'href': re.compile(r'^reply?')})
93 if reply:
94 reply.parent.extract()
95 td.name = 'div'
96 indent_width = (int(td.parent.find('td').img['width']) * 2) / 3
97 td['style'] = 'padding-left: ' + str(indent_width) + 'px'
98 comments = comments + com_title + td.prettify()
99 return comments
100
101 def get_hn_comments_html(self, url):
102 # Comments-only fragment, used when the article's main content comes
103 # from an external site and we want to append the HN discussion.
104 self.log('get_hn_comments_html(' + url + ')')
105 soup = self.index_to_soup(url)
106 main = soup.find('tr').findNextSiblings('tr', limit=2)[1].td
107 subtext = self.tag_to_string(main.find('td', 'subtext'))
108 comments = self._extract_comments_from_soup(main)
109 return '<hr/><h2>Hacker News Discussion</h2>' + '<p><em>' + subtext + '</em></p>' + comments
110
111 def get_hn_content(self, url):
112 self.log('get_hn_content(' + url + ')')
113 soup = self.index_to_soup(url)
114 main = soup.find('tr').findNextSiblings('tr', limit=2)[1].td
115
116 title_element = main.select('td.title .titleline a')[0]
117 title = self.tag_to_string(title_element)
118 link = title_element['href']
119 if link.startswith('item?'):
120 link = 'https://news.ycombinator.com/' + link
121 readable_link = link.rpartition('http://')[2].rpartition('https://')[2]
122
123 subtext = self.tag_to_string(main.find('td', 'subtext'))
124
125 title_content_td = main.find('td', 'title').findParent('tr').findNextSiblings('tr', limit=3)[2].findAll('td', limit=2)[1]
126 title_content = ''
127 if not title_content_td.find('form'):
128 title_content_td.name = 'div'
129 title_content = title_content_td.prettify()
130
131 comments = self._extract_comments_from_soup(main)
132 body = (
133 '<h3>'
134 + title
135 + '</h3><p><a href="'
136 + link
137 + '">'
138 + readable_link
139 + '</a><br/><strong>'
140 + subtext
141 + '</strong></p>'
142 + title_content
143 + '<hr/><h3>Comments</h3>'
144 + comments
145 )
146 return '<html><title>' + title + '</title><body>' + body + '</body></html>'
147
148 def get_obfuscated_article(self, url):
149 self.log('get_obfuscated_article with url=' + url)
150 if url.startswith('https://news.ycombinator.com'):
151 content = self.get_hn_content(url)
152 else:
153 is_image = False
154 for ext in ['.jpg', '.png', '.svg', '.gif', '.jpeg', '.tiff', '.bmp']:
155 if url.endswith(ext):
156 is_image = True
157 break
158
159 if is_image:
160 self.log('using image_content (' + url + ')')
161 content = '<html><body><img src="' + url + '"></body></html>'
162 else:
163 content = self.get_readable_content(url)
164
165 hn_url = self.url_to_comments.get(url)
166 if hn_url:
167 comments_html = self.get_hn_comments_html(hn_url)
168 if isinstance(content, bytes):
169 content = content.decode('utf-8', 'replace')
170 if '</body>' in content:
171 content = content.replace('</body>', comments_html + '</body>', 1)
172 else:
173 content = content + comments_html
174
175 if not isinstance(content, bytes):
176 content = content.encode('utf-8')
177 self.temp_files.append(PersistentTemporaryFile('_fa.html'))
178 self.temp_files[-1].write(content)
179 self.temp_files[-1].close()
180 return self.temp_files[-1].name
181
182 def is_link_wanted(self, url, tag):
183 if url.endswith('.pdf'):
184 return False
185 return True
186
187 def prettyify_url(self, url):
188 return urlparse(url).hostname
189
190 def populate_article_metadata(self, article, soup, first):
191 article.text_summary = self.prettyify_url(article.url)
192 article.summary = article.text_summary