Debian Codegraph @

recipes/substack.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 #
4 # Title: Substack
5 # License: GNU General Public License v3 – https://www.gnu.org/licenses/gpl-3.0.html
6 # Copyright: Nathan Cook ([email protected])
7 ##
8 # Written: 2020-12-18
9 # Updated: 2024-11-04
10 ##
11
12 __license__ = 'GNU General Public License v3 – https://www.gnu.org/licenses/gpl-3.0.html'
13 __copyright__ = 'Nathan Cook – 2020-12-19'
14 __version__ = 'v0.1.1'
15 __date__ = '2020-12-19'
16 __author__ = 'topynate'
17
18 import json
19 import re
20
21 from mechanize import Request
22
23 from calibre.web.feeds.news import BasicNewsRecipe, classes
24
25
26 class Substack(BasicNewsRecipe):
27 title = 'Substack'
28 __author__ = 'topynate, unkn0wn'
29 description = 'Use advanced menu if you want to add your own substack handles.'
30 oldest_article = 7
31 language = 'en'
32 max_articles_per_feed = 100
33 needs_subscription = 'optional'
34 use_embedded_content = False
35 masthead_url = 'https://substack.com/img/substack_wordmark.png'
36 cover_url = 'https://substack.com/img/substack.png'
37 extra_css = '''
38 .captioned-image-container, .image-container, .image-caption {font-size: small;}
39 pre {font-family: inherit; white-space: pre-wrap;}
40 '''
41 remove_empty_feeds = True
42 remove_attributes = ['style', 'height', 'width']
43 no_stylesheets = True
44
45 keep_only_tags = [classes('post-title post-subtitle subtitle available-content')]
46
47 remove_tags = [dict(name=['svg', 'source']), classes('subscribe-widget button-wrapper')]
48
49 recipe_specific_options = {
50 'auths': {
51 'short': 'enter the @handles you subscribe to:\nseperated by a space',
52 'long': '@julianmacfarlane @simplicius76 .... ....',
53 'default': '@julianmacfarlane @simplicius76 @caitlinjohnstone @michaelmoore @seymourhersh @geopolitiq',
54 },
55 'days': {
56 'short': 'Oldest article to download from this news source. In days ',
57 'long': 'For example, 0.5, gives you articles from the past 12 hours',
58 'default': str(oldest_article),
59 },
60 'res': {
61 'short': 'For hi-res images, select a resolution from the\nfollowing options: 800, 1000, 1200 or 1500',
62 'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use 400 or 300.',
63 'default': '600',
64 },
65 'rev': {
66 'short': 'Reverse the order of articles in each feed?',
67 'long': 'enter yes',
68 'default': 'no',
69 },
70 }
71
72 def __init__(self, *args, **kwargs):
73 BasicNewsRecipe.__init__(self, *args, **kwargs)
74 d = self.recipe_specific_options.get('days')
75 if d and isinstance(d, str):
76 self.oldest_article = float(d)
77 r = self.recipe_specific_options.get('rev')
78 if r and isinstance(r, str):
79 if r.lower().strip() == 'yes':
80 self.reverse_article_order = True
81
82 # Every Substack publication has an RSS feed at https://{name}.substack.com/feed.
83 # The same URL provides either all posts, or all free posts + previews of paid posts,
84 # depending on whether you're logged in.
85 # feeds = [
86 # ('Novum Lumen', 'https://novumlumen.substack.com/feed'), # gratuitously self-promotional example
87 # ]
88
89 def get_browser(self):
90 br = BasicNewsRecipe.get_browser(self)
91 if self.username is not None and self.password is not None:
92 br.open('https://substack.com/account/login?redirect=%2F&email=&with_password=')
93 data = json.dumps({'email': self.username, 'password': self.password, 'captcha_response': None})
94 req = Request(
95 url='https://substack.com/api/v1/email-login',
96 headers={
97 'Accept': '*/*',
98 'Content-Type': 'application/json',
99 'Origin': 'https://substack.com',
100 'Referer': 'https://substack.com/account/login?redirect=%2F&email=&with_password=',
101 },
102 data=data,
103 method='POST',
104 )
105 res = br.open(req)
106 if res.getcode() != 200:
107 raise ValueError('Login failed, check username and password')
108 return br
109
110 def get_feeds(self):
111 ans = []
112 u = self.recipe_specific_options.get('auths')
113 if u and isinstance(u, str):
114 for x in u.replace('@', ' ').split():
115 ans.append('https://' + x + '.substack.com/feed')
116 return ans
117
118 def preprocess_html(self, soup):
119 # Substack's poetry/preformatted blocks contain a hidden editor tooltip
120 # ("Text within this block will maintain its original spacing when
121 # published") that becomes visible once site CSS is stripped.
122 for tooltip in soup.findAll(string=lambda s: s and 'maintain its original spacing' in s):
123 tooltip.extract()
124 res = '600'
125 w = self.recipe_specific_options.get('res')
126 if w and isinstance(w, str):
127 res = w
128 for img in soup.findAll('img', attrs={'src': True}):
129 img['src'] = re.sub(r'w_\d+', 'w_' + res, img['src'])
130 return soup