Debian Codegraph @

recipes/todoist.recipe

raw
1 #!/usr/bin/env python
2 # vim:ft=python tabstop=8 expandtab shiftwidth=4 softtabstop=4
3 __license__ = 'GPL v3'
4 __copyright__ = '2025, rga5321'
5
6
7 import json
8 import re
9 from datetime import datetime
10 from time import strftime
11 from urllib.parse import urlparse
12
13 import mechanize
14
15 from calibre.web.feeds.news import BasicNewsRecipe
16
17 __version__ = '0.0.4'
18
19 '''
20 0.0.4: Migrating to new Todoist unified API v1
21 0.0.3: Parameters in recipe_specific_options
22 0.0.2: Calibre footer with the source URL. QR points to the article URL.
23 0.0.1: First working version
24
25 # Calibre parameters
26
27 Input them in command line as this example::
28 ebook-convert Todoist.recipe output.epub --recipe-specific-option=ARCHIVE_DOWNLOADED:False \
29 --recipe-specific-option=TODOIST_PROJECT_ID:YOUR_PROJECT_ID \
30 --recipe-specific-option=TODOIST_API_KEY:YOUR_API_KEY \
31 --recipe-specific-option=URL_KEYWORD_EXCEPTIONS:jotdown,elpais.com/gastronomia
32
33
34 **URL_KEYWORD_EXCEPTIONS** (list of keywords such as, if the URL of the article contains any keyword, then the plugin will ignore the article)
35
36 **ARCHIVE_DOWNLOADED** (True or False) do you want to archive articles after fetching
37
38 **TODOIST_PROJECT_ID** (string) your Todoist project ID, you can find it in the URL of your Todoist project, e.g. https://todoist.com/app/project/1234567890abcdef12345678
39
40 **TODOIST_API_KEY** (string) your Todoist API key, you can find it in your Todoist account settings under "Integrations" or "API tokens"
41 '''
42 # CONFIGURATION ###########################################################
43
44 import ast
45
46
47 # Aux funcion. String to boolean
48 def parse_env_bool(val):
49 return str(val).strip().lower() in ('true', '1', 'yes')
50
51
52 # Aux funcion. comma separated String to List
53 def parse_env_list(val):
54 try:
55 return ast.literal_eval(val)
56 except Exception:
57 return []
58
59
60 #############################################################################
61
62
63 class Todoist2ebook(BasicNewsRecipe):
64 recipe_specific_options = {
65 'ARCHIVE_DOWNLOADED': {
66 'short': 'Mark as read',
67 'long': 'Mark as read',
68 'default': False,
69 },
70 'TODOIST_PROJECT_ID': {'short': 'Proyect ID', 'long': 'Proyect ID'},
71 'TODOIST_API_KEY': {'short': 'API key', 'long': 'API KEY'},
72 'URL_KEYWORD_EXCEPTIONS': {
73 'short': 'URL keyword exceptions',
74 'long': 'List of keywords to ignore articles, e.g. ["example.com", "ignoreme.com"]',
75 'default': [],
76 },
77 }
78
79 __author__ = 'rga5321'
80 description = (
81 'Calibre recipe to fetch articles from a Todoist project. The recipe is available in Calibre and in'
82 ' the original repository https://github.com/rga5321/todoist2ebook, where an script to automate'
83 ' the ebook conversion and email sending is also available.'
84 )
85 publisher = 'Todoist.com'
86 category = 'info, custom, Todoist'
87
88 # User-configurable settings -----------------------------------------------
89
90 series_name = 'Todoist'
91 publication_type = 'magazine'
92 title = 'Todoist'
93 # timefmt = '' # uncomment to remove date from the filenames, if commented then you will get something like `Todoist [Wed, 13 May 2020]`
94 masthead_url = 'https://raw.githubusercontent.com/rga5321/todoist2ebook/master/img/todoist-logo.png'
95 # will make square cover; this will replace text and cover of the default
96 cover_url = 'https://raw.githubusercontent.com/rga5321/todoist2ebook/master/img/todoist-cover.png'
97 # --------------------------------------------------------------------------
98
99 # Inherited developer settings
100 auto_cleanup = True
101 no_stylesheets = True
102 use_embedded_content = False
103
104 # Custom developer settings
105 to_archive = []
106
107 simultaneous_downloads = 10
108
109 extra_css = '.calibre_navbar { visibility: hidden; }'
110
111 def __init__(self, *args, **kwargs):
112 super().__init__(*args, **kwargs)
113
114 # Init optional configuration parameters
115 self.archive_downloaded = parse_env_bool(self.recipe_specific_options['ARCHIVE_DOWNLOADED'])
116 self.keyword_exceptions = parse_env_list(self.recipe_specific_options['URL_KEYWORD_EXCEPTIONS'])
117
118 # Init mandatory configuration parameters
119 if self.recipe_specific_options.get('TODOIST_PROJECT_ID'):
120 self.todoist_project_id = self.recipe_specific_options['TODOIST_PROJECT_ID']
121 else:
122 self.abort_recipe_processing('TODOIST_PROJECT_ID mandatory parameter missing')
123
124 if self.recipe_specific_options.get('TODOIST_API_KEY'):
125 self.todoist_api_key = self.recipe_specific_options['TODOIST_API_KEY']
126 else:
127 self.abort_recipe_processing('TODOIST_API_KEY mandatory parameter missing')
128
129 def parse_index(self):
130
131 articles = []
132 section_dict = {} # dictionary with the domains and its articles.
133
134 url = f'https://api.todoist.com/api/v1/tasks?project_id={self.todoist_project_id}'
135 headers = {'Authorization': f'Bearer {self.todoist_api_key}'}
136 request = mechanize.Request(url, headers=headers)
137
138 response = self.browser.open(request)
139 if response.code != 200:
140 raise Exception('Could not retrieve Todoist tasks')
141 data = response.read().decode('utf-8')
142 tasks = json.loads(data)
143 if isinstance(tasks, dict) and 'results' in tasks:
144 tasks = tasks['results']
145 else:
146 self.abort_recipe_processing(
147 'Unexpected response format from Todoist API. '
148 'Expected {"results": [...]}, got keys: %s' % str(tasks.keys() if isinstance(tasks, dict) else type(tasks))
149 )
150 articles_todoist = []
151
152 url_regex = re.compile(r'\[([^\]]+)\]\(\s*(https?://[^\s\)]+)\s*\)')
153 for task in tasks:
154 match = url_regex.search(task['content'])
155 if match:
156 title = match.group(1).strip()
157 url = match.group(2).strip()
158 date_added = task.get('created_at', datetime.now().isoformat())
159 articles_todoist.append({
160 'title': title or url,
161 'url': url,
162 'date_added': date_added,
163 'item_id': task['id'],
164 })
165
166 if not articles_todoist:
167 self.abort_recipe_processing('No unread articles in the Todoist project "{}"'.format(self.todoist_project_id))
168 else:
169 for item in articles_todoist:
170 # If the URL contains any URL_KEYWORD_EXCEPTIONS, ignore article
171 if any(pattern in item['url'] for pattern in self.keyword_exceptions):
172 print('Ignoring article due to keyword patterns:' + item['url'])
173 del item
174 else:
175 # Extract domain from the URL
176 domain = urlparse(item['url']).netloc.replace('www.', '')
177
178 url = item['url']
179
180 # Add the article under its domain
181 if domain not in section_dict:
182 section_dict[domain] = [item]
183 else:
184 section_dict[domain].append(item)
185
186 print('Adding article: ' + item['url'] + ' to section: ' + domain)
187
188 ############ APPEND ARTS FOR EACH DOMAIN #############
189 # At this point the section_dict is completed
190
191 for section in section_dict:
192 arts = []
193 for item in section_dict.get(section):
194 try:
195 title = item['title']
196 except KeyError:
197 title = 'error: title'
198 try:
199 url = item['url']
200 except KeyError:
201 url = 'error: url'
202
203 arts.append({'title': title, 'url': url, 'date': item['date_added']})
204
205 if self.archive_downloaded and item['item_id'] not in self.to_archive:
206 self.to_archive.append(item['item_id'])
207
208 if arts:
209 articles.append((section, arts))
210
211 if not articles:
212 self.abort_recipe_processing('No articles in the Todoist project account %s to download' % (self.todoist_project_id))
213 return articles
214
215 def get_browser(self, *args, **kwargs):
216 self.browser = BasicNewsRecipe.get_browser(self)
217 return self.browser
218
219 def cleanup(self):
220 if not self.to_archive:
221 return
222
223 for task_id in self.to_archive:
224 url = f'https://api.todoist.com/api/v1/tasks/{task_id}/close'
225 req = mechanize.Request(
226 url,
227 headers={
228 'Authorization': f'Bearer {self.todoist_api_key}',
229 'Content-Type': 'application/json',
230 },
231 )
232 req.get_method = lambda: 'POST'
233
234 try:
235 br = mechanize.Browser()
236 response = br.open(req)
237 if response.code == 204:
238 print(f'Task {task_id} corectly closed.')
239 else:
240 print(f'Error while closing task {task_id}: {response.code}')
241 except Exception as e:
242 print(f'Exception while closing task {task_id}: {e}')
243
244 # TODO: This works with EPUB, but not mobi/azw3
245 # BUG: https://bugs.launchpad.net/calibre/+bug/1838486
246 def postprocess_book(self, oeb, opts, log):
247 oeb.metadata.add('series', self.series_name)
248
249 def _postprocess_html(self, soup, first_fetch, job_info):
250
251 title = soup.find('title').text # get title
252
253 h1s = soup.findAll('h1') # get all h1 headers
254 for h1 in h1s:
255 if title in h1.text:
256 h1 = h1.clear() # clean this tag, so the h1 will be there only
257
258 h2s = soup.findAll('h2') # get all h2 headers
259 for h2 in h2s:
260 if title in h2.text:
261 h2 = h2.clear() # clean this tag, so the h1 will be there only
262
263 body = soup.find('body')
264 new_tag = soup.new_tag('h1')
265 new_tag.append(title)
266 body.insert(0, new_tag)
267
268 return soup
269
270 def default_cover(self, cover_file):
271 """
272 Create a generic cover for recipes that don't have a cover
273 This override adds time to the cover
274 """
275 try:
276 from calibre.ebooks import calibre_cover
277
278 title = self.title
279 date = strftime(self.timefmt)
280 time = strftime('%a %d %b %Y %-H:%M')
281 img_data = calibre_cover(title, date, time)
282 cover_file.write(img_data)
283 cover_file.flush()
284 except Exception:
285 self.log.exception('Failed to generate default cover')
286 return False
287 return True