| 1 |
#!/usr/bin/env python |
| 2 |
# vim:ft=python tabstop=8 expandtab shiftwidth=4 softtabstop=4 |
| 3 |
__license__ = 'GPL v3' |
| 4 |
__copyright__ = '2025, rga5321' |
| 5 |
|
| 6 |
|
| 7 |
import json |
| 8 |
import re |
| 9 |
from datetime import datetime |
| 10 |
from time import strftime |
| 11 |
from urllib.parse import urlparse |
| 12 |
|
| 13 |
import mechanize |
| 14 |
|
| 15 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 16 |
|
| 17 |
__version__ = '0.0.4' |
| 18 |
|
| 19 |
''' |
| 20 |
0.0.4: Migrating to new Todoist unified API v1 |
| 21 |
0.0.3: Parameters in recipe_specific_options |
| 22 |
0.0.2: Calibre footer with the source URL. QR points to the article URL. |
| 23 |
0.0.1: First working version |
| 24 |
|
| 25 |
# Calibre parameters |
| 26 |
|
| 27 |
Input them in command line as this example:: |
| 28 |
ebook-convert Todoist.recipe output.epub --recipe-specific-option=ARCHIVE_DOWNLOADED:False \ |
| 29 |
--recipe-specific-option=TODOIST_PROJECT_ID:YOUR_PROJECT_ID \ |
| 30 |
--recipe-specific-option=TODOIST_API_KEY:YOUR_API_KEY \ |
| 31 |
--recipe-specific-option=URL_KEYWORD_EXCEPTIONS:jotdown,elpais.com/gastronomia |
| 32 |
|
| 33 |
|
| 34 |
**URL_KEYWORD_EXCEPTIONS** (list of keywords such as, if the URL of the article contains any keyword, then the plugin will ignore the article) |
| 35 |
|
| 36 |
**ARCHIVE_DOWNLOADED** (True or False) do you want to archive articles after fetching |
| 37 |
|
| 38 |
**TODOIST_PROJECT_ID** (string) your Todoist project ID, you can find it in the URL of your Todoist project, e.g. https://todoist.com/app/project/1234567890abcdef12345678 |
| 39 |
|
| 40 |
**TODOIST_API_KEY** (string) your Todoist API key, you can find it in your Todoist account settings under "Integrations" or "API tokens" |
| 41 |
''' |
| 42 |
# CONFIGURATION ########################################################### |
| 43 |
|
| 44 |
import ast |
| 45 |
|
| 46 |
|
| 47 |
# Aux funcion. String to boolean |
| 48 |
def parse_env_bool(val): |
| 49 |
return str(val).strip().lower() in ('true', '1', 'yes') |
| 50 |
|
| 51 |
|
| 52 |
# Aux funcion. comma separated String to List |
| 53 |
def parse_env_list(val): |
| 54 |
try: |
| 55 |
return ast.literal_eval(val) |
| 56 |
except Exception: |
| 57 |
return [] |
| 58 |
|
| 59 |
|
| 60 |
############################################################################# |
| 61 |
|
| 62 |
|
| 63 |
class Todoist2ebook(BasicNewsRecipe): |
| 64 |
recipe_specific_options = { |
| 65 |
'ARCHIVE_DOWNLOADED': { |
| 66 |
'short': 'Mark as read', |
| 67 |
'long': 'Mark as read', |
| 68 |
'default': False, |
| 69 |
}, |
| 70 |
'TODOIST_PROJECT_ID': {'short': 'Proyect ID', 'long': 'Proyect ID'}, |
| 71 |
'TODOIST_API_KEY': {'short': 'API key', 'long': 'API KEY'}, |
| 72 |
'URL_KEYWORD_EXCEPTIONS': { |
| 73 |
'short': 'URL keyword exceptions', |
| 74 |
'long': 'List of keywords to ignore articles, e.g. ["example.com", "ignoreme.com"]', |
| 75 |
'default': [], |
| 76 |
}, |
| 77 |
} |
| 78 |
|
| 79 |
__author__ = 'rga5321' |
| 80 |
description = ( |
| 81 |
'Calibre recipe to fetch articles from a Todoist project. The recipe is available in Calibre and in' |
| 82 |
' the original repository https://github.com/rga5321/todoist2ebook, where an script to automate' |
| 83 |
' the ebook conversion and email sending is also available.' |
| 84 |
) |
| 85 |
publisher = 'Todoist.com' |
| 86 |
category = 'info, custom, Todoist' |
| 87 |
|
| 88 |
# User-configurable settings ----------------------------------------------- |
| 89 |
|
| 90 |
series_name = 'Todoist' |
| 91 |
publication_type = 'magazine' |
| 92 |
title = 'Todoist' |
| 93 |
# timefmt = '' # uncomment to remove date from the filenames, if commented then you will get something like `Todoist [Wed, 13 May 2020]` |
| 94 |
masthead_url = 'https://raw.githubusercontent.com/rga5321/todoist2ebook/master/img/todoist-logo.png' |
| 95 |
# will make square cover; this will replace text and cover of the default |
| 96 |
cover_url = 'https://raw.githubusercontent.com/rga5321/todoist2ebook/master/img/todoist-cover.png' |
| 97 |
# -------------------------------------------------------------------------- |
| 98 |
|
| 99 |
# Inherited developer settings |
| 100 |
auto_cleanup = True |
| 101 |
no_stylesheets = True |
| 102 |
use_embedded_content = False |
| 103 |
|
| 104 |
# Custom developer settings |
| 105 |
to_archive = [] |
| 106 |
|
| 107 |
simultaneous_downloads = 10 |
| 108 |
|
| 109 |
extra_css = '.calibre_navbar { visibility: hidden; }' |
| 110 |
|
| 111 |
def __init__(self, *args, **kwargs): |
| 112 |
super().__init__(*args, **kwargs) |
| 113 |
|
| 114 |
# Init optional configuration parameters |
| 115 |
self.archive_downloaded = parse_env_bool(self.recipe_specific_options['ARCHIVE_DOWNLOADED']) |
| 116 |
self.keyword_exceptions = parse_env_list(self.recipe_specific_options['URL_KEYWORD_EXCEPTIONS']) |
| 117 |
|
| 118 |
# Init mandatory configuration parameters |
| 119 |
if self.recipe_specific_options.get('TODOIST_PROJECT_ID'): |
| 120 |
self.todoist_project_id = self.recipe_specific_options['TODOIST_PROJECT_ID'] |
| 121 |
else: |
| 122 |
self.abort_recipe_processing('TODOIST_PROJECT_ID mandatory parameter missing') |
| 123 |
|
| 124 |
if self.recipe_specific_options.get('TODOIST_API_KEY'): |
| 125 |
self.todoist_api_key = self.recipe_specific_options['TODOIST_API_KEY'] |
| 126 |
else: |
| 127 |
self.abort_recipe_processing('TODOIST_API_KEY mandatory parameter missing') |
| 128 |
|
| 129 |
def parse_index(self): |
| 130 |
|
| 131 |
articles = [] |
| 132 |
section_dict = {} # dictionary with the domains and its articles. |
| 133 |
|
| 134 |
url = f'https://api.todoist.com/api/v1/tasks?project_id={self.todoist_project_id}' |
| 135 |
headers = {'Authorization': f'Bearer {self.todoist_api_key}'} |
| 136 |
request = mechanize.Request(url, headers=headers) |
| 137 |
|
| 138 |
response = self.browser.open(request) |
| 139 |
if response.code != 200: |
| 140 |
raise Exception('Could not retrieve Todoist tasks') |
| 141 |
data = response.read().decode('utf-8') |
| 142 |
tasks = json.loads(data) |
| 143 |
if isinstance(tasks, dict) and 'results' in tasks: |
| 144 |
tasks = tasks['results'] |
| 145 |
else: |
| 146 |
self.abort_recipe_processing( |
| 147 |
'Unexpected response format from Todoist API. ' |
| 148 |
'Expected {"results": [...]}, got keys: %s' % str(tasks.keys() if isinstance(tasks, dict) else type(tasks)) |
| 149 |
) |
| 150 |
articles_todoist = [] |
| 151 |
|
| 152 |
url_regex = re.compile(r'\[([^\]]+)\]\(\s*(https?://[^\s\)]+)\s*\)') |
| 153 |
for task in tasks: |
| 154 |
match = url_regex.search(task['content']) |
| 155 |
if match: |
| 156 |
title = match.group(1).strip() |
| 157 |
url = match.group(2).strip() |
| 158 |
date_added = task.get('created_at', datetime.now().isoformat()) |
| 159 |
articles_todoist.append({ |
| 160 |
'title': title or url, |
| 161 |
'url': url, |
| 162 |
'date_added': date_added, |
| 163 |
'item_id': task['id'], |
| 164 |
}) |
| 165 |
|
| 166 |
if not articles_todoist: |
| 167 |
self.abort_recipe_processing('No unread articles in the Todoist project "{}"'.format(self.todoist_project_id)) |
| 168 |
else: |
| 169 |
for item in articles_todoist: |
| 170 |
# If the URL contains any URL_KEYWORD_EXCEPTIONS, ignore article |
| 171 |
if any(pattern in item['url'] for pattern in self.keyword_exceptions): |
| 172 |
print('Ignoring article due to keyword patterns:' + item['url']) |
| 173 |
del item |
| 174 |
else: |
| 175 |
# Extract domain from the URL |
| 176 |
domain = urlparse(item['url']).netloc.replace('www.', '') |
| 177 |
|
| 178 |
url = item['url'] |
| 179 |
|
| 180 |
# Add the article under its domain |
| 181 |
if domain not in section_dict: |
| 182 |
section_dict[domain] = [item] |
| 183 |
else: |
| 184 |
section_dict[domain].append(item) |
| 185 |
|
| 186 |
print('Adding article: ' + item['url'] + ' to section: ' + domain) |
| 187 |
|
| 188 |
############ APPEND ARTS FOR EACH DOMAIN ############# |
| 189 |
# At this point the section_dict is completed |
| 190 |
|
| 191 |
for section in section_dict: |
| 192 |
arts = [] |
| 193 |
for item in section_dict.get(section): |
| 194 |
try: |
| 195 |
title = item['title'] |
| 196 |
except KeyError: |
| 197 |
title = 'error: title' |
| 198 |
try: |
| 199 |
url = item['url'] |
| 200 |
except KeyError: |
| 201 |
url = 'error: url' |
| 202 |
|
| 203 |
arts.append({'title': title, 'url': url, 'date': item['date_added']}) |
| 204 |
|
| 205 |
if self.archive_downloaded and item['item_id'] not in self.to_archive: |
| 206 |
self.to_archive.append(item['item_id']) |
| 207 |
|
| 208 |
if arts: |
| 209 |
articles.append((section, arts)) |
| 210 |
|
| 211 |
if not articles: |
| 212 |
self.abort_recipe_processing('No articles in the Todoist project account %s to download' % (self.todoist_project_id)) |
| 213 |
return articles |
| 214 |
|
| 215 |
def get_browser(self, *args, **kwargs): |
| 216 |
self.browser = BasicNewsRecipe.get_browser(self) |
| 217 |
return self.browser |
| 218 |
|
| 219 |
def cleanup(self): |
| 220 |
if not self.to_archive: |
| 221 |
return |
| 222 |
|
| 223 |
for task_id in self.to_archive: |
| 224 |
url = f'https://api.todoist.com/api/v1/tasks/{task_id}/close' |
| 225 |
req = mechanize.Request( |
| 226 |
url, |
| 227 |
headers={ |
| 228 |
'Authorization': f'Bearer {self.todoist_api_key}', |
| 229 |
'Content-Type': 'application/json', |
| 230 |
}, |
| 231 |
) |
| 232 |
req.get_method = lambda: 'POST' |
| 233 |
|
| 234 |
try: |
| 235 |
br = mechanize.Browser() |
| 236 |
response = br.open(req) |
| 237 |
if response.code == 204: |
| 238 |
print(f'Task {task_id} corectly closed.') |
| 239 |
else: |
| 240 |
print(f'Error while closing task {task_id}: {response.code}') |
| 241 |
except Exception as e: |
| 242 |
print(f'Exception while closing task {task_id}: {e}') |
| 243 |
|
| 244 |
# TODO: This works with EPUB, but not mobi/azw3 |
| 245 |
# BUG: https://bugs.launchpad.net/calibre/+bug/1838486 |
| 246 |
def postprocess_book(self, oeb, opts, log): |
| 247 |
oeb.metadata.add('series', self.series_name) |
| 248 |
|
| 249 |
def _postprocess_html(self, soup, first_fetch, job_info): |
| 250 |
|
| 251 |
title = soup.find('title').text # get title |
| 252 |
|
| 253 |
h1s = soup.findAll('h1') # get all h1 headers |
| 254 |
for h1 in h1s: |
| 255 |
if title in h1.text: |
| 256 |
h1 = h1.clear() # clean this tag, so the h1 will be there only |
| 257 |
|
| 258 |
h2s = soup.findAll('h2') # get all h2 headers |
| 259 |
for h2 in h2s: |
| 260 |
if title in h2.text: |
| 261 |
h2 = h2.clear() # clean this tag, so the h1 will be there only |
| 262 |
|
| 263 |
body = soup.find('body') |
| 264 |
new_tag = soup.new_tag('h1') |
| 265 |
new_tag.append(title) |
| 266 |
body.insert(0, new_tag) |
| 267 |
|
| 268 |
return soup |
| 269 |
|
| 270 |
def default_cover(self, cover_file): |
| 271 |
""" |
| 272 |
Create a generic cover for recipes that don't have a cover |
| 273 |
This override adds time to the cover |
| 274 |
""" |
| 275 |
try: |
| 276 |
from calibre.ebooks import calibre_cover |
| 277 |
|
| 278 |
title = self.title |
| 279 |
date = strftime(self.timefmt) |
| 280 |
time = strftime('%a %d %b %Y %-H:%M') |
| 281 |
img_data = calibre_cover(title, date, time) |
| 282 |
cover_file.write(img_data) |
| 283 |
cover_file.flush() |
| 284 |
except Exception: |
| 285 |
self.log.exception('Failed to generate default cover') |
| 286 |
return False |
| 287 |
return True |