Debian Codegraph @

setup/check.py

raw
1 #!/usr/bin/env python
2
3
4 __license__ = 'GPL v3'
5 __copyright__ = '2009, Kovid Goyal <[email protected]>'
6 __docformat__ = 'restructuredtext en'
7
8 import concurrent.futures
9 import json
10 import os
11 import re
12 import subprocess
13 import sys
14
15 from setup import Command, edit_file, iswindows
16
17
18 def files_walker(root_path, ext):
19 for x in os.walk(root_path):
20 for f in x[-1]:
21 y = os.path.join(x[0], f)
22 if f.endswith(ext):
23 yield y
24
25
26 def checkable_python_files(SRC):
27 for dname in ('odf', 'calibre'):
28 for f in files_walker(os.path.join(SRC, dname), '.py'):
29 if not f.endswith('_ui.py'):
30 yield f
31
32
33 class Check(Command):
34 description = 'Check for errors in the calibre source code'
35 usage_help = 'To check specific files, specify them as command line arguments'
36 require_venv = True
37
38 def add_options(self, parser):
39 parser.add_option('--no-editor', default=False, action='store_true', help="Don't open the editor when a bad file is found.")
40 parser.add_option('--fix', '--auto-fix', default=False, action='store_true', help='Try to automatically fix errors with ruff.')
41
42 def _ruff_executable(self):
43 import shutil
44
45 ruff = self.j(self.PROJECT_ROOT, '.venv/bin/ruff')
46 if iswindows:
47 ruff += '.exe'
48 if not os.path.exists(ruff):
49 ruff = shutil.which('ruff') or 'ruff'
50 self._ruff_executable = lambda: ruff
51 return ruff
52
53 def _rapydscript_executable(self):
54 import shutil
55
56 rs = self.j(self.PROJECT_ROOT, '.venv/bin/rapydscript')
57 if iswindows:
58 rs += '.exe'
59 if not os.path.exists(rs):
60 rs = shutil.which('rapydscript') or 'rapydscript'
61 self._rapydscript_executable = lambda: rs
62 return rs
63
64 def _run_ruff(self, targets):
65 ruff = self._ruff_executable()
66 cmd = [ruff, 'check', '--output-format=json']
67 if self.auto_fix:
68 cmd.append('--fix')
69 p = subprocess.run(
70 cmd + targets,
71 capture_output=True,
72 text=True,
73 cwd=self.PROJECT_ROOT,
74 )
75 output_lines = []
76 files_with_errors = set()
77 if p.stdout:
78 try:
79 for d in json.loads(p.stdout):
80 fname = d['filename']
81 if not os.path.isabs(fname):
82 fname = os.path.join(self.PROJECT_ROOT, fname)
83 loc = d.get('location', {})
84 row = loc.get('row', 0)
85 col = loc.get('column', 0)
86 code = d.get('code', '')
87 msg = d.get('message', '')
88 output_lines.append(f'{fname}:{row}:{col}: {code} {msg}')
89 files_with_errors.add(fname)
90 except json.JSONDecodeError, KeyError:
91 output_lines.append(p.stdout.strip())
92 if p.stderr.strip():
93 output_lines.append(p.stderr.strip())
94 return p.returncode, '\n'.join(filter(None, output_lines)), files_with_errors
95
96 def _run_rapydscript(self, targets):
97 p = subprocess.run(
98 [self._rapydscript_executable(), 'lint'] + targets,
99 capture_output=True,
100 text=True,
101 )
102 combined = (p.stdout + p.stderr).strip()
103 files_with_errors = set()
104 if p.returncode != 0:
105 for line in combined.splitlines():
106 m = re.match(r'^(.+\.pyj):\d+', line)
107 if m:
108 fname = m.group(1)
109 if not os.path.isabs(fname):
110 fname = os.path.join(self.PROJECT_ROOT, fname)
111 files_with_errors.add(fname)
112 return p.returncode, combined, files_with_errors
113
114 def _run_changelog(self, wn_path):
115 changelog_file = self.j(self.d(self.SRC), 'Changelog.txt')
116 p = subprocess.run(
117 ['python', self.j(wn_path, 'whats_new.py'), changelog_file],
118 capture_output=True,
119 text=True,
120 )
121 return p.returncode, (p.stdout + p.stderr).strip()
122
123 def _check_file(self, f):
124 ext = os.path.splitext(f)[1]
125 if ext in {'.py', '.pyi', '.recipe'}:
126 ruff = self._ruff_executable()
127 p = subprocess.run([ruff, 'check', f], capture_output=True, text=True, cwd=self.PROJECT_ROOT)
128 output = (p.stdout + p.stderr).strip()
129 if p.returncode != 0 and output:
130 print(output, file=sys.stderr)
131 return p.returncode != 0
132 if ext == '.pyj':
133 p = subprocess.run([self._rapydscript_executable(), 'lint', f], capture_output=True, text=True)
134 output = (p.stdout + p.stderr).strip()
135 if p.returncode != 0 and output:
136 print(output, file=sys.stderr)
137 return p.returncode != 0
138 return False
139
140 def run(self, opts):
141 no_editor = opts.no_editor
142 self.auto_fix = opts.fix
143 cli_args = opts.cli_args
144
145 python_exts = {'.py', '.pyi', '.recipe'}
146 wn_path = os.path.expanduser('~/work/srv/main/static')
147 has_changelog = os.path.exists(wn_path) and not cli_args
148
149 if cli_args:
150 ruff_targets = []
151 rapydscript_targets = []
152 for f in cli_args:
153 ext = os.path.splitext(f)[1]
154 if ext in python_exts:
155 ruff_targets.append(f)
156 elif ext == '.pyj':
157 rapydscript_targets.append(f)
158 else:
159 ruff_targets.append(f)
160 rapydscript_targets.append(f)
161 else:
162 ruff_targets = ['.']
163 rapydscript_targets = [self.j(self.SRC, 'pyj')]
164
165 with concurrent.futures.ThreadPoolExecutor() as ex:
166 futures = {}
167 if ruff_targets:
168 futures['ruff'] = ex.submit(self._run_ruff, ruff_targets)
169 if rapydscript_targets:
170 futures['rapydscript'] = ex.submit(self._run_rapydscript, rapydscript_targets)
171 if has_changelog:
172 futures['changelog'] = ex.submit(self._run_changelog, wn_path)
173 results = {name: f.result() for name, f in futures.items()}
174
175 ruff_rc, ruff_output, ruff_files = results.get('ruff', (0, '', set()))
176 rs_rc, rs_output, rs_files = results.get('rapydscript', (0, '', set()))
177 cl_rc, cl_output = results.get('changelog', (0, ''))
178
179 if ruff_output:
180 print(ruff_output, file=sys.stderr)
181 if rs_output:
182 print(rs_output, file=sys.stderr)
183 if cl_output:
184 print(cl_output, file=sys.stderr)
185
186 had_errors = ruff_rc != 0 or rs_rc != 0 or cl_rc != 0
187
188 if no_editor and had_errors:
189 raise SystemExit(1)
190 elif no_editor:
191 return
192
193 # Errors that can't be resolved by opening a file in the editor
194 unresolvable = cl_rc != 0
195 if ruff_rc != 0 and not ruff_files:
196 unresolvable = True
197 if rs_rc != 0 and not rs_files:
198 unresolvable = True
199
200 for f in sorted(ruff_files | rs_files):
201 try:
202 edit_file(f)
203 except FileNotFoundError:
204 raise SystemExit(1)
205 if self._check_file(f):
206 raise SystemExit(1)
207
208 if unresolvable:
209 raise SystemExit(1)
210
211
212 class CheckAll(Command):
213 description = 'Perform every quality code test suite'
214 usage_help = 'To check specific files, specify them as command line arguments'
215
216 sub_commands = [
217 'type_check',
218 'check',
219 'fmt',
220 ]
221
222
223 class UpgradeSourceCode(Command):
224 description = 'Upgrade python source code'
225
226 def run(self, opts):
227 files = []
228 for f in os.listdir(os.path.dirname(os.path.abspath(__file__))):
229 q = os.path.join('setup', f)
230 if f.endswith('.py') and f != 'linux-installer.py' and not os.path.isdir(q):
231 files.append(q)
232 for path in checkable_python_files(self.SRC):
233 q = path.replace(os.sep, '/')
234 if '/metadata/sources/' in q or '/store/stores/' in q:
235 continue
236 files.append(q)
237 subprocess.call(['pyupgrade', '--py314-plus'] + files)