Debian Codegraph @

Lib/_pyrepl/_module_completer.py

raw
1 from __future__ import annotations
2
3 import importlib
4 import os
5 import pkgutil
6 import sys
7 import token
8 import tokenize
9 from importlib.machinery import FileFinder
10 from io import StringIO
11 from contextlib import contextmanager
12 from dataclasses import dataclass
13 from itertools import chain
14 from tokenize import TokenInfo
15
16 TYPE_CHECKING = False
17
18 if TYPE_CHECKING:
19 from typing import Any, Iterable, Iterator, Mapping
20
21
22 HARDCODED_SUBMODULES = {
23 # Standard library submodules that are not detected by pkgutil.iter_modules
24 # but can be imported, so should be proposed in completion
25 "collections": ["abc"],
26 "os": ["path"],
27 "xml.parsers.expat": ["errors", "model"],
28 }
29
30
31 def make_default_module_completer() -> ModuleCompleter:
32 # Inside pyrepl, __package__ is set to None by default
33 return ModuleCompleter(namespace={'__package__': None})
34
35
36 class ModuleCompleter:
37 """A completer for Python import statements.
38
39 Examples:
40 - import <tab>
41 - import foo<tab>
42 - import foo.<tab>
43 - import foo as bar, baz<tab>
44
45 - from <tab>
46 - from foo<tab>
47 - from foo import <tab>
48 - from foo import bar<tab>
49 - from foo import (bar as baz, qux<tab>
50 """
51
52 def __init__(self, namespace: Mapping[str, Any] | None = None) -> None:
53 self.namespace = namespace or {}
54 self._global_cache: list[pkgutil.ModuleInfo] = []
55 self._curr_sys_path: list[str] = sys.path[:]
56 self._stdlib_path = os.path.dirname(importlib.__path__[0])
57
58 def get_completions(self, line: str) -> list[str] | None:
59 """Return the next possible import completions for 'line'."""
60 result = ImportParser(line).parse()
61 if not result:
62 return None
63 try:
64 return self.complete(*result)
65 except Exception:
66 # Some unexpected error occurred, make it look like
67 # no completions are available
68 return []
69
70 def complete(self, from_name: str | None, name: str | None) -> list[str]:
71 if from_name is None:
72 # import x.y.z<tab>
73 assert name is not None
74 path, prefix = self.get_path_and_prefix(name)
75 modules = self.find_modules(path, prefix)
76 return [self.format_completion(path, module) for module in modules]
77
78 if name is None:
79 # from x.y.z<tab>
80 path, prefix = self.get_path_and_prefix(from_name)
81 modules = self.find_modules(path, prefix)
82 return [self.format_completion(path, module) for module in modules]
83
84 # from x.y import z<tab>
85 return self.find_modules(from_name, name)
86
87 def find_modules(self, path: str, prefix: str) -> list[str]:
88 """Find all modules under 'path' that start with 'prefix'."""
89 modules = self._find_modules(path, prefix)
90 # Filter out invalid module names
91 # (for example those containing dashes that cannot be imported with 'import')
92 return [mod for mod in modules if mod.isidentifier()]
93
94 def _find_modules(self, path: str, prefix: str) -> list[str]:
95 if not path:
96 # Top-level import (e.g. `import foo<tab>`` or `from foo<tab>`)`
97 builtin_modules = [name for name in sys.builtin_module_names
98 if self.is_suggestion_match(name, prefix)]
99 third_party_modules = [module.name for module in self.global_cache
100 if self.is_suggestion_match(module.name, prefix)]
101 return sorted(builtin_modules + third_party_modules)
102
103 if path.startswith('.'):
104 # Convert relative path to absolute path
105 package = self.namespace.get('__package__', '')
106 path = self.resolve_relative_name(path, package) # type: ignore[assignment]
107 if path is None:
108 return []
109
110 modules: Iterable[pkgutil.ModuleInfo] = self.global_cache
111 imported_module = sys.modules.get(path.split('.')[0])
112 if imported_module:
113 # Filter modules to those who name and specs match the
114 # imported module to avoid invalid suggestions
115 spec = imported_module.__spec__
116 if spec:
117 modules = [mod for mod in modules
118 if mod.name == spec.name
119 and mod.module_finder.find_spec(mod.name, None) == spec]
120 else:
121 modules = []
122
123 is_stdlib_import: bool | None = None
124 for segment in path.split('.'):
125 modules = [mod_info for mod_info in modules
126 if mod_info.ispkg and mod_info.name == segment]
127 if is_stdlib_import is None:
128 # Top-level import decide if we import from stdlib or not
129 is_stdlib_import = all(
130 self._is_stdlib_module(mod_info) for mod_info in modules
131 )
132 modules = self.iter_submodules(modules)
133
134 module_names = [module.name for module in modules]
135 if is_stdlib_import:
136 module_names.extend(HARDCODED_SUBMODULES.get(path, ()))
137 return [module_name for module_name in module_names
138 if self.is_suggestion_match(module_name, prefix)]
139
140 def _is_stdlib_module(self, module_info: pkgutil.ModuleInfo) -> bool:
141 return (isinstance(module_info.module_finder, FileFinder)
142 and module_info.module_finder.path == self._stdlib_path)
143
144 def is_suggestion_match(self, module_name: str, prefix: str) -> bool:
145 if prefix:
146 return module_name.startswith(prefix)
147 # For consistency with attribute completion, which
148 # does not suggest private attributes unless requested.
149 return not module_name.startswith("_")
150
151 def iter_submodules(self, parent_modules: list[pkgutil.ModuleInfo]) -> Iterator[pkgutil.ModuleInfo]:
152 """Iterate over all submodules of the given parent modules."""
153 specs = [info.module_finder.find_spec(info.name, None)
154 for info in parent_modules if info.ispkg]
155 search_locations = set(chain.from_iterable(
156 getattr(spec, 'submodule_search_locations', [])
157 for spec in specs if spec
158 ))
159 return pkgutil.iter_modules(search_locations)
160
161 def get_path_and_prefix(self, dotted_name: str) -> tuple[str, str]:
162 """
163 Split a dotted name into an import path and a
164 final prefix that is to be completed.
165
166 Examples:
167 'foo.bar' -> 'foo', 'bar'
168 'foo.' -> 'foo', ''
169 '.foo' -> '.', 'foo'
170 """
171 if '.' not in dotted_name:
172 return '', dotted_name
173 if dotted_name.startswith('.'):
174 stripped = dotted_name.lstrip('.')
175 dots = '.' * (len(dotted_name) - len(stripped))
176 if '.' not in stripped:
177 return dots, stripped
178 path, prefix = stripped.rsplit('.', 1)
179 return dots + path, prefix
180 path, prefix = dotted_name.rsplit('.', 1)
181 return path, prefix
182
183 def format_completion(self, path: str, module: str) -> str:
184 if path == '' or path.endswith('.'):
185 return f'{path}{module}'
186 return f'{path}.{module}'
187
188 def resolve_relative_name(self, name: str, package: str) -> str | None:
189 """Resolve a relative module name to an absolute name.
190
191 Example: resolve_relative_name('.foo', 'bar') -> 'bar.foo'
192 """
193 # taken from importlib._bootstrap
194 level = 0
195 for character in name:
196 if character != '.':
197 break
198 level += 1
199 bits = package.rsplit('.', level - 1)
200 if len(bits) < level:
201 return None
202 base = bits[0]
203 name = name[level:]
204 return f'{base}.{name}' if name else base
205
206 @property
207 def global_cache(self) -> list[pkgutil.ModuleInfo]:
208 """Global module cache"""
209 if not self._global_cache or self._curr_sys_path != sys.path:
210 self._curr_sys_path = sys.path[:]
211 self._global_cache = list(pkgutil.iter_modules())
212 return self._global_cache
213
214
215 class ImportParser:
216 """
217 Parses incomplete import statements that are
218 suitable for autocomplete suggestions.
219
220 Examples:
221 - import foo -> Result(from_name=None, name='foo')
222 - import foo. -> Result(from_name=None, name='foo.')
223 - from foo -> Result(from_name='foo', name=None)
224 - from foo import bar -> Result(from_name='foo', name='bar')
225 - from .foo import ( -> Result(from_name='.foo', name='')
226
227 Note that the parser works in reverse order, starting from the
228 last token in the input string. This makes the parser more robust
229 when parsing multiple statements.
230 """
231 _ignored_tokens = {
232 token.INDENT, token.DEDENT, token.COMMENT,
233 token.NL, token.NEWLINE, token.ENDMARKER
234 }
235 _keywords = {'import', 'from', 'as'}
236
237 def __init__(self, code: str) -> None:
238 self.code = code
239 tokens = []
240 try:
241 for t in tokenize.generate_tokens(StringIO(code).readline):
242 if t.type not in self._ignored_tokens:
243 tokens.append(t)
244 except tokenize.TokenError as e:
245 if 'unexpected EOF' not in str(e):
246 # unexpected EOF is fine, since we're parsing an
247 # incomplete statement, but other errors are not
248 # because we may not have all the tokens so it's
249 # safer to bail out
250 tokens = []
251 except SyntaxError:
252 tokens = []
253 self.tokens = TokenQueue(tokens[::-1])
254
255 def parse(self) -> tuple[str | None, str | None] | None:
256 if not (res := self._parse()):
257 return None
258 return res.from_name, res.name
259
260 def _parse(self) -> Result | None:
261 with self.tokens.save_state():
262 return self.parse_from_import()
263 with self.tokens.save_state():
264 return self.parse_import()
265
266 def parse_import(self) -> Result:
267 if self.code.rstrip().endswith('import') and self.code.endswith(' '):
268 return Result(name='')
269 if self.tokens.peek_string(','):
270 name = ''
271 else:
272 if self.code.endswith(' '):
273 raise ParseError('parse_import')
274 name = self.parse_dotted_name()
275 if name.startswith('.'):
276 raise ParseError('parse_import')
277 while self.tokens.peek_string(','):
278 self.tokens.pop()
279 self.parse_dotted_as_name()
280 if self.tokens.peek_string('import'):
281 return Result(name=name)
282 raise ParseError('parse_import')
283
284 def parse_from_import(self) -> Result:
285 stripped = self.code.rstrip()
286 if stripped.endswith('import') and self.code.endswith(' '):
287 return Result(from_name=self.parse_empty_from_import(), name='')
288 if stripped.endswith('from') and self.code.endswith(' '):
289 return Result(from_name='')
290 if self.tokens.peek_string('(') or self.tokens.peek_string(','):
291 return Result(from_name=self.parse_empty_from_import(), name='')
292 if self.code.endswith(' '):
293 raise ParseError('parse_from_import')
294 name = self.parse_dotted_name()
295 if '.' in name:
296 self.tokens.pop_string('from')
297 return Result(from_name=name)
298 if self.tokens.peek_string('from'):
299 return Result(from_name=name)
300 from_name = self.parse_empty_from_import()
301 return Result(from_name=from_name, name=name)
302
303 def parse_empty_from_import(self) -> str:
304 if self.tokens.peek_string(','):
305 self.tokens.pop()
306 self.parse_as_names()
307 if self.tokens.peek_string('('):
308 self.tokens.pop()
309 self.tokens.pop_string('import')
310 return self.parse_from()
311
312 def parse_from(self) -> str:
313 from_name = self.parse_dotted_name()
314 self.tokens.pop_string('from')
315 return from_name
316
317 def parse_dotted_as_name(self) -> str:
318 self.tokens.pop_name()
319 if self.tokens.peek_string('as'):
320 self.tokens.pop()
321 with self.tokens.save_state():
322 return self.parse_dotted_name()
323
324 def parse_dotted_name(self) -> str:
325 name = []
326 if self.tokens.peek_string('.'):
327 name.append('.')
328 self.tokens.pop()
329 if (self.tokens.peek_name()
330 and (tok := self.tokens.peek())
331 and tok.string not in self._keywords):
332 name.append(self.tokens.pop_name())
333 if not name:
334 raise ParseError('parse_dotted_name')
335 while self.tokens.peek_string('.'):
336 name.append('.')
337 self.tokens.pop()
338 if (self.tokens.peek_name()
339 and (tok := self.tokens.peek())
340 and tok.string not in self._keywords):
341 name.append(self.tokens.pop_name())
342 else:
343 break
344
345 while self.tokens.peek_string('.'):
346 name.append('.')
347 self.tokens.pop()
348 return ''.join(name[::-1])
349
350 def parse_as_names(self) -> None:
351 self.parse_as_name()
352 while self.tokens.peek_string(','):
353 self.tokens.pop()
354 self.parse_as_name()
355
356 def parse_as_name(self) -> None:
357 self.tokens.pop_name()
358 if self.tokens.peek_string('as'):
359 self.tokens.pop()
360 self.tokens.pop_name()
361
362
363 class ParseError(Exception):
364 pass
365
366
367 @dataclass(frozen=True)
368 class Result:
369 from_name: str | None = None
370 name: str | None = None
371
372
373 class TokenQueue:
374 """Provides helper functions for working with a sequence of tokens."""
375
376 def __init__(self, tokens: list[TokenInfo]) -> None:
377 self.tokens: list[TokenInfo] = tokens
378 self.index: int = 0
379 self.stack: list[int] = []
380
381 @contextmanager
382 def save_state(self) -> Any:
383 try:
384 self.stack.append(self.index)
385 yield
386 except ParseError:
387 self.index = self.stack.pop()
388 else:
389 self.stack.pop()
390
391 def __bool__(self) -> bool:
392 return self.index < len(self.tokens)
393
394 def peek(self) -> TokenInfo | None:
395 if not self:
396 return None
397 return self.tokens[self.index]
398
399 def peek_name(self) -> bool:
400 if not (tok := self.peek()):
401 return False
402 return tok.type == token.NAME
403
404 def pop_name(self) -> str:
405 tok = self.pop()
406 if tok.type != token.NAME:
407 raise ParseError('pop_name')
408 return tok.string
409
410 def peek_string(self, string: str) -> bool:
411 if not (tok := self.peek()):
412 return False
413 return tok.string == string
414
415 def pop_string(self, string: str) -> str:
416 tok = self.pop()
417 if tok.string != string:
418 raise ParseError('pop_string')
419 return tok.string
420
421 def pop(self) -> TokenInfo:
422 if not self:
423 raise ParseError('pop')
424 tok = self.tokens[self.index]
425 self.index += 1
426 return tok