| 1 |
from __future__ import annotations |
| 2 |
|
| 3 |
import importlib |
| 4 |
import os |
| 5 |
import pkgutil |
| 6 |
import sys |
| 7 |
import token |
| 8 |
import tokenize |
| 9 |
from importlib.machinery import FileFinder |
| 10 |
from io import StringIO |
| 11 |
from contextlib import contextmanager |
| 12 |
from dataclasses import dataclass |
| 13 |
from itertools import chain |
| 14 |
from tokenize import TokenInfo |
| 15 |
|
| 16 |
TYPE_CHECKING = False |
| 17 |
|
| 18 |
if TYPE_CHECKING: |
| 19 |
from typing import Any, Iterable, Iterator, Mapping |
| 20 |
|
| 21 |
|
| 22 |
HARDCODED_SUBMODULES = { |
| 23 |
# Standard library submodules that are not detected by pkgutil.iter_modules |
| 24 |
# but can be imported, so should be proposed in completion |
| 25 |
"collections": ["abc"], |
| 26 |
"os": ["path"], |
| 27 |
"xml.parsers.expat": ["errors", "model"], |
| 28 |
} |
| 29 |
|
| 30 |
|
| 31 |
def make_default_module_completer() -> ModuleCompleter: |
| 32 |
# Inside pyrepl, __package__ is set to None by default |
| 33 |
return ModuleCompleter(namespace={'__package__': None}) |
| 34 |
|
| 35 |
|
| 36 |
class ModuleCompleter: |
| 37 |
"""A completer for Python import statements. |
| 38 |
|
| 39 |
Examples: |
| 40 |
- import <tab> |
| 41 |
- import foo<tab> |
| 42 |
- import foo.<tab> |
| 43 |
- import foo as bar, baz<tab> |
| 44 |
|
| 45 |
- from <tab> |
| 46 |
- from foo<tab> |
| 47 |
- from foo import <tab> |
| 48 |
- from foo import bar<tab> |
| 49 |
- from foo import (bar as baz, qux<tab> |
| 50 |
""" |
| 51 |
|
| 52 |
def __init__(self, namespace: Mapping[str, Any] | None = None) -> None: |
| 53 |
self.namespace = namespace or {} |
| 54 |
self._global_cache: list[pkgutil.ModuleInfo] = [] |
| 55 |
self._curr_sys_path: list[str] = sys.path[:] |
| 56 |
self._stdlib_path = os.path.dirname(importlib.__path__[0]) |
| 57 |
|
| 58 |
def get_completions(self, line: str) -> list[str] | None: |
| 59 |
"""Return the next possible import completions for 'line'.""" |
| 60 |
result = ImportParser(line).parse() |
| 61 |
if not result: |
| 62 |
return None |
| 63 |
try: |
| 64 |
return self.complete(*result) |
| 65 |
except Exception: |
| 66 |
# Some unexpected error occurred, make it look like |
| 67 |
# no completions are available |
| 68 |
return [] |
| 69 |
|
| 70 |
def complete(self, from_name: str | None, name: str | None) -> list[str]: |
| 71 |
if from_name is None: |
| 72 |
# import x.y.z<tab> |
| 73 |
assert name is not None |
| 74 |
path, prefix = self.get_path_and_prefix(name) |
| 75 |
modules = self.find_modules(path, prefix) |
| 76 |
return [self.format_completion(path, module) for module in modules] |
| 77 |
|
| 78 |
if name is None: |
| 79 |
# from x.y.z<tab> |
| 80 |
path, prefix = self.get_path_and_prefix(from_name) |
| 81 |
modules = self.find_modules(path, prefix) |
| 82 |
return [self.format_completion(path, module) for module in modules] |
| 83 |
|
| 84 |
# from x.y import z<tab> |
| 85 |
return self.find_modules(from_name, name) |
| 86 |
|
| 87 |
def find_modules(self, path: str, prefix: str) -> list[str]: |
| 88 |
"""Find all modules under 'path' that start with 'prefix'.""" |
| 89 |
modules = self._find_modules(path, prefix) |
| 90 |
# Filter out invalid module names |
| 91 |
# (for example those containing dashes that cannot be imported with 'import') |
| 92 |
return [mod for mod in modules if mod.isidentifier()] |
| 93 |
|
| 94 |
def _find_modules(self, path: str, prefix: str) -> list[str]: |
| 95 |
if not path: |
| 96 |
# Top-level import (e.g. `import foo<tab>`` or `from foo<tab>`)` |
| 97 |
builtin_modules = [name for name in sys.builtin_module_names |
| 98 |
if self.is_suggestion_match(name, prefix)] |
| 99 |
third_party_modules = [module.name for module in self.global_cache |
| 100 |
if self.is_suggestion_match(module.name, prefix)] |
| 101 |
return sorted(builtin_modules + third_party_modules) |
| 102 |
|
| 103 |
if path.startswith('.'): |
| 104 |
# Convert relative path to absolute path |
| 105 |
package = self.namespace.get('__package__', '') |
| 106 |
path = self.resolve_relative_name(path, package) # type: ignore[assignment] |
| 107 |
if path is None: |
| 108 |
return [] |
| 109 |
|
| 110 |
modules: Iterable[pkgutil.ModuleInfo] = self.global_cache |
| 111 |
imported_module = sys.modules.get(path.split('.')[0]) |
| 112 |
if imported_module: |
| 113 |
# Filter modules to those who name and specs match the |
| 114 |
# imported module to avoid invalid suggestions |
| 115 |
spec = imported_module.__spec__ |
| 116 |
if spec: |
| 117 |
modules = [mod for mod in modules |
| 118 |
if mod.name == spec.name |
| 119 |
and mod.module_finder.find_spec(mod.name, None) == spec] |
| 120 |
else: |
| 121 |
modules = [] |
| 122 |
|
| 123 |
is_stdlib_import: bool | None = None |
| 124 |
for segment in path.split('.'): |
| 125 |
modules = [mod_info for mod_info in modules |
| 126 |
if mod_info.ispkg and mod_info.name == segment] |
| 127 |
if is_stdlib_import is None: |
| 128 |
# Top-level import decide if we import from stdlib or not |
| 129 |
is_stdlib_import = all( |
| 130 |
self._is_stdlib_module(mod_info) for mod_info in modules |
| 131 |
) |
| 132 |
modules = self.iter_submodules(modules) |
| 133 |
|
| 134 |
module_names = [module.name for module in modules] |
| 135 |
if is_stdlib_import: |
| 136 |
module_names.extend(HARDCODED_SUBMODULES.get(path, ())) |
| 137 |
return [module_name for module_name in module_names |
| 138 |
if self.is_suggestion_match(module_name, prefix)] |
| 139 |
|
| 140 |
def _is_stdlib_module(self, module_info: pkgutil.ModuleInfo) -> bool: |
| 141 |
return (isinstance(module_info.module_finder, FileFinder) |
| 142 |
and module_info.module_finder.path == self._stdlib_path) |
| 143 |
|
| 144 |
def is_suggestion_match(self, module_name: str, prefix: str) -> bool: |
| 145 |
if prefix: |
| 146 |
return module_name.startswith(prefix) |
| 147 |
# For consistency with attribute completion, which |
| 148 |
# does not suggest private attributes unless requested. |
| 149 |
return not module_name.startswith("_") |
| 150 |
|
| 151 |
def iter_submodules(self, parent_modules: list[pkgutil.ModuleInfo]) -> Iterator[pkgutil.ModuleInfo]: |
| 152 |
"""Iterate over all submodules of the given parent modules.""" |
| 153 |
specs = [info.module_finder.find_spec(info.name, None) |
| 154 |
for info in parent_modules if info.ispkg] |
| 155 |
search_locations = set(chain.from_iterable( |
| 156 |
getattr(spec, 'submodule_search_locations', []) |
| 157 |
for spec in specs if spec |
| 158 |
)) |
| 159 |
return pkgutil.iter_modules(search_locations) |
| 160 |
|
| 161 |
def get_path_and_prefix(self, dotted_name: str) -> tuple[str, str]: |
| 162 |
""" |
| 163 |
Split a dotted name into an import path and a |
| 164 |
final prefix that is to be completed. |
| 165 |
|
| 166 |
Examples: |
| 167 |
'foo.bar' -> 'foo', 'bar' |
| 168 |
'foo.' -> 'foo', '' |
| 169 |
'.foo' -> '.', 'foo' |
| 170 |
""" |
| 171 |
if '.' not in dotted_name: |
| 172 |
return '', dotted_name |
| 173 |
if dotted_name.startswith('.'): |
| 174 |
stripped = dotted_name.lstrip('.') |
| 175 |
dots = '.' * (len(dotted_name) - len(stripped)) |
| 176 |
if '.' not in stripped: |
| 177 |
return dots, stripped |
| 178 |
path, prefix = stripped.rsplit('.', 1) |
| 179 |
return dots + path, prefix |
| 180 |
path, prefix = dotted_name.rsplit('.', 1) |
| 181 |
return path, prefix |
| 182 |
|
| 183 |
def format_completion(self, path: str, module: str) -> str: |
| 184 |
if path == '' or path.endswith('.'): |
| 185 |
return f'{path}{module}' |
| 186 |
return f'{path}.{module}' |
| 187 |
|
| 188 |
def resolve_relative_name(self, name: str, package: str) -> str | None: |
| 189 |
"""Resolve a relative module name to an absolute name. |
| 190 |
|
| 191 |
Example: resolve_relative_name('.foo', 'bar') -> 'bar.foo' |
| 192 |
""" |
| 193 |
# taken from importlib._bootstrap |
| 194 |
level = 0 |
| 195 |
for character in name: |
| 196 |
if character != '.': |
| 197 |
break |
| 198 |
level += 1 |
| 199 |
bits = package.rsplit('.', level - 1) |
| 200 |
if len(bits) < level: |
| 201 |
return None |
| 202 |
base = bits[0] |
| 203 |
name = name[level:] |
| 204 |
return f'{base}.{name}' if name else base |
| 205 |
|
| 206 |
@property |
| 207 |
def global_cache(self) -> list[pkgutil.ModuleInfo]: |
| 208 |
"""Global module cache""" |
| 209 |
if not self._global_cache or self._curr_sys_path != sys.path: |
| 210 |
self._curr_sys_path = sys.path[:] |
| 211 |
self._global_cache = list(pkgutil.iter_modules()) |
| 212 |
return self._global_cache |
| 213 |
|
| 214 |
|
| 215 |
class ImportParser: |
| 216 |
""" |
| 217 |
Parses incomplete import statements that are |
| 218 |
suitable for autocomplete suggestions. |
| 219 |
|
| 220 |
Examples: |
| 221 |
- import foo -> Result(from_name=None, name='foo') |
| 222 |
- import foo. -> Result(from_name=None, name='foo.') |
| 223 |
- from foo -> Result(from_name='foo', name=None) |
| 224 |
- from foo import bar -> Result(from_name='foo', name='bar') |
| 225 |
- from .foo import ( -> Result(from_name='.foo', name='') |
| 226 |
|
| 227 |
Note that the parser works in reverse order, starting from the |
| 228 |
last token in the input string. This makes the parser more robust |
| 229 |
when parsing multiple statements. |
| 230 |
""" |
| 231 |
_ignored_tokens = { |
| 232 |
token.INDENT, token.DEDENT, token.COMMENT, |
| 233 |
token.NL, token.NEWLINE, token.ENDMARKER |
| 234 |
} |
| 235 |
_keywords = {'import', 'from', 'as'} |
| 236 |
|
| 237 |
def __init__(self, code: str) -> None: |
| 238 |
self.code = code |
| 239 |
tokens = [] |
| 240 |
try: |
| 241 |
for t in tokenize.generate_tokens(StringIO(code).readline): |
| 242 |
if t.type not in self._ignored_tokens: |
| 243 |
tokens.append(t) |
| 244 |
except tokenize.TokenError as e: |
| 245 |
if 'unexpected EOF' not in str(e): |
| 246 |
# unexpected EOF is fine, since we're parsing an |
| 247 |
# incomplete statement, but other errors are not |
| 248 |
# because we may not have all the tokens so it's |
| 249 |
# safer to bail out |
| 250 |
tokens = [] |
| 251 |
except SyntaxError: |
| 252 |
tokens = [] |
| 253 |
self.tokens = TokenQueue(tokens[::-1]) |
| 254 |
|
| 255 |
def parse(self) -> tuple[str | None, str | None] | None: |
| 256 |
if not (res := self._parse()): |
| 257 |
return None |
| 258 |
return res.from_name, res.name |
| 259 |
|
| 260 |
def _parse(self) -> Result | None: |
| 261 |
with self.tokens.save_state(): |
| 262 |
return self.parse_from_import() |
| 263 |
with self.tokens.save_state(): |
| 264 |
return self.parse_import() |
| 265 |
|
| 266 |
def parse_import(self) -> Result: |
| 267 |
if self.code.rstrip().endswith('import') and self.code.endswith(' '): |
| 268 |
return Result(name='') |
| 269 |
if self.tokens.peek_string(','): |
| 270 |
name = '' |
| 271 |
else: |
| 272 |
if self.code.endswith(' '): |
| 273 |
raise ParseError('parse_import') |
| 274 |
name = self.parse_dotted_name() |
| 275 |
if name.startswith('.'): |
| 276 |
raise ParseError('parse_import') |
| 277 |
while self.tokens.peek_string(','): |
| 278 |
self.tokens.pop() |
| 279 |
self.parse_dotted_as_name() |
| 280 |
if self.tokens.peek_string('import'): |
| 281 |
return Result(name=name) |
| 282 |
raise ParseError('parse_import') |
| 283 |
|
| 284 |
def parse_from_import(self) -> Result: |
| 285 |
stripped = self.code.rstrip() |
| 286 |
if stripped.endswith('import') and self.code.endswith(' '): |
| 287 |
return Result(from_name=self.parse_empty_from_import(), name='') |
| 288 |
if stripped.endswith('from') and self.code.endswith(' '): |
| 289 |
return Result(from_name='') |
| 290 |
if self.tokens.peek_string('(') or self.tokens.peek_string(','): |
| 291 |
return Result(from_name=self.parse_empty_from_import(), name='') |
| 292 |
if self.code.endswith(' '): |
| 293 |
raise ParseError('parse_from_import') |
| 294 |
name = self.parse_dotted_name() |
| 295 |
if '.' in name: |
| 296 |
self.tokens.pop_string('from') |
| 297 |
return Result(from_name=name) |
| 298 |
if self.tokens.peek_string('from'): |
| 299 |
return Result(from_name=name) |
| 300 |
from_name = self.parse_empty_from_import() |
| 301 |
return Result(from_name=from_name, name=name) |
| 302 |
|
| 303 |
def parse_empty_from_import(self) -> str: |
| 304 |
if self.tokens.peek_string(','): |
| 305 |
self.tokens.pop() |
| 306 |
self.parse_as_names() |
| 307 |
if self.tokens.peek_string('('): |
| 308 |
self.tokens.pop() |
| 309 |
self.tokens.pop_string('import') |
| 310 |
return self.parse_from() |
| 311 |
|
| 312 |
def parse_from(self) -> str: |
| 313 |
from_name = self.parse_dotted_name() |
| 314 |
self.tokens.pop_string('from') |
| 315 |
return from_name |
| 316 |
|
| 317 |
def parse_dotted_as_name(self) -> str: |
| 318 |
self.tokens.pop_name() |
| 319 |
if self.tokens.peek_string('as'): |
| 320 |
self.tokens.pop() |
| 321 |
with self.tokens.save_state(): |
| 322 |
return self.parse_dotted_name() |
| 323 |
|
| 324 |
def parse_dotted_name(self) -> str: |
| 325 |
name = [] |
| 326 |
if self.tokens.peek_string('.'): |
| 327 |
name.append('.') |
| 328 |
self.tokens.pop() |
| 329 |
if (self.tokens.peek_name() |
| 330 |
and (tok := self.tokens.peek()) |
| 331 |
and tok.string not in self._keywords): |
| 332 |
name.append(self.tokens.pop_name()) |
| 333 |
if not name: |
| 334 |
raise ParseError('parse_dotted_name') |
| 335 |
while self.tokens.peek_string('.'): |
| 336 |
name.append('.') |
| 337 |
self.tokens.pop() |
| 338 |
if (self.tokens.peek_name() |
| 339 |
and (tok := self.tokens.peek()) |
| 340 |
and tok.string not in self._keywords): |
| 341 |
name.append(self.tokens.pop_name()) |
| 342 |
else: |
| 343 |
break |
| 344 |
|
| 345 |
while self.tokens.peek_string('.'): |
| 346 |
name.append('.') |
| 347 |
self.tokens.pop() |
| 348 |
return ''.join(name[::-1]) |
| 349 |
|
| 350 |
def parse_as_names(self) -> None: |
| 351 |
self.parse_as_name() |
| 352 |
while self.tokens.peek_string(','): |
| 353 |
self.tokens.pop() |
| 354 |
self.parse_as_name() |
| 355 |
|
| 356 |
def parse_as_name(self) -> None: |
| 357 |
self.tokens.pop_name() |
| 358 |
if self.tokens.peek_string('as'): |
| 359 |
self.tokens.pop() |
| 360 |
self.tokens.pop_name() |
| 361 |
|
| 362 |
|
| 363 |
class ParseError(Exception): |
| 364 |
pass |
| 365 |
|
| 366 |
|
| 367 |
@dataclass(frozen=True) |
| 368 |
class Result: |
| 369 |
from_name: str | None = None |
| 370 |
name: str | None = None |
| 371 |
|
| 372 |
|
| 373 |
class TokenQueue: |
| 374 |
"""Provides helper functions for working with a sequence of tokens.""" |
| 375 |
|
| 376 |
def __init__(self, tokens: list[TokenInfo]) -> None: |
| 377 |
self.tokens: list[TokenInfo] = tokens |
| 378 |
self.index: int = 0 |
| 379 |
self.stack: list[int] = [] |
| 380 |
|
| 381 |
@contextmanager |
| 382 |
def save_state(self) -> Any: |
| 383 |
try: |
| 384 |
self.stack.append(self.index) |
| 385 |
yield |
| 386 |
except ParseError: |
| 387 |
self.index = self.stack.pop() |
| 388 |
else: |
| 389 |
self.stack.pop() |
| 390 |
|
| 391 |
def __bool__(self) -> bool: |
| 392 |
return self.index < len(self.tokens) |
| 393 |
|
| 394 |
def peek(self) -> TokenInfo | None: |
| 395 |
if not self: |
| 396 |
return None |
| 397 |
return self.tokens[self.index] |
| 398 |
|
| 399 |
def peek_name(self) -> bool: |
| 400 |
if not (tok := self.peek()): |
| 401 |
return False |
| 402 |
return tok.type == token.NAME |
| 403 |
|
| 404 |
def pop_name(self) -> str: |
| 405 |
tok = self.pop() |
| 406 |
if tok.type != token.NAME: |
| 407 |
raise ParseError('pop_name') |
| 408 |
return tok.string |
| 409 |
|
| 410 |
def peek_string(self, string: str) -> bool: |
| 411 |
if not (tok := self.peek()): |
| 412 |
return False |
| 413 |
return tok.string == string |
| 414 |
|
| 415 |
def pop_string(self, string: str) -> str: |
| 416 |
tok = self.pop() |
| 417 |
if tok.string != string: |
| 418 |
raise ParseError('pop_string') |
| 419 |
return tok.string |
| 420 |
|
| 421 |
def pop(self) -> TokenInfo: |
| 422 |
if not self: |
| 423 |
raise ParseError('pop') |
| 424 |
tok = self.tokens[self.index] |
| 425 |
self.index += 1 |
| 426 |
return tok |