Debian Codegraph @

sphinxcontrib/spelling/filters.py

raw
1 #
2 # Copyright (c) 2010 Doug Hellmann. All rights reserved.
3 #
4 """Spelling checker extension for Sphinx.
5 """
6
7 # TODO - Words with multiple uppercase letters treated as classes and ignored
8
9 import builtins
10 import importlib
11 import subprocess
12 import sys
13 from xmlrpc import client as xmlrpc_client
14
15 from enchant.tokenize import Filter, get_tokenizer, tokenize, unit_tokenize
16 from sphinx.util import logging
17
18 logger = logging.getLogger(__name__)
19
20
21 class AcronymFilter(Filter):
22 """If a word looks like an acronym (all upper case letters),
23 ignore it.
24 """
25 def _skip(self, word):
26 return (
27 word.isupper() or # all caps
28 # pluralized acronym ("URLs")
29 (word[-1].lower() == 's' and word[:-1].isupper())
30 )
31
32
33 class list_tokenize(tokenize):
34
35 def __init__(self, words):
36 super().__init__('')
37 self._words = words
38
39 def next(self):
40 if not self._words:
41 raise StopIteration()
42 word = self._words.pop(0)
43 return (word, 0)
44
45
46 class ContractionFilter(Filter):
47 """Strip common contractions from words.
48 """
49 splits = {
50 "aren't": ["are", "not"],
51 "can't": ["can", "not"],
52 "could've": ["could", "have"],
53 "couldn't": ["could", "not"],
54 "didn't": ["did", "not"],
55 "doesn't": ["does", "not"],
56 "don't": ["do", "not"],
57 "hadn't": ["had", "not"],
58 "hasn't": ["has", "not"],
59 "haven't": ["have", "not"],
60 "he'd": ["he", "would"],
61 "he'll": ["he", "will"],
62 "he's": ["he", "is"],
63 "how'd": ["how", "would"],
64 "how'll": ["how", "will"],
65 "how's": ["how", "is"],
66 "i'd": ["I", "would"],
67 "i'll": ["I", "will"],
68 "i'm": ["I", "am"],
69 "i've": ["I", "have"],
70 "isn't": ["is", "not"],
71 "it'd": ["it", "would"],
72 "it'll": ["it", "will"],
73 "it's": ["it", "is"],
74 "ma'am": ["madam"],
75 "might've": ["might", "have"],
76 "mightn't": ["might", "not"],
77 "must've": ["must", "have"],
78 "mustn't": ["must", "not"],
79 "o'": ["of"],
80 "o'clock": ["of", "the", "clock"],
81 "she'd": ["she", "would"],
82 "she'll": ["she", "will"],
83 "she's": ["she", "is"],
84 "should've": ["should", "have"],
85 "shouldn't": ["should", "not"],
86 "that'd": ["that", "would"],
87 "that'll": ["that", "will"],
88 "that's": ["that", "is"],
89 "they'd": ["they", "would"],
90 "they'll": ["they", "will"],
91 "they're": ["they", "are"],
92 "they've": ["they", "have"],
93 "wasn't": ["was", "not"],
94 "we'd": ["we", "would"],
95 "we'll": ["we", "will"],
96 "we're": ["we", "are"],
97 "we've": ["we", "have"],
98 "weren't": ["were", "not"],
99 "what'd": ["what", "would"],
100 "what'll": ["what", "will"],
101 "what're": ["what", "are"],
102 "what's": ["what", "is"],
103 "when'd": ["when", "would"],
104 "when'll": ["when", "will"],
105 "when's": ["when", "is"],
106 "where'd": ["where", "would"],
107 "where'll": ["where", "will"],
108 "where's": ["where", "is"],
109 "who'd": ["who", "would"],
110 "who'll": ["who", "will"],
111 "who's": ["who", "is"],
112 "why'd": ["why", "would"],
113 "why'll": ["why", "will"],
114 "why's": ["why", "is"],
115 "won't": ["will", "not"],
116 "would've": ["would", "have"],
117 "wouldn't": ["would", "not"],
118 "you'd": ["you", "would"],
119 "you'll": ["you", "will"],
120 "you're": ["you", "are"],
121 "you've": ["you", "have"],
122 }
123
124 def _split(self, word):
125 # Fixed responses
126 if word.lower() in self.splits:
127 return list_tokenize(self.splits[word.lower()])
128
129 # Possessive
130 if word.lower().endswith("'s"):
131 return unit_tokenize(word[:-2])
132
133 # * not
134 if word.lower().endswith("n't"):
135 return unit_tokenize(word[:-3])
136
137 return unit_tokenize(word)
138
139
140 class IgnoreWordsFilter(Filter):
141 """Given a set of words, ignore them all.
142 """
143
144 def __init__(self, tokenizer, word_set):
145 self.word_set = set(word_set)
146 super().__init__(tokenizer)
147
148 def _skip(self, word):
149 return word in self.word_set
150
151
152 class IgnoreWordsFilterFactory:
153
154 def __init__(self, words):
155 self.words = words
156
157 def __call__(self, tokenizer):
158 return IgnoreWordsFilter(tokenizer, self.words)
159
160
161 class PyPIFilterFactory(IgnoreWordsFilterFactory):
162 """Build an IgnoreWordsFilter for all of the names of packages on PyPI.
163 """
164 def __init__(self):
165 client = xmlrpc_client.ServerProxy('https://pypi.python.org/pypi')
166 super().__init__(client.list_packages())
167
168
169 class PythonBuiltinsFilter(Filter):
170 """Ignore names of built-in Python symbols.
171 """
172 def _skip(self, word):
173 return hasattr(builtins, word)
174
175
176 class ImportableModuleFilter(Filter):
177 """Ignore names of modules that we could import.
178 """
179 def __init__(self, tokenizer):
180 super().__init__(tokenizer)
181 self.found_modules = set(sys.builtin_module_names)
182 self.sought_modules = self.found_modules.copy()
183 # By adding __main__ to the list of sought modules but not
184 # found modules we ensure that it is never recognized as a
185 # valid module, which is consistent with the behavior before
186 # version 7.3.1. See
187 # https://github.com/sphinx-contrib/spelling/issues/141
188 self.sought_modules.add('__main__')
189
190 def _skip(self, word):
191 # If the word looks like a python module filename, strip the
192 # extension to avoid the side-effect of actually importing the
193 # module. This prevents, for example, 'setup.py' triggering an
194 # import of the setup module during a doc build, which makes
195 # it look like Sphinx is complaining about a commandline
196 # argument. See
197 # https://github.com/sphinx-contrib/spelling/issues/142
198 if word.endswith('.py'):
199 logger.debug(
200 'removing .py extension from %r before searching for module',
201 word)
202 word = word[:-3]
203
204 valid_module_name = all(n.isidentifier() for n in word.split('.'))
205 if not valid_module_name:
206 return False
207
208 if word not in self.sought_modules:
209 self.sought_modules.add(word)
210 try:
211 mod = importlib.util.find_spec(word)
212 except BaseException as err:
213 # This could be an ImportError, SystemExit, some more detailed
214 # error out of distutils, or something else triggered
215 # by failing to be able to import a parent package to
216 # use the metadata to search for a subpackage.
217 logger.debug('find_spec(%r) failed, invalid module name: %s',
218 word, err)
219 else:
220 if mod is not None:
221 self.found_modules.add(word)
222
223 return word in self.found_modules
224
225
226 class ContributorFilter(IgnoreWordsFilter):
227 """Accept information about contributors as spelled correctly.
228
229 Look in the git history for authors and committers and accept
230 tokens that are in the set.
231 """
232
233 _pretty_format = (
234 '%(trailers:key=Co-Authored-By,separator=%x0A)%x0A%an%x0A%cn'
235 )
236
237 def __init__(self, tokenizer):
238 contributors = self._get_contributors()
239 super().__init__(tokenizer, contributors)
240
241 def _get_contributors(self):
242 logger.info('Scanning contributors')
243 cmd = ['git', 'log', '--quiet', '--no-color',
244 f'--pretty=format:{self._pretty_format}']
245
246 try:
247 p = subprocess.run(cmd, check=True, stdout=subprocess.PIPE)
248 except subprocess.CalledProcessError as err:
249 logger.warning('Called: %s', ' '.join(cmd))
250 logger.warning('Failed to scan contributors: %s', err)
251 return set()
252 output = p.stdout.decode('utf-8')
253 tokenizer = get_tokenizer('en_US', filters=[])
254 return {word for word, pos in tokenizer(output)}