Debian Codegraph @

Lib/test/test_shlex.py

raw
1 import io
2 import itertools
3 import shlex
4 import string
5 import unittest
6 from test.support import cpython_only
7 from test.support import import_helper
8
9
10 # The original test data set was from shellwords, by Hartmut Goebel.
11
12 data = r"""x|x|
13 foo bar|foo|bar|
14 foo bar|foo|bar|
15 foo bar |foo|bar|
16 foo bar bla fasel|foo|bar|bla|fasel|
17 x y z xxxx|x|y|z|xxxx|
18 \x bar|\|x|bar|
19 \ x bar|\|x|bar|
20 \ bar|\|bar|
21 foo \x bar|foo|\|x|bar|
22 foo \ x bar|foo|\|x|bar|
23 foo \ bar|foo|\|bar|
24 foo "bar" bla|foo|"bar"|bla|
25 "foo" "bar" "bla"|"foo"|"bar"|"bla"|
26 "foo" bar "bla"|"foo"|bar|"bla"|
27 "foo" bar bla|"foo"|bar|bla|
28 foo 'bar' bla|foo|'bar'|bla|
29 'foo' 'bar' 'bla'|'foo'|'bar'|'bla'|
30 'foo' bar 'bla'|'foo'|bar|'bla'|
31 'foo' bar bla|'foo'|bar|bla|
32 blurb foo"bar"bar"fasel" baz|blurb|foo"bar"bar"fasel"|baz|
33 blurb foo'bar'bar'fasel' baz|blurb|foo'bar'bar'fasel'|baz|
34 ""|""|
35 ''|''|
36 foo "" bar|foo|""|bar|
37 foo '' bar|foo|''|bar|
38 foo "" "" "" bar|foo|""|""|""|bar|
39 foo '' '' '' bar|foo|''|''|''|bar|
40 \""|\|""|
41 "\"|"\"|
42 "foo\ bar"|"foo\ bar"|
43 "foo\\ bar"|"foo\\ bar"|
44 "foo\\ bar\"|"foo\\ bar\"|
45 "foo\\" bar\""|"foo\\"|bar|\|""|
46 "foo\\ bar\" dfadf"|"foo\\ bar\"|dfadf"|
47 "foo\\\ bar\" dfadf"|"foo\\\ bar\"|dfadf"|
48 "foo\\\x bar\" dfadf"|"foo\\\x bar\"|dfadf"|
49 "foo\x bar\" dfadf"|"foo\x bar\"|dfadf"|
50 \''|\|''|
51 'foo\ bar'|'foo\ bar'|
52 'foo\\ bar'|'foo\\ bar'|
53 "foo\\\x bar\" df'a\ 'df'|"foo\\\x bar\"|df'a|\|'df'|
54 \"foo"|\|"foo"|
55 \"foo"\x|\|"foo"|\|x|
56 "foo\x"|"foo\x"|
57 "foo\ "|"foo\ "|
58 foo\ xx|foo|\|xx|
59 foo\ x\x|foo|\|x|\|x|
60 foo\ x\x\""|foo|\|x|\|x|\|""|
61 "foo\ x\x"|"foo\ x\x"|
62 "foo\ x\x\\"|"foo\ x\x\\"|
63 "foo\ x\x\\""foobar"|"foo\ x\x\\"|"foobar"|
64 "foo\ x\x\\"\''"foobar"|"foo\ x\x\\"|\|''|"foobar"|
65 "foo\ x\x\\"\'"fo'obar"|"foo\ x\x\\"|\|'"fo'|obar"|
66 "foo\ x\x\\"\'"fo'obar" 'don'\''t'|"foo\ x\x\\"|\|'"fo'|obar"|'don'|\|''|t'|
67 'foo\ bar'|'foo\ bar'|
68 'foo\\ bar'|'foo\\ bar'|
69 foo\ bar|foo|\|bar|
70 foo#bar\nbaz|foobaz|
71 :-) ;-)|:|-|)|;|-|)|
72 áéíóú|á|é|í|ó|ú|
73 """
74
75 posix_data = r"""x|x|
76 foo bar|foo|bar|
77 foo bar|foo|bar|
78 foo bar |foo|bar|
79 foo bar bla fasel|foo|bar|bla|fasel|
80 x y z xxxx|x|y|z|xxxx|
81 \x bar|x|bar|
82 \ x bar| x|bar|
83 \ bar| bar|
84 foo \x bar|foo|x|bar|
85 foo \ x bar|foo| x|bar|
86 foo \ bar|foo| bar|
87 foo "bar" bla|foo|bar|bla|
88 "foo" "bar" "bla"|foo|bar|bla|
89 "foo" bar "bla"|foo|bar|bla|
90 "foo" bar bla|foo|bar|bla|
91 foo 'bar' bla|foo|bar|bla|
92 'foo' 'bar' 'bla'|foo|bar|bla|
93 'foo' bar 'bla'|foo|bar|bla|
94 'foo' bar bla|foo|bar|bla|
95 blurb foo"bar"bar"fasel" baz|blurb|foobarbarfasel|baz|
96 blurb foo'bar'bar'fasel' baz|blurb|foobarbarfasel|baz|
97 ""||
98 ''||
99 foo "" bar|foo||bar|
100 foo '' bar|foo||bar|
101 foo "" "" "" bar|foo||||bar|
102 foo '' '' '' bar|foo||||bar|
103 \"|"|
104 "\""|"|
105 "foo\ bar"|foo\ bar|
106 "foo\\ bar"|foo\ bar|
107 "foo\\ bar\""|foo\ bar"|
108 "foo\\" bar\"|foo\|bar"|
109 "foo\\ bar\" dfadf"|foo\ bar" dfadf|
110 "foo\\\ bar\" dfadf"|foo\\ bar" dfadf|
111 "foo\\\x bar\" dfadf"|foo\\x bar" dfadf|
112 "foo\x bar\" dfadf"|foo\x bar" dfadf|
113 \'|'|
114 'foo\ bar'|foo\ bar|
115 'foo\\ bar'|foo\\ bar|
116 "foo\\\x bar\" df'a\ 'df"|foo\\x bar" df'a\ 'df|
117 \"foo|"foo|
118 \"foo\x|"foox|
119 "foo\x"|foo\x|
120 "foo\ "|foo\ |
121 foo\ xx|foo xx|
122 foo\ x\x|foo xx|
123 foo\ x\x\"|foo xx"|
124 "foo\ x\x"|foo\ x\x|
125 "foo\ x\x\\"|foo\ x\x\|
126 "foo\ x\x\\""foobar"|foo\ x\x\foobar|
127 "foo\ x\x\\"\'"foobar"|foo\ x\x\'foobar|
128 "foo\ x\x\\"\'"fo'obar"|foo\ x\x\'fo'obar|
129 "foo\ x\x\\"\'"fo'obar" 'don'\''t'|foo\ x\x\'fo'obar|don't|
130 "foo\ x\x\\"\'"fo'obar" 'don'\''t' \\|foo\ x\x\'fo'obar|don't|\|
131 'foo\ bar'|foo\ bar|
132 'foo\\ bar'|foo\\ bar|
133 foo\ bar|foo bar|
134 foo#bar\nbaz|foo|baz|
135 :-) ;-)|:-)|;-)|
136 áéíóú|áéíóú|
137 """
138
139 class ShlexTest(unittest.TestCase):
140 def setUp(self):
141 self.data = [x.split("|")[:-1]
142 for x in data.splitlines()]
143 self.posix_data = [x.split("|")[:-1]
144 for x in posix_data.splitlines()]
145 for item in self.data:
146 item[0] = item[0].replace(r"\n", "\n")
147 for item in self.posix_data:
148 item[0] = item[0].replace(r"\n", "\n")
149
150 def splitTest(self, data, comments):
151 for i in range(len(data)):
152 l = shlex.split(data[i][0], comments=comments)
153 self.assertEqual(l, data[i][1:],
154 "%s: %s != %s" %
155 (data[i][0], l, data[i][1:]))
156
157 def oldSplit(self, s):
158 ret = []
159 lex = shlex.shlex(io.StringIO(s))
160 tok = lex.get_token()
161 while tok:
162 ret.append(tok)
163 tok = lex.get_token()
164 return ret
165
166 def testSplitNone(self):
167 with self.assertRaises(ValueError):
168 shlex.split(None)
169
170 def testSplitPosix(self):
171 """Test data splitting with posix parser"""
172 self.splitTest(self.posix_data, comments=True)
173
174 def testCompat(self):
175 """Test compatibility interface"""
176 for i in range(len(self.data)):
177 l = self.oldSplit(self.data[i][0])
178 self.assertEqual(l, self.data[i][1:],
179 "%s: %s != %s" %
180 (self.data[i][0], l, self.data[i][1:]))
181
182 def testSyntaxSplitAmpersandAndPipe(self):
183 """Test handling of syntax splitting of &, |"""
184 # Could take these forms: &&, &, |&, ;&, ;;&
185 # of course, the same applies to | and ||
186 # these should all parse to the same output
187 for delimiter in ('&&', '&', '|&', ';&', ';;&',
188 '||', '|', '&|', ';|', ';;|'):
189 src = ['echo hi %s echo bye' % delimiter,
190 'echo hi%secho bye' % delimiter]
191 ref = ['echo', 'hi', delimiter, 'echo', 'bye']
192 for ss, ws in itertools.product(src, (False, True)):
193 s = shlex.shlex(ss, punctuation_chars=True)
194 s.whitespace_split = ws
195 result = list(s)
196 self.assertEqual(ref, result,
197 "While splitting '%s' [ws=%s]" % (ss, ws))
198
199 def testSyntaxSplitSemicolon(self):
200 """Test handling of syntax splitting of ;"""
201 # Could take these forms: ;, ;;, ;&, ;;&
202 # these should all parse to the same output
203 for delimiter in (';', ';;', ';&', ';;&'):
204 src = ['echo hi %s echo bye' % delimiter,
205 'echo hi%s echo bye' % delimiter,
206 'echo hi%secho bye' % delimiter]
207 ref = ['echo', 'hi', delimiter, 'echo', 'bye']
208 for ss, ws in itertools.product(src, (False, True)):
209 s = shlex.shlex(ss, punctuation_chars=True)
210 s.whitespace_split = ws
211 result = list(s)
212 self.assertEqual(ref, result,
213 "While splitting '%s' [ws=%s]" % (ss, ws))
214
215 def testSyntaxSplitRedirect(self):
216 """Test handling of syntax splitting of >"""
217 # of course, the same applies to <, |
218 # these should all parse to the same output
219 for delimiter in ('<', '|'):
220 src = ['echo hi %s out' % delimiter,
221 'echo hi%s out' % delimiter,
222 'echo hi%sout' % delimiter]
223 ref = ['echo', 'hi', delimiter, 'out']
224 for ss, ws in itertools.product(src, (False, True)):
225 s = shlex.shlex(ss, punctuation_chars=True)
226 result = list(s)
227 self.assertEqual(ref, result,
228 "While splitting '%s' [ws=%s]" % (ss, ws))
229
230 def testSyntaxSplitParen(self):
231 """Test handling of syntax splitting of ()"""
232 # these should all parse to the same output
233 src = ['( echo hi )',
234 '(echo hi)']
235 ref = ['(', 'echo', 'hi', ')']
236 for ss, ws in itertools.product(src, (False, True)):
237 s = shlex.shlex(ss, punctuation_chars=True)
238 s.whitespace_split = ws
239 result = list(s)
240 self.assertEqual(ref, result,
241 "While splitting '%s' [ws=%s]" % (ss, ws))
242
243 def testSyntaxSplitCustom(self):
244 """Test handling of syntax splitting with custom chars"""
245 ss = "~/a&&b-c --color=auto||d *.py?"
246 ref = ['~/a', '&', '&', 'b-c', '--color=auto', '||', 'd', '*.py?']
247 s = shlex.shlex(ss, punctuation_chars="|")
248 result = list(s)
249 self.assertEqual(ref, result, "While splitting '%s' [ws=False]" % ss)
250 ref = ['~/a&&b-c', '--color=auto', '||', 'd', '*.py?']
251 s = shlex.shlex(ss, punctuation_chars="|")
252 s.whitespace_split = True
253 result = list(s)
254 self.assertEqual(ref, result, "While splitting '%s' [ws=True]" % ss)
255
256 def testTokenTypes(self):
257 """Test that tokens are split with types as expected."""
258 for source, expected in (
259 ('a && b || c',
260 [('a', 'a'), ('&&', 'c'), ('b', 'a'),
261 ('||', 'c'), ('c', 'a')]),
262 ):
263 s = shlex.shlex(source, punctuation_chars=True)
264 observed = []
265 while True:
266 t = s.get_token()
267 if t == s.eof:
268 break
269 if t[0] in s.punctuation_chars:
270 tt = 'c'
271 else:
272 tt = 'a'
273 observed.append((t, tt))
274 self.assertEqual(observed, expected)
275
276 def testPunctuationInWordChars(self):
277 """Test that any punctuation chars are removed from wordchars"""
278 s = shlex.shlex('a_b__c', punctuation_chars='_')
279 self.assertNotIn('_', s.wordchars)
280 self.assertEqual(list(s), ['a', '_', 'b', '__', 'c'])
281
282 def testPunctuationWithWhitespaceSplit(self):
283 """Test that with whitespace_split, behaviour is as expected"""
284 s = shlex.shlex('a && b || c', punctuation_chars='&')
285 # whitespace_split is False, so splitting will be based on
286 # punctuation_chars
287 self.assertEqual(list(s), ['a', '&&', 'b', '|', '|', 'c'])
288 s = shlex.shlex('a && b || c', punctuation_chars='&')
289 s.whitespace_split = True
290 # whitespace_split is True, so splitting will be based on
291 # white space
292 self.assertEqual(list(s), ['a', '&&', 'b', '||', 'c'])
293
294 def testPunctuationWithPosix(self):
295 """Test that punctuation_chars and posix behave correctly together."""
296 # see Issue #29132
297 s = shlex.shlex('f >"abc"', posix=True, punctuation_chars=True)
298 self.assertEqual(list(s), ['f', '>', 'abc'])
299 s = shlex.shlex('f >\\"abc\\"', posix=True, punctuation_chars=True)
300 self.assertEqual(list(s), ['f', '>', '"abc"'])
301
302 def testEmptyStringHandling(self):
303 """Test that parsing of empty strings is correctly handled."""
304 # see Issue #21999
305 expected = ['', ')', 'abc']
306 for punct in (False, True):
307 s = shlex.shlex("'')abc", posix=True, punctuation_chars=punct)
308 slist = list(s)
309 self.assertEqual(slist, expected)
310 expected = ["''", ')', 'abc']
311 s = shlex.shlex("'')abc", punctuation_chars=True)
312 self.assertEqual(list(s), expected)
313
314 def testUnicodeHandling(self):
315 """Test punctuation_chars and whitespace_split handle unicode."""
316 ss = "\u2119\u01b4\u2602\u210c\u00f8\u1f24"
317 # Should be parsed as one complete token (whitespace_split=True).
318 ref = ['\u2119\u01b4\u2602\u210c\u00f8\u1f24']
319 s = shlex.shlex(ss, punctuation_chars=True)
320 s.whitespace_split = True
321 self.assertEqual(list(s), ref)
322 # Without whitespace_split, uses wordchars and splits on all.
323 ref = ['\u2119', '\u01b4', '\u2602', '\u210c', '\u00f8', '\u1f24']
324 s = shlex.shlex(ss, punctuation_chars=True)
325 self.assertEqual(list(s), ref)
326
327 def testQuote(self):
328 safeunquoted = string.ascii_letters + string.digits + '@%_-+=:,./'
329 unicode_sample = '\xe9\xe0\xdf' # e + acute accent, a + grave, sharp s
330 unsafe = '"`$\\!' + unicode_sample
331
332 self.assertEqual(shlex.quote(''), "''")
333 self.assertEqual(shlex.quote(None), "''")
334 self.assertEqual(shlex.quote(safeunquoted), safeunquoted)
335 self.assertEqual(shlex.quote('test file name'), "'test file name'")
336 for u in unsafe:
337 self.assertEqual(shlex.quote('test%sname' % u),
338 "'test%sname'" % u)
339 for u in unsafe:
340 self.assertEqual(shlex.quote("test%s'name'" % u),
341 "'test%s'\"'\"'name'\"'\"''" % u)
342 self.assertRaises(TypeError, shlex.quote, 42)
343 self.assertRaises(TypeError, shlex.quote, b"abc")
344
345 def testJoin(self):
346 for split_command, command in [
347 (['a ', 'b'], "'a ' b"),
348 (['a', ' b'], "a ' b'"),
349 (['a', ' ', 'b'], "a ' ' b"),
350 (['"a', 'b"'], '\'"a\' \'b"\''),
351 ]:
352 with self.subTest(command=command):
353 joined = shlex.join(split_command)
354 self.assertEqual(joined, command)
355
356 def testJoinRoundtrip(self):
357 all_data = self.data + self.posix_data
358 for command, *split_command in all_data:
359 with self.subTest(command=command):
360 joined = shlex.join(split_command)
361 resplit = shlex.split(joined)
362 self.assertEqual(split_command, resplit)
363
364 def testPunctuationCharsReadOnly(self):
365 punctuation_chars = "/|$%^"
366 shlex_instance = shlex.shlex(punctuation_chars=punctuation_chars)
367 self.assertEqual(shlex_instance.punctuation_chars, punctuation_chars)
368 with self.assertRaises(AttributeError):
369 shlex_instance.punctuation_chars = False
370
371 @cpython_only
372 def test_lazy_imports(self):
373 import_helper.ensure_lazy_imports('shlex', {'collections', 're', 'os'})
374
375
376 # Allow this test to be used with old shlex.py
377 if not getattr(shlex, "split", None):
378 for methname in dir(ShlexTest):
379 if methname.startswith("test") and methname != "testCompat":
380 delattr(ShlexTest, methname)
381
382 if __name__ == "__main__":
383 unittest.main()