Debian Codegraph @

Lib/test/test_string_literals.py

raw
1 r"""Test correct treatment of various string literals by the parser.
2
3 There are four types of string literals:
4
5 'abc' -- normal str
6 r'abc' -- raw str
7 b'xyz' -- normal bytes
8 br'xyz' | rb'xyz' -- raw bytes
9
10 The difference between normal and raw strings is of course that in a
11 raw string, \ escapes (while still used to determine the end of the
12 literal) are not interpreted, so that r'\x00' contains four
13 characters: a backslash, an x, and two zeros; while '\x00' contains a
14 single character (code point zero).
15
16 The tricky thing is what should happen when non-ASCII bytes are used
17 inside literals. For bytes literals, this is considered illegal. But
18 for str literals, those bytes are supposed to be decoded using the
19 encoding declared for the file (UTF-8 by default).
20
21 We have to test this with various file encodings. We also test it with
22 exec()/eval(), which uses a different code path.
23
24 This file is really about correct treatment of encodings and
25 backslashes. It doesn't concern itself with issues like single
26 vs. double quotes or singly- vs. triply-quoted strings: that's dealt
27 with elsewhere (I assume).
28 """
29
30 import os
31 import sys
32 import shutil
33 import tempfile
34 import unittest
35 import warnings
36
37
38 TEMPLATE = r"""# coding: %s
39 a = 'x'
40 assert ord(a) == 120
41 b = '\x01'
42 assert ord(b) == 1
43 c = r'\x01'
44 assert list(map(ord, c)) == [92, 120, 48, 49]
45 d = '\x81'
46 assert ord(d) == 0x81
47 e = r'\x81'
48 assert list(map(ord, e)) == [92, 120, 56, 49]
49 f = '\u1881'
50 assert ord(f) == 0x1881
51 g = r'\u1881'
52 assert list(map(ord, g)) == [92, 117, 49, 56, 56, 49]
53 h = '\U0001d120'
54 assert ord(h) == 0x1d120
55 i = r'\U0001d120'
56 assert list(map(ord, i)) == [92, 85, 48, 48, 48, 49, 100, 49, 50, 48]
57 """
58
59
60 def byte(i):
61 return bytes([i])
62
63
64 class TestLiterals(unittest.TestCase):
65
66 def setUp(self):
67 self.save_path = sys.path[:]
68 self.tmpdir = tempfile.mkdtemp()
69 sys.path.insert(0, self.tmpdir)
70
71 def tearDown(self):
72 sys.path[:] = self.save_path
73 shutil.rmtree(self.tmpdir, ignore_errors=True)
74
75 def test_template(self):
76 # Check that the template doesn't contain any non-printables
77 # except for \n.
78 for c in TEMPLATE:
79 assert c == '\n' or ' ' <= c <= '~', repr(c)
80
81 def test_eval_str_normal(self):
82 self.assertEqual(eval(""" 'x' """), 'x')
83 self.assertEqual(eval(r""" '\x01' """), chr(1))
84 self.assertEqual(eval(""" '\x01' """), chr(1))
85 self.assertEqual(eval(r""" '\x81' """), chr(0x81))
86 self.assertEqual(eval(""" '\x81' """), chr(0x81))
87 self.assertEqual(eval(r""" '\u1881' """), chr(0x1881))
88 self.assertEqual(eval(""" '\u1881' """), chr(0x1881))
89 self.assertEqual(eval(r""" '\U0001d120' """), chr(0x1d120))
90 self.assertEqual(eval(""" '\U0001d120' """), chr(0x1d120))
91
92 def test_eval_str_incomplete(self):
93 self.assertRaises(SyntaxError, eval, r""" '\x' """)
94 self.assertRaises(SyntaxError, eval, r""" '\x0' """)
95 self.assertRaises(SyntaxError, eval, r""" '\u' """)
96 self.assertRaises(SyntaxError, eval, r""" '\u0' """)
97 self.assertRaises(SyntaxError, eval, r""" '\u00' """)
98 self.assertRaises(SyntaxError, eval, r""" '\u000' """)
99 self.assertRaises(SyntaxError, eval, r""" '\U' """)
100 self.assertRaises(SyntaxError, eval, r""" '\U0' """)
101 self.assertRaises(SyntaxError, eval, r""" '\U00' """)
102 self.assertRaises(SyntaxError, eval, r""" '\U000' """)
103 self.assertRaises(SyntaxError, eval, r""" '\U0000' """)
104 self.assertRaises(SyntaxError, eval, r""" '\U00000' """)
105 self.assertRaises(SyntaxError, eval, r""" '\U000000' """)
106 self.assertRaises(SyntaxError, eval, r""" '\U0000000' """)
107
108 def test_eval_str_invalid_escape(self):
109 for b in range(1, 128):
110 if b in b"""\n\r"'01234567NU\\abfnrtuvx""":
111 continue
112 with self.assertWarns(SyntaxWarning):
113 self.assertEqual(eval(r"'\%c'" % b), '\\' + chr(b))
114
115 with warnings.catch_warnings(record=True) as w:
116 warnings.simplefilter('always', category=SyntaxWarning)
117 eval("'''\n\\z'''")
118 self.assertEqual(len(w), 1)
119 self.assertEqual(str(w[0].message), r'"\z" is an invalid escape sequence. '
120 r'Such sequences will not work in the future. '
121 r'Did you mean "\\z"? A raw string is also an option.')
122 self.assertEqual(w[0].filename, '<string>')
123 self.assertEqual(w[0].lineno, 2)
124
125 with warnings.catch_warnings(record=True) as w:
126 warnings.simplefilter('error', category=SyntaxWarning)
127 with self.assertRaises(SyntaxError) as cm:
128 eval("'''\n\\z'''")
129 exc = cm.exception
130 self.assertEqual(w, [])
131 self.assertEqual(exc.msg, r'"\z" is an invalid escape sequence. '
132 r'Did you mean "\\z"? A raw string is also an option.')
133 self.assertEqual(exc.filename, '<string>')
134 self.assertEqual(exc.lineno, 2)
135 self.assertEqual(exc.offset, 1)
136
137 # Check that the warning is raised only once if there are syntax errors
138
139 with warnings.catch_warnings(record=True) as w:
140 warnings.simplefilter('always', category=SyntaxWarning)
141 with self.assertRaises(SyntaxError) as cm:
142 eval("'\\e' $")
143 exc = cm.exception
144 self.assertEqual(len(w), 1)
145 self.assertEqual(w[0].category, SyntaxWarning)
146 self.assertRegex(str(w[0].message), 'invalid escape sequence')
147 self.assertEqual(w[0].filename, '<string>')
148
149 def test_eval_str_invalid_octal_escape(self):
150 for i in range(0o400, 0o1000):
151 with self.assertWarns(SyntaxWarning):
152 self.assertEqual(eval(r"'\%o'" % i), chr(i))
153
154 with warnings.catch_warnings(record=True) as w:
155 warnings.simplefilter('always', category=SyntaxWarning)
156 eval("'''\n\\407'''")
157 self.assertEqual(len(w), 1)
158 self.assertEqual(str(w[0].message),
159 r'"\407" is an invalid octal escape sequence. '
160 r'Such sequences will not work in the future. '
161 r'Did you mean "\\407"? A raw string is also an option.')
162 self.assertEqual(w[0].filename, '<string>')
163 self.assertEqual(w[0].lineno, 2)
164
165 with warnings.catch_warnings(record=True) as w:
166 warnings.simplefilter('error', category=SyntaxWarning)
167 with self.assertRaises(SyntaxError) as cm:
168 eval("'''\n\\407'''")
169 exc = cm.exception
170 self.assertEqual(w, [])
171 self.assertEqual(exc.msg, r'"\407" is an invalid octal escape sequence. '
172 r'Did you mean "\\407"? A raw string is also an option.')
173 self.assertEqual(exc.filename, '<string>')
174 self.assertEqual(exc.lineno, 2)
175 self.assertEqual(exc.offset, 1)
176
177 def test_invalid_escape_locations_with_offset(self):
178 with warnings.catch_warnings(record=True) as w:
179 warnings.simplefilter('always', category=SyntaxWarning)
180 eval("\"'''''''''''''''''''''invalid\\ Escape\"")
181 self.assertEqual(len(w), 1)
182 self.assertEqual(str(w[0].message),
183 r'"\ " is an invalid escape sequence. Such sequences '
184 r'will not work in the future. Did you mean "\\ "? '
185 r'A raw string is also an option.')
186 self.assertEqual(w[0].filename, '<string>')
187 self.assertEqual(w[0].lineno, 1)
188
189 with warnings.catch_warnings(record=True) as w:
190 warnings.simplefilter('always', category=SyntaxWarning)
191 eval("\"''Incorrect \\ logic?\"")
192 self.assertEqual(len(w), 1)
193 self.assertEqual(str(w[0].message),
194 r'"\ " is an invalid escape sequence. Such sequences '
195 r'will not work in the future. Did you mean "\\ "? '
196 r'A raw string is also an option.')
197 self.assertEqual(w[0].filename, '<string>')
198 self.assertEqual(w[0].lineno, 1)
199
200 def test_eval_str_raw(self):
201 self.assertEqual(eval(""" r'x' """), 'x')
202 self.assertEqual(eval(r""" r'\x01' """), '\\' + 'x01')
203 self.assertEqual(eval(""" r'\x01' """), chr(1))
204 self.assertEqual(eval(r""" r'\x81' """), '\\' + 'x81')
205 self.assertEqual(eval(""" r'\x81' """), chr(0x81))
206 self.assertEqual(eval(r""" r'\u1881' """), '\\' + 'u1881')
207 self.assertEqual(eval(""" r'\u1881' """), chr(0x1881))
208 self.assertEqual(eval(r""" r'\U0001d120' """), '\\' + 'U0001d120')
209 self.assertEqual(eval(""" r'\U0001d120' """), chr(0x1d120))
210
211 def test_eval_bytes_normal(self):
212 self.assertEqual(eval(""" b'x' """), b'x')
213 self.assertEqual(eval(r""" b'\x01' """), byte(1))
214 self.assertEqual(eval(""" b'\x01' """), byte(1))
215 self.assertEqual(eval(r""" b'\x81' """), byte(0x81))
216 self.assertRaises(SyntaxError, eval, """ b'\x81' """)
217 self.assertEqual(eval(r""" br'\u1881' """), b'\\' + b'u1881')
218 self.assertRaises(SyntaxError, eval, """ b'\u1881' """)
219 self.assertEqual(eval(r""" br'\U0001d120' """), b'\\' + b'U0001d120')
220 self.assertRaises(SyntaxError, eval, """ b'\U0001d120' """)
221
222 def test_eval_bytes_incomplete(self):
223 self.assertRaises(SyntaxError, eval, r""" b'\x' """)
224 self.assertRaises(SyntaxError, eval, r""" b'\x0' """)
225
226 def test_eval_bytes_invalid_escape(self):
227 for b in range(1, 128):
228 if b in b"""\n\r"'01234567\\abfnrtvx""":
229 continue
230 with self.assertWarns(SyntaxWarning):
231 self.assertEqual(eval(r"b'\%c'" % b), b'\\' + bytes([b]))
232
233 with warnings.catch_warnings(record=True) as w:
234 warnings.simplefilter('always', category=SyntaxWarning)
235 eval("b'''\n\\z'''")
236 self.assertEqual(len(w), 1)
237 self.assertEqual(str(w[0].message), r'"\z" is an invalid escape sequence. '
238 r'Such sequences will not work in the future. '
239 r'Did you mean "\\z"? A raw string is also an option.')
240 self.assertEqual(w[0].filename, '<string>')
241 self.assertEqual(w[0].lineno, 2)
242
243 with warnings.catch_warnings(record=True) as w:
244 warnings.simplefilter('error', category=SyntaxWarning)
245 with self.assertRaises(SyntaxError) as cm:
246 eval("b'''\n\\z'''")
247 exc = cm.exception
248 self.assertEqual(w, [])
249 self.assertEqual(exc.msg, r'"\z" is an invalid escape sequence. '
250 r'Did you mean "\\z"? A raw string is also an option.')
251 self.assertEqual(exc.filename, '<string>')
252 self.assertEqual(exc.lineno, 2)
253
254 def test_eval_bytes_invalid_octal_escape(self):
255 for i in range(0o400, 0o1000):
256 with self.assertWarns(SyntaxWarning):
257 self.assertEqual(eval(r"b'\%o'" % i), bytes([i & 0o377]))
258
259 with warnings.catch_warnings(record=True) as w:
260 warnings.simplefilter('always', category=SyntaxWarning)
261 eval("b'''\n\\407'''")
262 self.assertEqual(len(w), 1)
263 self.assertEqual(str(w[0].message), r'"\407" is an invalid octal escape sequence. '
264 r'Such sequences will not work in the future. '
265 r'Did you mean "\\407"? A raw string is also an option.')
266 self.assertEqual(w[0].filename, '<string>')
267 self.assertEqual(w[0].lineno, 2)
268
269 with warnings.catch_warnings(record=True) as w:
270 warnings.simplefilter('error', category=SyntaxWarning)
271 with self.assertRaises(SyntaxError) as cm:
272 eval("b'''\n\\407'''")
273 exc = cm.exception
274 self.assertEqual(w, [])
275 self.assertEqual(exc.msg, r'"\407" is an invalid octal escape sequence. '
276 r'Did you mean "\\407"? A raw string is also an option.')
277 self.assertEqual(exc.filename, '<string>')
278 self.assertEqual(exc.lineno, 2)
279
280 def test_eval_bytes_raw(self):
281 self.assertEqual(eval(""" br'x' """), b'x')
282 self.assertEqual(eval(""" rb'x' """), b'x')
283 self.assertEqual(eval(r""" br'\x01' """), b'\\' + b'x01')
284 self.assertEqual(eval(r""" rb'\x01' """), b'\\' + b'x01')
285 self.assertEqual(eval(""" br'\x01' """), byte(1))
286 self.assertEqual(eval(""" rb'\x01' """), byte(1))
287 self.assertEqual(eval(r""" br'\x81' """), b"\\" + b"x81")
288 self.assertEqual(eval(r""" rb'\x81' """), b"\\" + b"x81")
289 self.assertRaises(SyntaxError, eval, """ br'\x81' """)
290 self.assertRaises(SyntaxError, eval, """ rb'\x81' """)
291 self.assertEqual(eval(r""" br'\u1881' """), b"\\" + b"u1881")
292 self.assertEqual(eval(r""" rb'\u1881' """), b"\\" + b"u1881")
293 self.assertRaises(SyntaxError, eval, """ br'\u1881' """)
294 self.assertRaises(SyntaxError, eval, """ rb'\u1881' """)
295 self.assertEqual(eval(r""" br'\U0001d120' """), b"\\" + b"U0001d120")
296 self.assertEqual(eval(r""" rb'\U0001d120' """), b"\\" + b"U0001d120")
297 self.assertRaises(SyntaxError, eval, """ br'\U0001d120' """)
298 self.assertRaises(SyntaxError, eval, """ rb'\U0001d120' """)
299 self.assertRaises(SyntaxError, eval, """ bb'' """)
300 self.assertRaises(SyntaxError, eval, """ rr'' """)
301 self.assertRaises(SyntaxError, eval, """ brr'' """)
302 self.assertRaises(SyntaxError, eval, """ bbr'' """)
303 self.assertRaises(SyntaxError, eval, """ rrb'' """)
304 self.assertRaises(SyntaxError, eval, """ rbb'' """)
305
306 def test_eval_str_u(self):
307 self.assertEqual(eval(""" u'x' """), 'x')
308 self.assertEqual(eval(""" U'\u00e4' """), 'ä')
309 self.assertEqual(eval(""" u'\N{LATIN SMALL LETTER A WITH DIAERESIS}' """), 'ä')
310 self.assertRaises(SyntaxError, eval, """ ur'' """)
311 self.assertRaises(SyntaxError, eval, """ ru'' """)
312 self.assertRaises(SyntaxError, eval, """ bu'' """)
313 self.assertRaises(SyntaxError, eval, """ ub'' """)
314
315 def test_uppercase_prefixes(self):
316 self.assertEqual(eval(""" B'x' """), b'x')
317 self.assertEqual(eval(r""" R'\x01' """), r'\x01')
318 self.assertEqual(eval(r""" BR'\x01' """), br'\x01')
319 self.assertEqual(eval(""" F'{1+1}' """), f'{1+1}')
320 self.assertEqual(eval(r""" U'\U0001d120' """), u'\U0001d120')
321
322 def check_encoding(self, encoding, extra=""):
323 modname = "xx_" + encoding.replace("-", "_")
324 fn = os.path.join(self.tmpdir, modname + ".py")
325 f = open(fn, "w", encoding=encoding)
326 try:
327 f.write(TEMPLATE % encoding)
328 f.write(extra)
329 finally:
330 f.close()
331 __import__(modname)
332 del sys.modules[modname]
333
334 def test_file_utf_8(self):
335 extra = "z = '\u1234'; assert ord(z) == 0x1234\n"
336 self.check_encoding("utf-8", extra)
337
338 def test_file_utf_8_error(self):
339 extra = "b'\x80'\n"
340 self.assertRaises(SyntaxError, self.check_encoding, "utf-8", extra)
341
342 def test_file_utf8(self):
343 self.check_encoding("utf-8")
344
345 def test_file_iso_8859_1(self):
346 self.check_encoding("iso-8859-1")
347
348 def test_file_latin_1(self):
349 self.check_encoding("latin-1")
350
351 def test_file_latin9(self):
352 self.check_encoding("latin9")
353
354
355 if __name__ == "__main__":
356 unittest.main()