| 1 |
r"""Test correct treatment of various string literals by the parser. |
| 2 |
|
| 3 |
There are four types of string literals: |
| 4 |
|
| 5 |
'abc' -- normal str |
| 6 |
r'abc' -- raw str |
| 7 |
b'xyz' -- normal bytes |
| 8 |
br'xyz' | rb'xyz' -- raw bytes |
| 9 |
|
| 10 |
The difference between normal and raw strings is of course that in a |
| 11 |
raw string, \ escapes (while still used to determine the end of the |
| 12 |
literal) are not interpreted, so that r'\x00' contains four |
| 13 |
characters: a backslash, an x, and two zeros; while '\x00' contains a |
| 14 |
single character (code point zero). |
| 15 |
|
| 16 |
The tricky thing is what should happen when non-ASCII bytes are used |
| 17 |
inside literals. For bytes literals, this is considered illegal. But |
| 18 |
for str literals, those bytes are supposed to be decoded using the |
| 19 |
encoding declared for the file (UTF-8 by default). |
| 20 |
|
| 21 |
We have to test this with various file encodings. We also test it with |
| 22 |
exec()/eval(), which uses a different code path. |
| 23 |
|
| 24 |
This file is really about correct treatment of encodings and |
| 25 |
backslashes. It doesn't concern itself with issues like single |
| 26 |
vs. double quotes or singly- vs. triply-quoted strings: that's dealt |
| 27 |
with elsewhere (I assume). |
| 28 |
""" |
| 29 |
|
| 30 |
import os |
| 31 |
import sys |
| 32 |
import shutil |
| 33 |
import tempfile |
| 34 |
import unittest |
| 35 |
import warnings |
| 36 |
|
| 37 |
|
| 38 |
TEMPLATE = r"""# coding: %s |
| 39 |
a = 'x' |
| 40 |
assert ord(a) == 120 |
| 41 |
b = '\x01' |
| 42 |
assert ord(b) == 1 |
| 43 |
c = r'\x01' |
| 44 |
assert list(map(ord, c)) == [92, 120, 48, 49] |
| 45 |
d = '\x81' |
| 46 |
assert ord(d) == 0x81 |
| 47 |
e = r'\x81' |
| 48 |
assert list(map(ord, e)) == [92, 120, 56, 49] |
| 49 |
f = '\u1881' |
| 50 |
assert ord(f) == 0x1881 |
| 51 |
g = r'\u1881' |
| 52 |
assert list(map(ord, g)) == [92, 117, 49, 56, 56, 49] |
| 53 |
h = '\U0001d120' |
| 54 |
assert ord(h) == 0x1d120 |
| 55 |
i = r'\U0001d120' |
| 56 |
assert list(map(ord, i)) == [92, 85, 48, 48, 48, 49, 100, 49, 50, 48] |
| 57 |
""" |
| 58 |
|
| 59 |
|
| 60 |
def byte(i): |
| 61 |
return bytes([i]) |
| 62 |
|
| 63 |
|
| 64 |
class TestLiterals(unittest.TestCase): |
| 65 |
|
| 66 |
def setUp(self): |
| 67 |
self.save_path = sys.path[:] |
| 68 |
self.tmpdir = tempfile.mkdtemp() |
| 69 |
sys.path.insert(0, self.tmpdir) |
| 70 |
|
| 71 |
def tearDown(self): |
| 72 |
sys.path[:] = self.save_path |
| 73 |
shutil.rmtree(self.tmpdir, ignore_errors=True) |
| 74 |
|
| 75 |
def test_template(self): |
| 76 |
# Check that the template doesn't contain any non-printables |
| 77 |
# except for \n. |
| 78 |
for c in TEMPLATE: |
| 79 |
assert c == '\n' or ' ' <= c <= '~', repr(c) |
| 80 |
|
| 81 |
def test_eval_str_normal(self): |
| 82 |
self.assertEqual(eval(""" 'x' """), 'x') |
| 83 |
self.assertEqual(eval(r""" '\x01' """), chr(1)) |
| 84 |
self.assertEqual(eval(""" '\x01' """), chr(1)) |
| 85 |
self.assertEqual(eval(r""" '\x81' """), chr(0x81)) |
| 86 |
self.assertEqual(eval(""" '\x81' """), chr(0x81)) |
| 87 |
self.assertEqual(eval(r""" '\u1881' """), chr(0x1881)) |
| 88 |
self.assertEqual(eval(""" '\u1881' """), chr(0x1881)) |
| 89 |
self.assertEqual(eval(r""" '\U0001d120' """), chr(0x1d120)) |
| 90 |
self.assertEqual(eval(""" '\U0001d120' """), chr(0x1d120)) |
| 91 |
|
| 92 |
def test_eval_str_incomplete(self): |
| 93 |
self.assertRaises(SyntaxError, eval, r""" '\x' """) |
| 94 |
self.assertRaises(SyntaxError, eval, r""" '\x0' """) |
| 95 |
self.assertRaises(SyntaxError, eval, r""" '\u' """) |
| 96 |
self.assertRaises(SyntaxError, eval, r""" '\u0' """) |
| 97 |
self.assertRaises(SyntaxError, eval, r""" '\u00' """) |
| 98 |
self.assertRaises(SyntaxError, eval, r""" '\u000' """) |
| 99 |
self.assertRaises(SyntaxError, eval, r""" '\U' """) |
| 100 |
self.assertRaises(SyntaxError, eval, r""" '\U0' """) |
| 101 |
self.assertRaises(SyntaxError, eval, r""" '\U00' """) |
| 102 |
self.assertRaises(SyntaxError, eval, r""" '\U000' """) |
| 103 |
self.assertRaises(SyntaxError, eval, r""" '\U0000' """) |
| 104 |
self.assertRaises(SyntaxError, eval, r""" '\U00000' """) |
| 105 |
self.assertRaises(SyntaxError, eval, r""" '\U000000' """) |
| 106 |
self.assertRaises(SyntaxError, eval, r""" '\U0000000' """) |
| 107 |
|
| 108 |
def test_eval_str_invalid_escape(self): |
| 109 |
for b in range(1, 128): |
| 110 |
if b in b"""\n\r"'01234567NU\\abfnrtuvx""": |
| 111 |
continue |
| 112 |
with self.assertWarns(SyntaxWarning): |
| 113 |
self.assertEqual(eval(r"'\%c'" % b), '\\' + chr(b)) |
| 114 |
|
| 115 |
with warnings.catch_warnings(record=True) as w: |
| 116 |
warnings.simplefilter('always', category=SyntaxWarning) |
| 117 |
eval("'''\n\\z'''") |
| 118 |
self.assertEqual(len(w), 1) |
| 119 |
self.assertEqual(str(w[0].message), r'"\z" is an invalid escape sequence. ' |
| 120 |
r'Such sequences will not work in the future. ' |
| 121 |
r'Did you mean "\\z"? A raw string is also an option.') |
| 122 |
self.assertEqual(w[0].filename, '<string>') |
| 123 |
self.assertEqual(w[0].lineno, 2) |
| 124 |
|
| 125 |
with warnings.catch_warnings(record=True) as w: |
| 126 |
warnings.simplefilter('error', category=SyntaxWarning) |
| 127 |
with self.assertRaises(SyntaxError) as cm: |
| 128 |
eval("'''\n\\z'''") |
| 129 |
exc = cm.exception |
| 130 |
self.assertEqual(w, []) |
| 131 |
self.assertEqual(exc.msg, r'"\z" is an invalid escape sequence. ' |
| 132 |
r'Did you mean "\\z"? A raw string is also an option.') |
| 133 |
self.assertEqual(exc.filename, '<string>') |
| 134 |
self.assertEqual(exc.lineno, 2) |
| 135 |
self.assertEqual(exc.offset, 1) |
| 136 |
|
| 137 |
# Check that the warning is raised only once if there are syntax errors |
| 138 |
|
| 139 |
with warnings.catch_warnings(record=True) as w: |
| 140 |
warnings.simplefilter('always', category=SyntaxWarning) |
| 141 |
with self.assertRaises(SyntaxError) as cm: |
| 142 |
eval("'\\e' $") |
| 143 |
exc = cm.exception |
| 144 |
self.assertEqual(len(w), 1) |
| 145 |
self.assertEqual(w[0].category, SyntaxWarning) |
| 146 |
self.assertRegex(str(w[0].message), 'invalid escape sequence') |
| 147 |
self.assertEqual(w[0].filename, '<string>') |
| 148 |
|
| 149 |
def test_eval_str_invalid_octal_escape(self): |
| 150 |
for i in range(0o400, 0o1000): |
| 151 |
with self.assertWarns(SyntaxWarning): |
| 152 |
self.assertEqual(eval(r"'\%o'" % i), chr(i)) |
| 153 |
|
| 154 |
with warnings.catch_warnings(record=True) as w: |
| 155 |
warnings.simplefilter('always', category=SyntaxWarning) |
| 156 |
eval("'''\n\\407'''") |
| 157 |
self.assertEqual(len(w), 1) |
| 158 |
self.assertEqual(str(w[0].message), |
| 159 |
r'"\407" is an invalid octal escape sequence. ' |
| 160 |
r'Such sequences will not work in the future. ' |
| 161 |
r'Did you mean "\\407"? A raw string is also an option.') |
| 162 |
self.assertEqual(w[0].filename, '<string>') |
| 163 |
self.assertEqual(w[0].lineno, 2) |
| 164 |
|
| 165 |
with warnings.catch_warnings(record=True) as w: |
| 166 |
warnings.simplefilter('error', category=SyntaxWarning) |
| 167 |
with self.assertRaises(SyntaxError) as cm: |
| 168 |
eval("'''\n\\407'''") |
| 169 |
exc = cm.exception |
| 170 |
self.assertEqual(w, []) |
| 171 |
self.assertEqual(exc.msg, r'"\407" is an invalid octal escape sequence. ' |
| 172 |
r'Did you mean "\\407"? A raw string is also an option.') |
| 173 |
self.assertEqual(exc.filename, '<string>') |
| 174 |
self.assertEqual(exc.lineno, 2) |
| 175 |
self.assertEqual(exc.offset, 1) |
| 176 |
|
| 177 |
def test_invalid_escape_locations_with_offset(self): |
| 178 |
with warnings.catch_warnings(record=True) as w: |
| 179 |
warnings.simplefilter('always', category=SyntaxWarning) |
| 180 |
eval("\"'''''''''''''''''''''invalid\\ Escape\"") |
| 181 |
self.assertEqual(len(w), 1) |
| 182 |
self.assertEqual(str(w[0].message), |
| 183 |
r'"\ " is an invalid escape sequence. Such sequences ' |
| 184 |
r'will not work in the future. Did you mean "\\ "? ' |
| 185 |
r'A raw string is also an option.') |
| 186 |
self.assertEqual(w[0].filename, '<string>') |
| 187 |
self.assertEqual(w[0].lineno, 1) |
| 188 |
|
| 189 |
with warnings.catch_warnings(record=True) as w: |
| 190 |
warnings.simplefilter('always', category=SyntaxWarning) |
| 191 |
eval("\"''Incorrect \\ logic?\"") |
| 192 |
self.assertEqual(len(w), 1) |
| 193 |
self.assertEqual(str(w[0].message), |
| 194 |
r'"\ " is an invalid escape sequence. Such sequences ' |
| 195 |
r'will not work in the future. Did you mean "\\ "? ' |
| 196 |
r'A raw string is also an option.') |
| 197 |
self.assertEqual(w[0].filename, '<string>') |
| 198 |
self.assertEqual(w[0].lineno, 1) |
| 199 |
|
| 200 |
def test_eval_str_raw(self): |
| 201 |
self.assertEqual(eval(""" r'x' """), 'x') |
| 202 |
self.assertEqual(eval(r""" r'\x01' """), '\\' + 'x01') |
| 203 |
self.assertEqual(eval(""" r'\x01' """), chr(1)) |
| 204 |
self.assertEqual(eval(r""" r'\x81' """), '\\' + 'x81') |
| 205 |
self.assertEqual(eval(""" r'\x81' """), chr(0x81)) |
| 206 |
self.assertEqual(eval(r""" r'\u1881' """), '\\' + 'u1881') |
| 207 |
self.assertEqual(eval(""" r'\u1881' """), chr(0x1881)) |
| 208 |
self.assertEqual(eval(r""" r'\U0001d120' """), '\\' + 'U0001d120') |
| 209 |
self.assertEqual(eval(""" r'\U0001d120' """), chr(0x1d120)) |
| 210 |
|
| 211 |
def test_eval_bytes_normal(self): |
| 212 |
self.assertEqual(eval(""" b'x' """), b'x') |
| 213 |
self.assertEqual(eval(r""" b'\x01' """), byte(1)) |
| 214 |
self.assertEqual(eval(""" b'\x01' """), byte(1)) |
| 215 |
self.assertEqual(eval(r""" b'\x81' """), byte(0x81)) |
| 216 |
self.assertRaises(SyntaxError, eval, """ b'\x81' """) |
| 217 |
self.assertEqual(eval(r""" br'\u1881' """), b'\\' + b'u1881') |
| 218 |
self.assertRaises(SyntaxError, eval, """ b'\u1881' """) |
| 219 |
self.assertEqual(eval(r""" br'\U0001d120' """), b'\\' + b'U0001d120') |
| 220 |
self.assertRaises(SyntaxError, eval, """ b'\U0001d120' """) |
| 221 |
|
| 222 |
def test_eval_bytes_incomplete(self): |
| 223 |
self.assertRaises(SyntaxError, eval, r""" b'\x' """) |
| 224 |
self.assertRaises(SyntaxError, eval, r""" b'\x0' """) |
| 225 |
|
| 226 |
def test_eval_bytes_invalid_escape(self): |
| 227 |
for b in range(1, 128): |
| 228 |
if b in b"""\n\r"'01234567\\abfnrtvx""": |
| 229 |
continue |
| 230 |
with self.assertWarns(SyntaxWarning): |
| 231 |
self.assertEqual(eval(r"b'\%c'" % b), b'\\' + bytes([b])) |
| 232 |
|
| 233 |
with warnings.catch_warnings(record=True) as w: |
| 234 |
warnings.simplefilter('always', category=SyntaxWarning) |
| 235 |
eval("b'''\n\\z'''") |
| 236 |
self.assertEqual(len(w), 1) |
| 237 |
self.assertEqual(str(w[0].message), r'"\z" is an invalid escape sequence. ' |
| 238 |
r'Such sequences will not work in the future. ' |
| 239 |
r'Did you mean "\\z"? A raw string is also an option.') |
| 240 |
self.assertEqual(w[0].filename, '<string>') |
| 241 |
self.assertEqual(w[0].lineno, 2) |
| 242 |
|
| 243 |
with warnings.catch_warnings(record=True) as w: |
| 244 |
warnings.simplefilter('error', category=SyntaxWarning) |
| 245 |
with self.assertRaises(SyntaxError) as cm: |
| 246 |
eval("b'''\n\\z'''") |
| 247 |
exc = cm.exception |
| 248 |
self.assertEqual(w, []) |
| 249 |
self.assertEqual(exc.msg, r'"\z" is an invalid escape sequence. ' |
| 250 |
r'Did you mean "\\z"? A raw string is also an option.') |
| 251 |
self.assertEqual(exc.filename, '<string>') |
| 252 |
self.assertEqual(exc.lineno, 2) |
| 253 |
|
| 254 |
def test_eval_bytes_invalid_octal_escape(self): |
| 255 |
for i in range(0o400, 0o1000): |
| 256 |
with self.assertWarns(SyntaxWarning): |
| 257 |
self.assertEqual(eval(r"b'\%o'" % i), bytes([i & 0o377])) |
| 258 |
|
| 259 |
with warnings.catch_warnings(record=True) as w: |
| 260 |
warnings.simplefilter('always', category=SyntaxWarning) |
| 261 |
eval("b'''\n\\407'''") |
| 262 |
self.assertEqual(len(w), 1) |
| 263 |
self.assertEqual(str(w[0].message), r'"\407" is an invalid octal escape sequence. ' |
| 264 |
r'Such sequences will not work in the future. ' |
| 265 |
r'Did you mean "\\407"? A raw string is also an option.') |
| 266 |
self.assertEqual(w[0].filename, '<string>') |
| 267 |
self.assertEqual(w[0].lineno, 2) |
| 268 |
|
| 269 |
with warnings.catch_warnings(record=True) as w: |
| 270 |
warnings.simplefilter('error', category=SyntaxWarning) |
| 271 |
with self.assertRaises(SyntaxError) as cm: |
| 272 |
eval("b'''\n\\407'''") |
| 273 |
exc = cm.exception |
| 274 |
self.assertEqual(w, []) |
| 275 |
self.assertEqual(exc.msg, r'"\407" is an invalid octal escape sequence. ' |
| 276 |
r'Did you mean "\\407"? A raw string is also an option.') |
| 277 |
self.assertEqual(exc.filename, '<string>') |
| 278 |
self.assertEqual(exc.lineno, 2) |
| 279 |
|
| 280 |
def test_eval_bytes_raw(self): |
| 281 |
self.assertEqual(eval(""" br'x' """), b'x') |
| 282 |
self.assertEqual(eval(""" rb'x' """), b'x') |
| 283 |
self.assertEqual(eval(r""" br'\x01' """), b'\\' + b'x01') |
| 284 |
self.assertEqual(eval(r""" rb'\x01' """), b'\\' + b'x01') |
| 285 |
self.assertEqual(eval(""" br'\x01' """), byte(1)) |
| 286 |
self.assertEqual(eval(""" rb'\x01' """), byte(1)) |
| 287 |
self.assertEqual(eval(r""" br'\x81' """), b"\\" + b"x81") |
| 288 |
self.assertEqual(eval(r""" rb'\x81' """), b"\\" + b"x81") |
| 289 |
self.assertRaises(SyntaxError, eval, """ br'\x81' """) |
| 290 |
self.assertRaises(SyntaxError, eval, """ rb'\x81' """) |
| 291 |
self.assertEqual(eval(r""" br'\u1881' """), b"\\" + b"u1881") |
| 292 |
self.assertEqual(eval(r""" rb'\u1881' """), b"\\" + b"u1881") |
| 293 |
self.assertRaises(SyntaxError, eval, """ br'\u1881' """) |
| 294 |
self.assertRaises(SyntaxError, eval, """ rb'\u1881' """) |
| 295 |
self.assertEqual(eval(r""" br'\U0001d120' """), b"\\" + b"U0001d120") |
| 296 |
self.assertEqual(eval(r""" rb'\U0001d120' """), b"\\" + b"U0001d120") |
| 297 |
self.assertRaises(SyntaxError, eval, """ br'\U0001d120' """) |
| 298 |
self.assertRaises(SyntaxError, eval, """ rb'\U0001d120' """) |
| 299 |
self.assertRaises(SyntaxError, eval, """ bb'' """) |
| 300 |
self.assertRaises(SyntaxError, eval, """ rr'' """) |
| 301 |
self.assertRaises(SyntaxError, eval, """ brr'' """) |
| 302 |
self.assertRaises(SyntaxError, eval, """ bbr'' """) |
| 303 |
self.assertRaises(SyntaxError, eval, """ rrb'' """) |
| 304 |
self.assertRaises(SyntaxError, eval, """ rbb'' """) |
| 305 |
|
| 306 |
def test_eval_str_u(self): |
| 307 |
self.assertEqual(eval(""" u'x' """), 'x') |
| 308 |
self.assertEqual(eval(""" U'\u00e4' """), 'ä') |
| 309 |
self.assertEqual(eval(""" u'\N{LATIN SMALL LETTER A WITH DIAERESIS}' """), 'ä') |
| 310 |
self.assertRaises(SyntaxError, eval, """ ur'' """) |
| 311 |
self.assertRaises(SyntaxError, eval, """ ru'' """) |
| 312 |
self.assertRaises(SyntaxError, eval, """ bu'' """) |
| 313 |
self.assertRaises(SyntaxError, eval, """ ub'' """) |
| 314 |
|
| 315 |
def test_uppercase_prefixes(self): |
| 316 |
self.assertEqual(eval(""" B'x' """), b'x') |
| 317 |
self.assertEqual(eval(r""" R'\x01' """), r'\x01') |
| 318 |
self.assertEqual(eval(r""" BR'\x01' """), br'\x01') |
| 319 |
self.assertEqual(eval(""" F'{1+1}' """), f'{1+1}') |
| 320 |
self.assertEqual(eval(r""" U'\U0001d120' """), u'\U0001d120') |
| 321 |
|
| 322 |
def check_encoding(self, encoding, extra=""): |
| 323 |
modname = "xx_" + encoding.replace("-", "_") |
| 324 |
fn = os.path.join(self.tmpdir, modname + ".py") |
| 325 |
f = open(fn, "w", encoding=encoding) |
| 326 |
try: |
| 327 |
f.write(TEMPLATE % encoding) |
| 328 |
f.write(extra) |
| 329 |
finally: |
| 330 |
f.close() |
| 331 |
__import__(modname) |
| 332 |
del sys.modules[modname] |
| 333 |
|
| 334 |
def test_file_utf_8(self): |
| 335 |
extra = "z = '\u1234'; assert ord(z) == 0x1234\n" |
| 336 |
self.check_encoding("utf-8", extra) |
| 337 |
|
| 338 |
def test_file_utf_8_error(self): |
| 339 |
extra = "b'\x80'\n" |
| 340 |
self.assertRaises(SyntaxError, self.check_encoding, "utf-8", extra) |
| 341 |
|
| 342 |
def test_file_utf8(self): |
| 343 |
self.check_encoding("utf-8") |
| 344 |
|
| 345 |
def test_file_iso_8859_1(self): |
| 346 |
self.check_encoding("iso-8859-1") |
| 347 |
|
| 348 |
def test_file_latin_1(self): |
| 349 |
self.check_encoding("latin-1") |
| 350 |
|
| 351 |
def test_file_latin9(self): |
| 352 |
self.check_encoding("latin9") |
| 353 |
|
| 354 |
|
| 355 |
if __name__ == "__main__": |
| 356 |
unittest.main() |