| 1 |
import codecs |
| 2 |
from collections import OrderedDict |
| 3 |
from test.test_json import PyTest, CTest |
| 4 |
|
| 5 |
|
| 6 |
class TestUnicode: |
| 7 |
# test_encoding1 and test_encoding2 from 2.x are irrelevant (only str |
| 8 |
# is supported as input, not bytes). |
| 9 |
|
| 10 |
def test_encoding3(self): |
| 11 |
u = '\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' |
| 12 |
j = self.dumps(u) |
| 13 |
self.assertEqual(j, '"\\u03b1\\u03a9"') |
| 14 |
|
| 15 |
def test_encoding4(self): |
| 16 |
u = '\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' |
| 17 |
j = self.dumps([u]) |
| 18 |
self.assertEqual(j, '["\\u03b1\\u03a9"]') |
| 19 |
|
| 20 |
def test_encoding5(self): |
| 21 |
u = '\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' |
| 22 |
j = self.dumps(u, ensure_ascii=False) |
| 23 |
self.assertEqual(j, f'"{u}"') |
| 24 |
|
| 25 |
def test_encoding6(self): |
| 26 |
u = '\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' |
| 27 |
j = self.dumps([u], ensure_ascii=False) |
| 28 |
self.assertEqual(j, f'["{u}"]') |
| 29 |
|
| 30 |
def test_encoding7(self): |
| 31 |
u = '\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' |
| 32 |
j = self.dumps(u + "\n", ensure_ascii=False) |
| 33 |
self.assertEqual(j, f'"{u}\\n"') |
| 34 |
|
| 35 |
def test_ascii_non_printable_encode(self): |
| 36 |
u = '\b\t\n\f\r\x00\x1f\x7f' |
| 37 |
self.assertEqual(self.dumps(u), |
| 38 |
'"\\b\\t\\n\\f\\r\\u0000\\u001f\\u007f"') |
| 39 |
self.assertEqual(self.dumps(u, ensure_ascii=False), |
| 40 |
'"\\b\\t\\n\\f\\r\\u0000\\u001f\x7f"') |
| 41 |
|
| 42 |
def test_ascii_non_printable_decode(self): |
| 43 |
self.assertEqual(self.loads('"\\b\\t\\n\\f\\r"'), |
| 44 |
'\b\t\n\f\r') |
| 45 |
s = ''.join(map(chr, range(32))) |
| 46 |
for c in s: |
| 47 |
self.assertRaises(self.JSONDecodeError, self.loads, f'"{c}"') |
| 48 |
self.assertEqual(self.loads(f'"{s}"', strict=False), s) |
| 49 |
self.assertEqual(self.loads('"\x7f"'), '\x7f') |
| 50 |
|
| 51 |
def test_escaped_decode(self): |
| 52 |
self.assertEqual(self.loads('"\\b\\t\\n\\f\\r"'), '\b\t\n\f\r') |
| 53 |
self.assertEqual(self.loads('"\\"\\\\\\/"'), '"\\/') |
| 54 |
for c in set(map(chr, range(0x100))) - set('"\\/bfnrt'): |
| 55 |
self.assertRaises(self.JSONDecodeError, self.loads, f'"\\{c}"') |
| 56 |
self.assertRaises(self.JSONDecodeError, self.loads, f'"\\{c}"', strict=False) |
| 57 |
|
| 58 |
def test_big_unicode_encode(self): |
| 59 |
u = '\U0001d120' |
| 60 |
self.assertEqual(self.dumps(u), '"\\ud834\\udd20"') |
| 61 |
self.assertEqual(self.dumps(u, ensure_ascii=False), '"\U0001d120"') |
| 62 |
|
| 63 |
def test_big_unicode_decode(self): |
| 64 |
u = 'z\U0001d120x' |
| 65 |
self.assertEqual(self.loads(f'"{u}"'), u) |
| 66 |
self.assertEqual(self.loads('"z\\ud834\\udd20x"'), u) |
| 67 |
|
| 68 |
def test_unicode_decode(self): |
| 69 |
for i in range(0, 0xd7ff): |
| 70 |
u = chr(i) |
| 71 |
s = f'"\\u{i:04x}"' |
| 72 |
self.assertEqual(self.loads(s), u) |
| 73 |
|
| 74 |
def test_single_surrogate_encode(self): |
| 75 |
self.assertEqual(self.dumps('\uD83D'), '"\\ud83d"') |
| 76 |
self.assertEqual(self.dumps('\uD83D', ensure_ascii=False), '"\ud83d"') |
| 77 |
self.assertEqual(self.dumps('\uDC0D'), '"\\udc0d"') |
| 78 |
self.assertEqual(self.dumps('\uDC0D', ensure_ascii=False), '"\udc0d"') |
| 79 |
|
| 80 |
def test_single_surrogate_decode(self): |
| 81 |
self.assertEqual(self.loads('"\uD83D"'), '\ud83d') |
| 82 |
self.assertEqual(self.loads('"\\uD83D"'), '\ud83d') |
| 83 |
self.assertEqual(self.loads('"\udc0d"'), '\udc0d') |
| 84 |
self.assertEqual(self.loads('"\\udc0d"'), '\udc0d') |
| 85 |
|
| 86 |
def test_unicode_preservation(self): |
| 87 |
self.assertEqual(type(self.loads('""')), str) |
| 88 |
self.assertEqual(type(self.loads('"a"')), str) |
| 89 |
self.assertEqual(type(self.loads('["a"]')[0]), str) |
| 90 |
|
| 91 |
def test_bytes_encode(self): |
| 92 |
self.assertRaises(TypeError, self.dumps, b"hi") |
| 93 |
self.assertRaises(TypeError, self.dumps, [b"hi"]) |
| 94 |
|
| 95 |
def test_bytes_decode(self): |
| 96 |
for encoding, bom in [ |
| 97 |
('utf-8', codecs.BOM_UTF8), |
| 98 |
('utf-16be', codecs.BOM_UTF16_BE), |
| 99 |
('utf-16le', codecs.BOM_UTF16_LE), |
| 100 |
('utf-32be', codecs.BOM_UTF32_BE), |
| 101 |
('utf-32le', codecs.BOM_UTF32_LE), |
| 102 |
]: |
| 103 |
data = ["a\xb5\u20ac\U0001d120"] |
| 104 |
encoded = self.dumps(data).encode(encoding) |
| 105 |
self.assertEqual(self.loads(bom + encoded), data) |
| 106 |
self.assertEqual(self.loads(encoded), data) |
| 107 |
self.assertRaises(UnicodeDecodeError, self.loads, b'["\x80"]') |
| 108 |
# RFC-7159 and ECMA-404 extend JSON to allow documents that |
| 109 |
# consist of only a string, which can present a special case |
| 110 |
# not covered by the encoding detection patterns specified in |
| 111 |
# RFC-4627 for utf-16-le (XX 00 XX 00). |
| 112 |
self.assertEqual(self.loads('"\u2600"'.encode('utf-16-le')), |
| 113 |
'\u2600') |
| 114 |
# Encoding detection for small (<4) bytes objects |
| 115 |
# is implemented as a special case. RFC-7159 and ECMA-404 |
| 116 |
# allow single codepoint JSON documents which are only two |
| 117 |
# bytes in utf-16 encodings w/o BOM. |
| 118 |
self.assertEqual(self.loads(b'5\x00'), 5) |
| 119 |
self.assertEqual(self.loads(b'\x007'), 7) |
| 120 |
self.assertEqual(self.loads(b'57'), 57) |
| 121 |
|
| 122 |
def test_object_pairs_hook_with_unicode(self): |
| 123 |
s = '{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}' |
| 124 |
p = [("xkd", 1), ("kcw", 2), ("art", 3), ("hxm", 4), |
| 125 |
("qrt", 5), ("pad", 6), ("hoy", 7)] |
| 126 |
self.assertEqual(self.loads(s), eval(s)) |
| 127 |
self.assertEqual(self.loads(s, object_pairs_hook = lambda x: x), p) |
| 128 |
od = self.loads(s, object_pairs_hook = OrderedDict) |
| 129 |
self.assertEqual(od, OrderedDict(p)) |
| 130 |
self.assertEqual(type(od), OrderedDict) |
| 131 |
# the object_pairs_hook takes priority over the object_hook |
| 132 |
self.assertEqual(self.loads(s, object_pairs_hook = OrderedDict, |
| 133 |
object_hook = lambda x: None), |
| 134 |
OrderedDict(p)) |
| 135 |
|
| 136 |
|
| 137 |
class TestPyUnicode(TestUnicode, PyTest): pass |
| 138 |
class TestCUnicode(TestUnicode, CTest): pass |