Debian Codegraph @

Lib/test/test_json/test_unicode.py

raw
1 import codecs
2 from collections import OrderedDict
3 from test.test_json import PyTest, CTest
4
5
6 class TestUnicode:
7 # test_encoding1 and test_encoding2 from 2.x are irrelevant (only str
8 # is supported as input, not bytes).
9
10 def test_encoding3(self):
11 u = '\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
12 j = self.dumps(u)
13 self.assertEqual(j, '"\\u03b1\\u03a9"')
14
15 def test_encoding4(self):
16 u = '\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
17 j = self.dumps([u])
18 self.assertEqual(j, '["\\u03b1\\u03a9"]')
19
20 def test_encoding5(self):
21 u = '\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
22 j = self.dumps(u, ensure_ascii=False)
23 self.assertEqual(j, f'"{u}"')
24
25 def test_encoding6(self):
26 u = '\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
27 j = self.dumps([u], ensure_ascii=False)
28 self.assertEqual(j, f'["{u}"]')
29
30 def test_encoding7(self):
31 u = '\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
32 j = self.dumps(u + "\n", ensure_ascii=False)
33 self.assertEqual(j, f'"{u}\\n"')
34
35 def test_ascii_non_printable_encode(self):
36 u = '\b\t\n\f\r\x00\x1f\x7f'
37 self.assertEqual(self.dumps(u),
38 '"\\b\\t\\n\\f\\r\\u0000\\u001f\\u007f"')
39 self.assertEqual(self.dumps(u, ensure_ascii=False),
40 '"\\b\\t\\n\\f\\r\\u0000\\u001f\x7f"')
41
42 def test_ascii_non_printable_decode(self):
43 self.assertEqual(self.loads('"\\b\\t\\n\\f\\r"'),
44 '\b\t\n\f\r')
45 s = ''.join(map(chr, range(32)))
46 for c in s:
47 self.assertRaises(self.JSONDecodeError, self.loads, f'"{c}"')
48 self.assertEqual(self.loads(f'"{s}"', strict=False), s)
49 self.assertEqual(self.loads('"\x7f"'), '\x7f')
50
51 def test_escaped_decode(self):
52 self.assertEqual(self.loads('"\\b\\t\\n\\f\\r"'), '\b\t\n\f\r')
53 self.assertEqual(self.loads('"\\"\\\\\\/"'), '"\\/')
54 for c in set(map(chr, range(0x100))) - set('"\\/bfnrt'):
55 self.assertRaises(self.JSONDecodeError, self.loads, f'"\\{c}"')
56 self.assertRaises(self.JSONDecodeError, self.loads, f'"\\{c}"', strict=False)
57
58 def test_big_unicode_encode(self):
59 u = '\U0001d120'
60 self.assertEqual(self.dumps(u), '"\\ud834\\udd20"')
61 self.assertEqual(self.dumps(u, ensure_ascii=False), '"\U0001d120"')
62
63 def test_big_unicode_decode(self):
64 u = 'z\U0001d120x'
65 self.assertEqual(self.loads(f'"{u}"'), u)
66 self.assertEqual(self.loads('"z\\ud834\\udd20x"'), u)
67
68 def test_unicode_decode(self):
69 for i in range(0, 0xd7ff):
70 u = chr(i)
71 s = f'"\\u{i:04x}"'
72 self.assertEqual(self.loads(s), u)
73
74 def test_single_surrogate_encode(self):
75 self.assertEqual(self.dumps('\uD83D'), '"\\ud83d"')
76 self.assertEqual(self.dumps('\uD83D', ensure_ascii=False), '"\ud83d"')
77 self.assertEqual(self.dumps('\uDC0D'), '"\\udc0d"')
78 self.assertEqual(self.dumps('\uDC0D', ensure_ascii=False), '"\udc0d"')
79
80 def test_single_surrogate_decode(self):
81 self.assertEqual(self.loads('"\uD83D"'), '\ud83d')
82 self.assertEqual(self.loads('"\\uD83D"'), '\ud83d')
83 self.assertEqual(self.loads('"\udc0d"'), '\udc0d')
84 self.assertEqual(self.loads('"\\udc0d"'), '\udc0d')
85
86 def test_unicode_preservation(self):
87 self.assertEqual(type(self.loads('""')), str)
88 self.assertEqual(type(self.loads('"a"')), str)
89 self.assertEqual(type(self.loads('["a"]')[0]), str)
90
91 def test_bytes_encode(self):
92 self.assertRaises(TypeError, self.dumps, b"hi")
93 self.assertRaises(TypeError, self.dumps, [b"hi"])
94
95 def test_bytes_decode(self):
96 for encoding, bom in [
97 ('utf-8', codecs.BOM_UTF8),
98 ('utf-16be', codecs.BOM_UTF16_BE),
99 ('utf-16le', codecs.BOM_UTF16_LE),
100 ('utf-32be', codecs.BOM_UTF32_BE),
101 ('utf-32le', codecs.BOM_UTF32_LE),
102 ]:
103 data = ["a\xb5\u20ac\U0001d120"]
104 encoded = self.dumps(data).encode(encoding)
105 self.assertEqual(self.loads(bom + encoded), data)
106 self.assertEqual(self.loads(encoded), data)
107 self.assertRaises(UnicodeDecodeError, self.loads, b'["\x80"]')
108 # RFC-7159 and ECMA-404 extend JSON to allow documents that
109 # consist of only a string, which can present a special case
110 # not covered by the encoding detection patterns specified in
111 # RFC-4627 for utf-16-le (XX 00 XX 00).
112 self.assertEqual(self.loads('"\u2600"'.encode('utf-16-le')),
113 '\u2600')
114 # Encoding detection for small (<4) bytes objects
115 # is implemented as a special case. RFC-7159 and ECMA-404
116 # allow single codepoint JSON documents which are only two
117 # bytes in utf-16 encodings w/o BOM.
118 self.assertEqual(self.loads(b'5\x00'), 5)
119 self.assertEqual(self.loads(b'\x007'), 7)
120 self.assertEqual(self.loads(b'57'), 57)
121
122 def test_object_pairs_hook_with_unicode(self):
123 s = '{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}'
124 p = [("xkd", 1), ("kcw", 2), ("art", 3), ("hxm", 4),
125 ("qrt", 5), ("pad", 6), ("hoy", 7)]
126 self.assertEqual(self.loads(s), eval(s))
127 self.assertEqual(self.loads(s, object_pairs_hook = lambda x: x), p)
128 od = self.loads(s, object_pairs_hook = OrderedDict)
129 self.assertEqual(od, OrderedDict(p))
130 self.assertEqual(type(od), OrderedDict)
131 # the object_pairs_hook takes priority over the object_hook
132 self.assertEqual(self.loads(s, object_pairs_hook = OrderedDict,
133 object_hook = lambda x: None),
134 OrderedDict(p))
135
136
137 class TestPyUnicode(TestUnicode, PyTest): pass
138 class TestCUnicode(TestUnicode, CTest): pass