Debian Codegraph @

tests/utils_tests/test_encoding.py

raw
1 import datetime
2 import inspect
3 import sys
4 import unittest
5 from pathlib import Path
6 from unittest import mock
7 from urllib.parse import quote, quote_plus
8
9 from django.test import SimpleTestCase
10 from django.utils.encoding import (
11 DjangoUnicodeDecodeError,
12 escape_uri_path,
13 filepath_to_uri,
14 force_bytes,
15 force_str,
16 get_system_encoding,
17 iri_to_uri,
18 repercent_broken_unicode,
19 smart_bytes,
20 smart_str,
21 uri_to_iri,
22 )
23 from django.utils.functional import SimpleLazyObject
24 from django.utils.translation import gettext_lazy
25 from django.utils.version import PYPY
26
27
28 class TestEncodingUtils(SimpleTestCase):
29 def test_force_str_exception(self):
30 """
31 Broken __str__ actually raises an error.
32 """
33
34 class MyString:
35 def __str__(self):
36 return b"\xc3\xb6\xc3\xa4\xc3\xbc"
37
38 # str(s) raises a TypeError if the result is not a text type.
39 with self.assertRaises(TypeError):
40 force_str(MyString())
41
42 def test_force_str_lazy(self):
43 s = SimpleLazyObject(lambda: "x")
44 self.assertIs(type(force_str(s)), str)
45
46 def test_force_str_DjangoUnicodeDecodeError(self):
47 reason = "unexpected end of data" if PYPY else "invalid start byte"
48 msg = (
49 f"'utf-8' codec can't decode byte 0xff in position 0: {reason}. "
50 "You passed in b'\\xff' (<class 'bytes'>)"
51 )
52 with self.assertRaisesMessage(DjangoUnicodeDecodeError, msg):
53 force_str(b"\xff")
54
55 def test_force_bytes_exception(self):
56 """
57 force_bytes knows how to convert to bytes an exception
58 containing non-ASCII characters in its args.
59 """
60 error_msg = "This is an exception, voilà"
61 exc = ValueError(error_msg)
62 self.assertEqual(force_bytes(exc), error_msg.encode())
63 self.assertEqual(
64 force_bytes(exc, encoding="ascii", errors="ignore"),
65 b"This is an exception, voil",
66 )
67
68 def test_force_bytes_strings_only(self):
69 today = datetime.date.today()
70 self.assertEqual(force_bytes(today, strings_only=True), today)
71
72 def test_force_bytes_encoding(self):
73 error_msg = "This is an exception, voilà".encode()
74 result = force_bytes(error_msg, encoding="ascii", errors="ignore")
75 self.assertEqual(result, b"This is an exception, voil")
76
77 def test_force_bytes_memory_view(self):
78 data = b"abc"
79 result = force_bytes(memoryview(data))
80 # Type check is needed because memoryview(bytes) == bytes.
81 self.assertIs(type(result), bytes)
82 self.assertEqual(result, data)
83
84 def test_smart_bytes(self):
85 class Test:
86 def __str__(self):
87 return "ŠĐĆŽćžšđ"
88
89 lazy_func = gettext_lazy("x")
90 self.assertIs(smart_bytes(lazy_func), lazy_func)
91 self.assertEqual(
92 smart_bytes(Test()),
93 b"\xc5\xa0\xc4\x90\xc4\x86\xc5\xbd\xc4\x87\xc5\xbe\xc5\xa1\xc4\x91",
94 )
95 self.assertEqual(smart_bytes(1), b"1")
96 self.assertEqual(smart_bytes("foo"), b"foo")
97
98 def test_smart_str(self):
99 class Test:
100 def __str__(self):
101 return "ŠĐĆŽćžšđ"
102
103 lazy_func = gettext_lazy("x")
104 self.assertIs(smart_str(lazy_func), lazy_func)
105 self.assertEqual(
106 smart_str(Test()), "\u0160\u0110\u0106\u017d\u0107\u017e\u0161\u0111"
107 )
108 self.assertEqual(smart_str(1), "1")
109 self.assertEqual(smart_str("foo"), "foo")
110
111 def test_get_default_encoding(self):
112 with mock.patch("locale.getlocale", side_effect=Exception):
113 self.assertEqual(get_system_encoding(), "ascii")
114
115 def test_repercent_broken_unicode_recursion_error(self):
116 # Prepare a string long enough to force a recursion error if the tested
117 # function uses recursion.
118 data = b"\xfc" * sys.getrecursionlimit()
119 try:
120 self.assertEqual(
121 repercent_broken_unicode(data), b"%FC" * sys.getrecursionlimit()
122 )
123 except RecursionError:
124 self.fail("Unexpected RecursionError raised.")
125
126 def test_repercent_broken_unicode_small_fragments(self):
127 data = b"test\xfctest\xfctest\xfc"
128 decoded_paths = []
129
130 def mock_quote(*args, **kwargs):
131 # The second frame is the call to repercent_broken_unicode().
132 decoded_paths.append(inspect.currentframe().f_back.f_locals["path"])
133 return quote(*args, **kwargs)
134
135 with mock.patch("django.utils.encoding.quote", mock_quote):
136 self.assertEqual(repercent_broken_unicode(data), b"test%FCtest%FCtest%FC")
137
138 # decode() is called on smaller fragment of the path each time.
139 self.assertEqual(
140 decoded_paths,
141 [b"test\xfctest\xfctest\xfc", b"test\xfctest\xfc", b"test\xfc"],
142 )
143
144
145 class TestRFC3987IEncodingUtils(unittest.TestCase):
146 def test_filepath_to_uri(self):
147 self.assertIsNone(filepath_to_uri(None))
148 self.assertEqual(
149 filepath_to_uri("upload\\чубака.mp4"),
150 "upload/%D1%87%D1%83%D0%B1%D0%B0%D0%BA%D0%B0.mp4",
151 )
152 self.assertEqual(filepath_to_uri(Path("upload/test.png")), "upload/test.png")
153 self.assertEqual(filepath_to_uri(Path("upload\\test.png")), "upload/test.png")
154
155 def test_iri_to_uri(self):
156 cases = [
157 # Valid UTF-8 sequences are encoded.
158 ("red%09rosé#red", "red%09ros%C3%A9#red"),
159 ("/blog/for/Jürgen Münster/", "/blog/for/J%C3%BCrgen%20M%C3%BCnster/"),
160 (
161 "locations/%s" % quote_plus("Paris & Orléans"),
162 "locations/Paris+%26+Orl%C3%A9ans",
163 ),
164 # Reserved chars remain unescaped.
165 ("%&", "%&"),
166 ("red&♥ros%#red", "red&%E2%99%A5ros%#red"),
167 (gettext_lazy("red&♥ros%#red"), "red&%E2%99%A5ros%#red"),
168 ]
169
170 for iri, uri in cases:
171 with self.subTest(iri):
172 self.assertEqual(iri_to_uri(iri), uri)
173
174 # Test idempotency.
175 self.assertEqual(iri_to_uri(iri_to_uri(iri)), uri)
176
177 def test_uri_to_iri(self):
178 cases = [
179 (None, None),
180 # Valid UTF-8 sequences are decoded.
181 ("/%e2%89%Ab%E2%99%a5%E2%89%aB/", "/≫♥≫/"),
182 ("/%E2%99%A5%E2%99%A5/?utf8=%E2%9C%93", "/♥♥/?utf8=✓"),
183 ("/%41%5a%6B/", "/AZk/"),
184 # Reserved and non-URL valid ASCII chars are not decoded.
185 ("/%25%20%02%41%7b/", "/%25%20%02A%7b/"),
186 # Broken UTF-8 sequences remain escaped.
187 ("/%AAd%AAj%AAa%AAn%AAg%AAo%AA/", "/%AAd%AAj%AAa%AAn%AAg%AAo%AA/"),
188 ("/%E2%99%A5%E2%E2%99%A5/", "/♥%E2♥/"),
189 ("/%E2%99%A5%E2%99%E2%99%A5/", "/♥%E2%99♥/"),
190 ("/%E2%E2%99%A5%E2%99%A5%99/", "/%E2♥♥%99/"),
191 (
192 "/%E2%99%A5%E2%99%A5/?utf8=%9C%93%E2%9C%93%9C%93",
193 "/♥♥/?utf8=%9C%93✓%9C%93",
194 ),
195 ]
196
197 for uri, iri in cases:
198 with self.subTest(uri):
199 self.assertEqual(uri_to_iri(uri), iri)
200
201 # Test idempotency.
202 self.assertEqual(uri_to_iri(uri_to_iri(uri)), iri)
203
204 def test_complementarity(self):
205 cases = [
206 (
207 "/blog/for/J%C3%BCrgen%20M%C3%BCnster/",
208 "/blog/for/J\xfcrgen%20M\xfcnster/",
209 ),
210 ("%&", "%&"),
211 ("red&%E2%99%A5ros%#red", "red&♥ros%#red"),
212 ("/%E2%99%A5%E2%99%A5/", "/♥♥/"),
213 ("/%E2%99%A5%E2%99%A5/?utf8=%E2%9C%93", "/♥♥/?utf8=✓"),
214 ("/%25%20%02%7b/", "/%25%20%02%7b/"),
215 ("/%AAd%AAj%AAa%AAn%AAg%AAo%AA/", "/%AAd%AAj%AAa%AAn%AAg%AAo%AA/"),
216 ("/%E2%99%A5%E2%E2%99%A5/", "/♥%E2♥/"),
217 ("/%E2%99%A5%E2%99%E2%99%A5/", "/♥%E2%99♥/"),
218 ("/%E2%E2%99%A5%E2%99%A5%99/", "/%E2♥♥%99/"),
219 (
220 "/%E2%99%A5%E2%99%A5/?utf8=%9C%93%E2%9C%93%9C%93",
221 "/♥♥/?utf8=%9C%93✓%9C%93",
222 ),
223 ]
224
225 for uri, iri in cases:
226 with self.subTest(uri):
227 self.assertEqual(iri_to_uri(uri_to_iri(uri)), uri)
228 self.assertEqual(uri_to_iri(iri_to_uri(iri)), iri)
229
230 def test_escape_uri_path(self):
231 cases = [
232 (
233 "/;some/=awful/?path/:with/@lots/&of/+awful/chars",
234 "/%3Bsome/%3Dawful/%3Fpath/:with/@lots/&of/+awful/chars",
235 ),
236 ("/foo#bar", "/foo%23bar"),
237 ("/foo?bar", "/foo%3Fbar"),
238 ]
239 for uri, expected in cases:
240 with self.subTest(uri):
241 self.assertEqual(escape_uri_path(uri), expected)