Debian Codegraph @

tests/utils_tests/test_text.py

raw
1 import json
2 import sys
3
4 from django.core.exceptions import SuspiciousFileOperation
5 from django.test import SimpleTestCase
6 from django.utils import text
7 from django.utils.functional import lazystr
8 from django.utils.text import format_lazy
9 from django.utils.translation import gettext_lazy, override
10
11 IS_WIDE_BUILD = len("\U0001f4a9") == 1
12
13
14 class TestUtilsText(SimpleTestCase):
15 def test_get_text_list(self):
16 self.assertEqual(text.get_text_list(["a", "b", "c", "d"]), "a, b, c or d")
17 self.assertEqual(text.get_text_list(["a", "b", "c"], "and"), "a, b and c")
18 self.assertEqual(text.get_text_list(["a", "b"], "and"), "a and b")
19 self.assertEqual(text.get_text_list(["a"]), "a")
20 self.assertEqual(text.get_text_list([]), "")
21 with override("ar"):
22 self.assertEqual(text.get_text_list(["a", "b", "c"]), "a، b أو c")
23
24 def test_smart_split(self):
25 testdata = [
26 ('This is "a person" test.', ["This", "is", '"a person"', "test."]),
27 ('This is "a person\'s" test.', ["This", "is", '"a person\'s"', "test."]),
28 ('This is "a person\\"s" test.', ["This", "is", '"a person\\"s"', "test."]),
29 ("\"a 'one", ['"a', "'one"]),
30 ("all friends' tests", ["all", "friends'", "tests"]),
31 (
32 'url search_page words="something else"',
33 ["url", "search_page", 'words="something else"'],
34 ),
35 (
36 "url search_page words='something else'",
37 ["url", "search_page", "words='something else'"],
38 ),
39 (
40 'url search_page words "something else"',
41 ["url", "search_page", "words", '"something else"'],
42 ),
43 (
44 'url search_page words-"something else"',
45 ["url", "search_page", 'words-"something else"'],
46 ),
47 ("url search_page words=hello", ["url", "search_page", "words=hello"]),
48 (
49 'url search_page words="something else',
50 ["url", "search_page", 'words="something', "else"],
51 ),
52 ("cut:','|cut:' '", ["cut:','|cut:' '"]),
53 (lazystr("a b c d"), ["a", "b", "c", "d"]), # Test for #20231
54 ]
55 for test, expected in testdata:
56 with self.subTest(value=test):
57 self.assertEqual(list(text.smart_split(test)), expected)
58
59 def test_truncate_chars(self):
60 truncator = text.Truncator("The quick brown fox jumped over the lazy dog.")
61 self.assertEqual(
62 "The quick brown fox jumped over the lazy dog.", truncator.chars(100)
63 ),
64 self.assertEqual("The quick brown fox …", truncator.chars(21))
65 self.assertEqual("The quick brown fo.....", truncator.chars(23, "....."))
66 self.assertEqual(".....", truncator.chars(4, "....."))
67
68 nfc = text.Truncator("o\xfco\xfco\xfco\xfc")
69 nfd = text.Truncator("ou\u0308ou\u0308ou\u0308ou\u0308")
70 self.assertEqual("oüoüoüoü", nfc.chars(8))
71 self.assertEqual("oüoüoüoü", nfd.chars(8))
72 self.assertEqual("oü…", nfc.chars(3))
73 self.assertEqual("oü…", nfd.chars(3))
74
75 # Ensure the final length is calculated correctly when there are
76 # combining characters with no precomposed form, and that combining
77 # characters are not split up.
78 truncator = text.Truncator("-B\u030aB\u030a----8")
79 self.assertEqual("-B\u030a…", truncator.chars(3))
80 self.assertEqual("-B\u030aB\u030a-…", truncator.chars(5))
81 self.assertEqual("-B\u030aB\u030a----8", truncator.chars(8))
82
83 # Ensure the length of the end text is correctly calculated when it
84 # contains combining characters with no precomposed form.
85 truncator = text.Truncator("-----")
86 self.assertEqual("---B\u030a", truncator.chars(4, "B\u030a"))
87 self.assertEqual("-----", truncator.chars(5, "B\u030a"))
88
89 # Make a best effort to shorten to the desired length, but requesting
90 # a length shorter than the ellipsis shouldn't break
91 self.assertEqual("...", text.Truncator("asdf").chars(1, truncate="..."))
92 # lazy strings are handled correctly
93 self.assertEqual(
94 text.Truncator(lazystr("The quick brown fox")).chars(10), "The quick…"
95 )
96
97 def test_truncate_chars_html(self):
98 truncator = text.Truncator(
99 '<p id="par"><strong><em>The quick brown fox jumped over the lazy dog.</em>'
100 "</strong></p>"
101 )
102 self.assertEqual(
103 '<p id="par"><strong><em>The quick brown fox jumped over the lazy dog.</em>'
104 "</strong></p>",
105 truncator.chars(80, html=True),
106 )
107 self.assertEqual(
108 '<p id="par"><strong><em>The quick brown fox jumped over the lazy dog.</em>'
109 "</strong></p>",
110 truncator.chars(46, html=True),
111 )
112 self.assertEqual(
113 '<p id="par"><strong><em>The quick brown fox jumped over the lazy dog…</em>'
114 "</strong></p>",
115 truncator.chars(45, html=True),
116 )
117 self.assertEqual(
118 '<p id="par"><strong><em>The quick…</em></strong></p>',
119 truncator.chars(10, html=True),
120 )
121 self.assertEqual(
122 '<p id="par"><strong><em>…</em></strong></p>',
123 truncator.chars(1, html=True),
124 )
125 self.assertEqual("", truncator.chars(0, html=True))
126 self.assertEqual("", truncator.chars(-1, html=True))
127 self.assertEqual(
128 '<p id="par"><strong><em>The qu....</em></strong></p>',
129 truncator.chars(10, "....", html=True),
130 )
131 self.assertEqual(
132 '<p id="par"><strong><em>The quick </em></strong></p>',
133 truncator.chars(10, "", html=True),
134 )
135 truncator = text.Truncator("foo</p>")
136 self.assertEqual("foo</p>", truncator.chars(5, html=True))
137
138 def test_truncate_chars_html_with_newline_inside_tag(self):
139 truncator = text.Truncator(
140 '<p>The quick <a href="xyz.html"\n id="mylink">brown fox</a> jumped over '
141 "the lazy dog.</p>"
142 )
143 self.assertEqual(
144 '<p>The quick <a href="xyz.html"\n id="mylink">brow…</a></p>',
145 truncator.chars(15, html=True),
146 )
147 self.assertEqual(
148 "<p>Th…</p>",
149 truncator.chars(3, html=True),
150 )
151
152 def test_truncate_chars_html_with_void_elements(self):
153 truncator = text.Truncator(
154 "<br/>The <hr />quick brown fox jumped over the lazy dog."
155 )
156 self.assertEqual("<br/>The <hr />quick brown…", truncator.chars(16, html=True))
157 truncator = text.Truncator(
158 "<br>The <hr/>quick <em>brown fox</em> jumped over the lazy dog."
159 )
160 self.assertEqual(
161 "<br>The <hr/>quick <em>brown…</em>", truncator.chars(16, html=True)
162 )
163 self.assertEqual("<br>The <hr/>q…", truncator.chars(6, html=True))
164 self.assertEqual("<br>The <hr/>…", truncator.chars(5, html=True))
165 self.assertEqual("<br>The…", truncator.chars(4, html=True))
166 self.assertEqual("<br>Th…", truncator.chars(3, html=True))
167
168 def test_truncate_chars_html_with_html_entities(self):
169 truncator = text.Truncator(
170 "<i>Buenos d&iacute;as! &#x00bf;C&oacute;mo est&aacute;?</i>"
171 )
172 self.assertEqual(
173 "<i>Buenos días! ¿Cómo está?</i>",
174 truncator.chars(40, html=True),
175 )
176 self.assertEqual(
177 "<i>Buenos días…</i>",
178 truncator.chars(12, html=True),
179 )
180 self.assertEqual(
181 "<i>Buenos días! ¿Cómo está…</i>",
182 truncator.chars(24, html=True),
183 )
184 truncator = text.Truncator("<p>I &lt;3 python, what about you?</p>")
185 self.assertEqual("<p>I &lt;3 python, wh…</p>", truncator.chars(16, html=True))
186
187 def test_truncate_chars_html_with_misnested_tags(self):
188 # LIFO removal keeps all tags when a middle tag is closed out of order.
189 # With <a><b><c></b>, the </b> doesn't match <c>, so all tags remain
190 # in the stack and are properly closed at truncation.
191 truncator = text.Truncator("<a><b><c></b>XXXX")
192 self.assertEqual(
193 truncator.chars(2, html=True, truncate=""),
194 "<a><b><c></b>XX</c></b></a>",
195 )
196
197 def test_truncate_words(self):
198 truncator = text.Truncator("The quick brown fox jumped over the lazy dog.")
199 self.assertEqual(
200 "The quick brown fox jumped over the lazy dog.", truncator.words(10)
201 )
202 self.assertEqual("The quick brown fox…", truncator.words(4))
203 self.assertEqual("The quick brown fox[snip]", truncator.words(4, "[snip]"))
204 # lazy strings are handled correctly
205 truncator = text.Truncator(
206 lazystr("The quick brown fox jumped over the lazy dog.")
207 )
208 self.assertEqual("The quick brown fox…", truncator.words(4))
209 self.assertEqual("", truncator.words(0))
210 self.assertEqual("", truncator.words(-1))
211
212 def test_truncate_html_words(self):
213 truncator = text.Truncator(
214 '<p id="par"><strong><em>The quick brown fox jumped over the lazy dog.</em>'
215 "</strong></p>"
216 )
217 self.assertEqual(
218 '<p id="par"><strong><em>The quick brown fox jumped over the lazy dog.</em>'
219 "</strong></p>",
220 truncator.words(10, html=True),
221 )
222 self.assertEqual(
223 '<p id="par"><strong><em>The quick brown fox…</em></strong></p>',
224 truncator.words(4, html=True),
225 )
226 self.assertEqual(
227 "",
228 truncator.words(0, html=True),
229 )
230 self.assertEqual(
231 '<p id="par"><strong><em>The quick brown fox....</em></strong></p>',
232 truncator.words(4, "....", html=True),
233 )
234 self.assertEqual(
235 '<p id="par"><strong><em>The quick brown fox</em></strong></p>',
236 truncator.words(4, "", html=True),
237 )
238
239 truncator = text.Truncator(
240 "<p>The quick \t brown fox jumped over the lazy dog.</p>"
241 )
242 self.assertEqual(
243 "<p>The quick brown fox…</p>",
244 truncator.words(4, html=True),
245 )
246
247 # Test with new line inside tag
248 truncator = text.Truncator(
249 '<p>The quick <a href="xyz.html"\n id="mylink">brown fox</a> jumped over '
250 "the lazy dog.</p>"
251 )
252 self.assertEqual(
253 '<p>The quick <a href="xyz.html"\n id="mylink">brown…</a></p>',
254 truncator.words(3, html=True),
255 )
256 self.assertEqual(
257 "<p>The…</p>",
258 truncator.words(1, html=True),
259 )
260
261 # Test self-closing tags
262 truncator = text.Truncator(
263 "<br/>The <hr />quick brown fox jumped over the lazy dog."
264 )
265 self.assertEqual("<br/>The <hr />quick brown…", truncator.words(3, html=True))
266 truncator = text.Truncator(
267 "<br>The <hr/>quick <em>brown fox</em> jumped over the lazy dog."
268 )
269 self.assertEqual(
270 "<br>The <hr/>quick <em>brown…</em>", truncator.words(3, html=True)
271 )
272
273 # Test html entities
274 truncator = text.Truncator(
275 "<i>Buenos d&iacute;as! &#x00bf;C&oacute;mo est&aacute;?</i>"
276 )
277 self.assertEqual(
278 "<i>Buenos días! ¿Cómo…</i>",
279 truncator.words(3, html=True),
280 )
281 truncator = text.Truncator("<p>I &lt;3 python, what about you?</p>")
282 self.assertEqual("<p>I &lt;3 python,…</p>", truncator.words(3, html=True))
283
284 truncator = text.Truncator("foo</p>")
285 self.assertEqual("foo</p>", truncator.words(3, html=True))
286
287 # Only open brackets.
288 truncator = text.Truncator("<" * 60_000)
289 self.assertEqual(truncator.words(1, html=True), "&lt;…")
290
291 # Tags with special chars in attrs.
292 truncator = text.Truncator(
293 """<i style="margin: 5%; font: *;">Hello, my dear lady!</i>"""
294 )
295 self.assertEqual(
296 """<i style="margin: 5%; font: *;">Hello, my dear…</i>""",
297 truncator.words(3, html=True),
298 )
299
300 # Tags with special non-latin chars in attrs.
301 truncator = text.Truncator("""<p data-x="א">Hello, my dear lady!</p>""")
302 self.assertEqual(
303 """<p data-x="א">Hello, my dear…</p>""",
304 truncator.words(3, html=True),
305 )
306
307 # Misplaced brackets.
308 truncator = text.Truncator("hello >< world")
309 self.assertEqual(truncator.words(1, html=True), "hello…")
310 self.assertEqual(truncator.words(2, html=True), "hello &gt;…")
311 self.assertEqual(truncator.words(3, html=True), "hello &gt;&lt;…")
312 self.assertEqual(truncator.words(4, html=True), "hello &gt;&lt; world")
313
314 def test_wrap(self):
315 digits = "1234 67 9"
316 self.assertEqual(text.wrap(digits, 100), "1234 67 9")
317 self.assertEqual(text.wrap(digits, 9), "1234 67 9")
318 self.assertEqual(text.wrap(digits, 8), "1234 67\n9")
319
320 self.assertEqual(text.wrap("short\na long line", 7), "short\na long\nline")
321 self.assertEqual(
322 text.wrap("do-not-break-long-words please? ok", 8),
323 "do-not-break-long-words\nplease?\nok",
324 )
325
326 long_word = "l%sng" % ("o" * 20)
327 self.assertEqual(text.wrap(long_word, 20), long_word)
328 self.assertEqual(
329 text.wrap("a %s word" % long_word, 10), "a\n%s\nword" % long_word
330 )
331 self.assertEqual(text.wrap(lazystr(digits), 100), "1234 67 9")
332
333 def test_normalize_newlines(self):
334 self.assertEqual(
335 text.normalize_newlines("abc\ndef\rghi\r\n"), "abc\ndef\nghi\n"
336 )
337 self.assertEqual(text.normalize_newlines("\n\r\r\n\r"), "\n\n\n\n")
338 self.assertEqual(text.normalize_newlines("abcdefghi"), "abcdefghi")
339 self.assertEqual(text.normalize_newlines(""), "")
340 self.assertEqual(
341 text.normalize_newlines(lazystr("abc\ndef\rghi\r\n")), "abc\ndef\nghi\n"
342 )
343
344 def test_phone2numeric(self):
345 numeric = text.phone2numeric("0800 flowers")
346 self.assertEqual(numeric, "0800 3569377")
347 lazy_numeric = lazystr(text.phone2numeric("0800 flowers"))
348 self.assertEqual(lazy_numeric, "0800 3569377")
349
350 def test_slugify(self):
351 items = (
352 # given - expected - Unicode?
353 ("Hello, World!", "hello-world", False),
354 ("spam & eggs", "spam-eggs", False),
355 (" multiple---dash and space ", "multiple-dash-and-space", False),
356 ("\t whitespace-in-value \n", "whitespace-in-value", False),
357 ("underscore_in-value", "underscore_in-value", False),
358 ("__strip__underscore-value___", "strip__underscore-value", False),
359 ("--strip-dash-value---", "strip-dash-value", False),
360 ("__strip-mixed-value---", "strip-mixed-value", False),
361 ("_ -strip-mixed-value _-", "strip-mixed-value", False),
362 ("spam & ıçüş", "spam-ıçüş", True),
363 ("foo ıç bar", "foo-ıç-bar", True),
364 (" foo ıç bar", "foo-ıç-bar", True),
365 ("你好", "你好", True),
366 ("İstanbul", "istanbul", True),
367 )
368 for value, output, is_unicode in items:
369 with self.subTest(value=value):
370 self.assertEqual(text.slugify(value, allow_unicode=is_unicode), output)
371 # Interning the result may be useful, e.g. when fed to Path.
372 with self.subTest("intern"):
373 self.assertEqual(sys.intern(text.slugify("a")), "a")
374
375 def test_unescape_string_literal(self):
376 items = [
377 ('"abc"', "abc"),
378 ("'abc'", "abc"),
379 ('"a "bc""', 'a "bc"'),
380 ("''ab' c'", "'ab' c"),
381 ]
382 for value, output in items:
383 with self.subTest(value=value):
384 self.assertEqual(text.unescape_string_literal(value), output)
385 self.assertEqual(text.unescape_string_literal(lazystr(value)), output)
386
387 def test_unescape_string_literal_invalid_value(self):
388 items = ["", "abc", "'abc\""]
389 for item in items:
390 msg = f"Not a string literal: {item!r}"
391 with self.assertRaisesMessage(ValueError, msg):
392 text.unescape_string_literal(item)
393
394 def test_get_valid_filename(self):
395 filename = "^&'@{}[],$=!-#()%+~_123.txt"
396 self.assertEqual(text.get_valid_filename(filename), "-_123.txt")
397 self.assertEqual(text.get_valid_filename(lazystr(filename)), "-_123.txt")
398 msg = "Could not derive file name from '???'"
399 with self.assertRaisesMessage(SuspiciousFileOperation, msg):
400 text.get_valid_filename("???")
401 # After sanitizing this would yield '..'.
402 msg = "Could not derive file name from '$.$.$'"
403 with self.assertRaisesMessage(SuspiciousFileOperation, msg):
404 text.get_valid_filename("$.$.$")
405
406 def test_compress_sequence(self):
407 data = [{"key": i} for i in range(10)]
408 seq = list(json.JSONEncoder().iterencode(data))
409 seq = [s.encode() for s in seq]
410 actual_length = len(b"".join(seq))
411 out = text.compress_sequence(seq)
412 compressed_length = len(b"".join(out))
413 self.assertLess(compressed_length, actual_length)
414
415 def test_format_lazy(self):
416 self.assertEqual("django/test", format_lazy("{}/{}", "django", lazystr("test")))
417 self.assertEqual("django/test", format_lazy("{0}/{1}", *("django", "test")))
418 self.assertEqual(
419 "django/test", format_lazy("{a}/{b}", **{"a": "django", "b": "test"})
420 )
421 self.assertEqual(
422 "django/test", format_lazy("{a[0]}/{a[1]}", a=("django", "test"))
423 )
424
425 t = {}
426 s = format_lazy("{0[a]}-{p[a]}", t, p=t)
427 t["a"] = lazystr("django")
428 self.assertEqual("django-django", s)
429 t["a"] = "update"
430 self.assertEqual("update-update", s)
431
432 # The format string can be lazy. (string comes from contrib.admin)
433 s = format_lazy(
434 gettext_lazy("Added {name} “{object}”."),
435 name="article",
436 object="My first try",
437 )
438 with override("fr"):
439 self.assertEqual("Ajout de article «\xa0My first try\xa0».", s)