Debian Codegraph @

django/utils/text.py

raw
1 import gzip
2 import re
3 import secrets
4 import textwrap
5 import unicodedata
6 from collections import deque
7 from gzip import GzipFile
8 from gzip import compress as gzip_compress
9 from html import escape
10 from html.parser import HTMLParser
11 from io import BytesIO
12
13 from django.core.exceptions import SuspiciousFileOperation
14 from django.utils.functional import (
15 SimpleLazyObject,
16 cached_property,
17 keep_lazy_text,
18 lazy,
19 )
20 from django.utils.regex_helper import _lazy_re_compile
21 from django.utils.translation import gettext as _
22 from django.utils.translation import gettext_lazy, pgettext
23
24
25 @keep_lazy_text
26 def capfirst(x):
27 """Capitalize the first letter of a string."""
28 if not x:
29 return x
30 if not isinstance(x, str):
31 x = str(x)
32 return x[0].upper() + x[1:]
33
34
35 # Set up regular expressions
36 re_newlines = _lazy_re_compile(r"\r\n|\r") # Used in normalize_newlines
37 re_camel_case = _lazy_re_compile(r"(((?<=[a-z])[A-Z])|([A-Z](?![A-Z]|$)))")
38
39
40 @keep_lazy_text
41 def wrap(text, width):
42 """
43 A word-wrap function that preserves existing line breaks. Expects that
44 existing line breaks are posix newlines.
45
46 Preserve all white space except added line breaks consume the space on
47 which they break the line.
48
49 Don't wrap long words, thus the output text may have lines longer than
50 ``width``.
51 """
52
53 wrapper = textwrap.TextWrapper(
54 width=width,
55 break_long_words=False,
56 break_on_hyphens=False,
57 replace_whitespace=False,
58 )
59 result = []
60 for line in text.splitlines():
61 wrapped = wrapper.wrap(line)
62 if not wrapped:
63 # If `line` contains only whitespaces that are dropped, restore it.
64 result.append(line)
65 else:
66 result.extend(wrapped)
67 if text.endswith("\n"):
68 # If `text` ends with a newline, preserve it.
69 result.append("")
70 return "\n".join(result)
71
72
73 def add_truncation_text(text, truncate=None):
74 if truncate is None:
75 truncate = pgettext(
76 "String to return when truncating text", "%(truncated_text)s…"
77 )
78 if "%(truncated_text)s" in truncate:
79 return truncate % {"truncated_text": text}
80 # The truncation text didn't contain the %(truncated_text)s string
81 # replacement argument so just append it to the text.
82 if text.endswith(truncate):
83 # But don't append the truncation text if the current text already ends
84 # in this.
85 return text
86 return f"{text}{truncate}"
87
88
89 def calculate_truncate_chars_length(length, replacement):
90 truncate_len = length
91 for char in add_truncation_text("", replacement):
92 if not unicodedata.combining(char):
93 truncate_len -= 1
94 if truncate_len == 0:
95 break
96 return truncate_len
97
98
99 class TruncateHTMLParser(HTMLParser):
100 class TruncationCompleted(Exception):
101 pass
102
103 def __init__(self, *, length, replacement, convert_charrefs=True):
104 super().__init__(convert_charrefs=convert_charrefs)
105 self.tags = deque()
106 self.output = ""
107 self.remaining = length
108 self.replacement = replacement
109
110 @cached_property
111 def void_elements(self):
112 from django.utils.html import VOID_ELEMENTS
113
114 return VOID_ELEMENTS
115
116 def handle_startendtag(self, tag, attrs):
117 self.handle_starttag(tag, attrs)
118 if tag not in self.void_elements:
119 self.handle_endtag(tag)
120
121 def handle_starttag(self, tag, attrs):
122 self.output += self.get_starttag_text()
123 if tag not in self.void_elements:
124 self.tags.appendleft(tag)
125
126 def handle_endtag(self, tag):
127 if tag not in self.void_elements:
128 self.output += f"</{tag}>"
129 # Remove from the stack only if the tag matches the most recently
130 # opened tag (LIFO). This avoids O(n) linear scans for unmatched
131 # end tags if `deque.remove()` would be called.
132 if self.tags and self.tags[0] == tag:
133 self.tags.popleft()
134
135 def handle_data(self, data):
136 data, output = self.process(data)
137 data_len = len(data)
138 if self.remaining < data_len:
139 self.remaining = 0
140 self.output += add_truncation_text(output, self.replacement)
141 raise self.TruncationCompleted
142 self.remaining -= data_len
143 self.output += output
144
145 def feed(self, data):
146 try:
147 super().feed(data)
148 except self.TruncationCompleted:
149 self.output += "".join([f"</{tag}>" for tag in self.tags])
150 self.tags.clear()
151 self.reset()
152 else:
153 # No data was handled.
154 self.reset()
155
156
157 class TruncateCharsHTMLParser(TruncateHTMLParser):
158 def __init__(self, *, length, replacement, convert_charrefs=True):
159 self.length = length
160 self.processed_chars = 0
161 super().__init__(
162 length=calculate_truncate_chars_length(length, replacement),
163 replacement=replacement,
164 convert_charrefs=convert_charrefs,
165 )
166
167 def process(self, data):
168 self.processed_chars += len(data)
169 if (self.processed_chars == self.length) and (
170 len(self.output) + len(data) == len(self.rawdata)
171 ):
172 self.output += data
173 raise self.TruncationCompleted
174 output = escape("".join(data[: self.remaining]))
175 return data, output
176
177
178 class TruncateWordsHTMLParser(TruncateHTMLParser):
179 def process(self, data):
180 data = re.split(r"(?<=\S)\s+(?=\S)", data)
181 output = escape(" ".join(data[: self.remaining]))
182 return data, output
183
184
185 class Truncator(SimpleLazyObject):
186 """
187 An object used to truncate text, either by characters or words.
188 """
189
190 def __init__(self, text):
191 super().__init__(lambda: str(text))
192
193 def chars(self, num, truncate=None, html=False):
194 """
195 Return the text truncated to be no longer than the specified number
196 of characters.
197
198 `truncate` specifies what should be used to notify that the string has
199 been truncated, defaulting to a translatable string of an ellipsis.
200 """
201 self._setup()
202 length = int(num)
203 if length <= 0:
204 return ""
205 text = unicodedata.normalize("NFC", self._wrapped)
206
207 if html:
208 parser = TruncateCharsHTMLParser(length=length, replacement=truncate)
209 parser.feed(text)
210 parser.close()
211 return parser.output
212 return self._text_chars(length, truncate, text)
213
214 def _text_chars(self, length, truncate, text):
215 """Truncate a string after a certain number of chars."""
216 truncate_len = calculate_truncate_chars_length(length, truncate)
217 s_len = 0
218 end_index = None
219 for i, char in enumerate(text):
220 if unicodedata.combining(char):
221 # Don't consider combining characters
222 # as adding to the string length
223 continue
224 s_len += 1
225 if end_index is None and s_len > truncate_len:
226 end_index = i
227 if s_len > length:
228 # Return the truncated string
229 return add_truncation_text(text[: end_index or 0], truncate)
230
231 # Return the original string since no truncation was necessary
232 return text
233
234 def words(self, num, truncate=None, html=False):
235 """
236 Truncate a string after a certain number of words. `truncate` specifies
237 what should be used to notify that the string has been truncated,
238 defaulting to ellipsis.
239 """
240 self._setup()
241 length = int(num)
242 if length <= 0:
243 return ""
244 if html:
245 parser = TruncateWordsHTMLParser(length=length, replacement=truncate)
246 parser.feed(self._wrapped)
247 parser.close()
248 return parser.output
249 return self._text_words(length, truncate)
250
251 def _text_words(self, length, truncate):
252 """
253 Truncate a string after a certain number of words.
254
255 Strip newlines in the string.
256 """
257 words = self._wrapped.split()
258 if len(words) > length:
259 words = words[:length]
260 return add_truncation_text(" ".join(words), truncate)
261 return " ".join(words)
262
263
264 @keep_lazy_text
265 def get_valid_filename(name):
266 """
267 Return the given string converted to a string that can be used for a clean
268 filename. Remove leading and trailing spaces; convert other spaces to
269 underscores; and remove anything that is not an alphanumeric, dash,
270 underscore, or dot.
271 >>> get_valid_filename("john's portrait in 2004.jpg")
272 'johns_portrait_in_2004.jpg'
273 """
274 s = str(name).strip().replace(" ", "_")
275 s = re.sub(r"(?u)[^-\w.]", "", s)
276 if s in {"", ".", ".."}:
277 raise SuspiciousFileOperation("Could not derive file name from '%s'" % name)
278 return s
279
280
281 @keep_lazy_text
282 def get_text_list(list_, last_word=gettext_lazy("or")):
283 """
284 >>> get_text_list(['a', 'b', 'c', 'd'])
285 'a, b, c or d'
286 >>> get_text_list(['a', 'b', 'c'], 'and')
287 'a, b and c'
288 >>> get_text_list(['a', 'b'], 'and')
289 'a and b'
290 >>> get_text_list(['a'])
291 'a'
292 >>> get_text_list([])
293 ''
294 """
295 if not list_:
296 return ""
297 if len(list_) == 1:
298 return str(list_[0])
299 return "%s %s %s" % (
300 # Translators: This string is used as a separator between list elements
301 _(", ").join(str(i) for i in list_[:-1]),
302 str(last_word),
303 str(list_[-1]),
304 )
305
306
307 @keep_lazy_text
308 def normalize_newlines(text):
309 """Normalize CRLF and CR newlines to just LF."""
310 return re_newlines.sub("\n", str(text))
311
312
313 @keep_lazy_text
314 def phone2numeric(phone):
315 """Convert a phone number with letters into its numeric equivalent."""
316 char2number = {
317 "a": "2",
318 "b": "2",
319 "c": "2",
320 "d": "3",
321 "e": "3",
322 "f": "3",
323 "g": "4",
324 "h": "4",
325 "i": "4",
326 "j": "5",
327 "k": "5",
328 "l": "5",
329 "m": "6",
330 "n": "6",
331 "o": "6",
332 "p": "7",
333 "q": "7",
334 "r": "7",
335 "s": "7",
336 "t": "8",
337 "u": "8",
338 "v": "8",
339 "w": "9",
340 "x": "9",
341 "y": "9",
342 "z": "9",
343 }
344 return "".join(char2number.get(c, c) for c in phone.lower())
345
346
347 def _get_random_filename(max_random_bytes):
348 return b"a" * secrets.randbelow(max_random_bytes)
349
350
351 def compress_string(s, *, max_random_bytes=None):
352 compressed_data = gzip_compress(s, compresslevel=6, mtime=0)
353
354 if not max_random_bytes:
355 return compressed_data
356
357 compressed_view = memoryview(compressed_data)
358 header = bytearray(compressed_view[:10])
359 header[3] = gzip.FNAME
360
361 filename = _get_random_filename(max_random_bytes) + b"\x00"
362
363 return bytes(header) + filename + compressed_view[10:]
364
365
366 class StreamingBuffer(BytesIO):
367 def read(self):
368 ret = self.getvalue()
369 self.seek(0)
370 self.truncate()
371 return ret
372
373
374 # Like compress_string, but for iterators of strings.
375 def compress_sequence(sequence, *, max_random_bytes=None):
376 buf = StreamingBuffer()
377 filename = _get_random_filename(max_random_bytes) if max_random_bytes else None
378 with GzipFile(
379 filename=filename, mode="wb", compresslevel=6, fileobj=buf, mtime=0
380 ) as zfile:
381 # Output headers...
382 yield buf.read()
383 for item in sequence:
384 zfile.write(item)
385 data = buf.read()
386 if data:
387 yield data
388 yield buf.read()
389
390
391 # Expression to match some_token and some_token="with spaces" (and similarly
392 # for single-quoted strings).
393 smart_split_re = _lazy_re_compile(
394 r"""
395 ((?:
396 [^\s'"]*
397 (?:
398 (?:"(?:[^"\\]|\\.)*" | '(?:[^'\\]|\\.)*')
399 [^\s'"]*
400 )+
401 ) | \S+)
402 """,
403 re.VERBOSE,
404 )
405
406
407 def smart_split(text):
408 r"""
409 Generator that splits a string by spaces, leaving quoted phrases together.
410 Supports both single and double quotes, and supports escaping quotes with
411 backslashes. In the output, strings will keep their initial and trailing
412 quote marks and escaped quotes will remain escaped (the results can then
413 be further processed with unescape_string_literal()).
414
415 >>> list(smart_split(r'This is "a person\'s" test.'))
416 ['This', 'is', '"a person\\\'s"', 'test.']
417 >>> list(smart_split(r"Another 'person\'s' test."))
418 ['Another', "'person\\'s'", 'test.']
419 >>> list(smart_split(r'A "\"funky\" style" test.'))
420 ['A', '"\\"funky\\" style"', 'test.']
421 """
422 for bit in smart_split_re.finditer(str(text)):
423 yield bit[0]
424
425
426 @keep_lazy_text
427 def unescape_string_literal(s):
428 r"""
429 Convert quoted string literals to unquoted strings with escaped quotes and
430 backslashes unquoted::
431
432 >>> unescape_string_literal('"abc"')
433 'abc'
434 >>> unescape_string_literal("'abc'")
435 'abc'
436 >>> unescape_string_literal('"a \"bc\""')
437 'a "bc"'
438 >>> unescape_string_literal("'\'ab\' c'")
439 "'ab' c"
440 """
441 if not s or s[0] not in "\"'" or s[-1] != s[0]:
442 raise ValueError("Not a string literal: %r" % s)
443 quote = s[0]
444 return s[1:-1].replace(r"\%s" % quote, quote).replace(r"\\", "\\")
445
446
447 @keep_lazy_text
448 def slugify(value, allow_unicode=False):
449 """
450 Convert to ASCII if 'allow_unicode' is False. Convert spaces or repeated
451 dashes to single dashes. Remove characters that aren't alphanumerics,
452 underscores, or hyphens. Convert to lowercase. Also strip leading and
453 trailing whitespace, dashes, and underscores.
454 """
455 value = str(value)
456 if allow_unicode:
457 value = unicodedata.normalize("NFKC", value)
458 else:
459 value = (
460 unicodedata.normalize("NFKD", value)
461 .encode("ascii", "ignore")
462 .decode("ascii")
463 )
464 value = re.sub(r"[^\w\s-]", "", value.lower())
465 return re.sub(r"[-\s]+", "-", value).strip("-_")
466
467
468 def camel_case_to_spaces(value):
469 """
470 Split CamelCase and convert to lowercase. Strip surrounding whitespace.
471 """
472 return re_camel_case.sub(r" \1", value).strip().lower()
473
474
475 def _format_lazy(format_string, *args, **kwargs):
476 """
477 Apply str.format() on 'format_string' where format_string, args,
478 and/or kwargs might be lazy.
479 """
480 return format_string.format(*args, **kwargs)
481
482
483 format_lazy = lazy(_format_lazy, str)