Debian Codegraph @

Lib/encodings/idna.py

raw
1 # This module implements the RFCs 3490 (IDNA) and 3491 (Nameprep)
2
3 import stringprep, re, codecs
4 from unicodedata import ucd_3_2_0 as unicodedata
5
6 # IDNA section 3.1
7 dots = re.compile("[\u002E\u3002\uFF0E\uFF61]")
8
9 # IDNA section 5
10 ace_prefix = b"xn--"
11 sace_prefix = "xn--"
12
13 # This assumes query strings, so AllowUnassigned is true
14 def nameprep(label): # type: (str) -> str
15 # Map
16 newlabel = []
17 for c in label:
18 if stringprep.in_table_b1(c):
19 # Map to nothing
20 continue
21 newlabel.append(stringprep.map_table_b2(c))
22 label = "".join(newlabel)
23
24 # Normalize
25 label = unicodedata.normalize("NFKC", label)
26
27 # Prohibit
28 for i, c in enumerate(label):
29 if stringprep.in_table_c12(c) or \
30 stringprep.in_table_c22(c) or \
31 stringprep.in_table_c3(c) or \
32 stringprep.in_table_c4(c) or \
33 stringprep.in_table_c5(c) or \
34 stringprep.in_table_c6(c) or \
35 stringprep.in_table_c7(c) or \
36 stringprep.in_table_c8(c) or \
37 stringprep.in_table_c9(c):
38 raise UnicodeEncodeError("idna", label, i, i+1, f"Invalid character {c!r}")
39
40 # Check bidi
41 RandAL = [stringprep.in_table_d1(x) for x in label]
42 if any(RandAL):
43 # There is a RandAL char in the string. Must perform further
44 # tests:
45 # 1) The characters in section 5.8 MUST be prohibited.
46 # This is table C.8, which was already checked
47 # 2) If a string contains any RandALCat character, the string
48 # MUST NOT contain any LCat character.
49 for i, x in enumerate(label):
50 if stringprep.in_table_d2(x):
51 raise UnicodeEncodeError("idna", label, i, i+1,
52 "Violation of BIDI requirement 2")
53 # 3) If a string contains any RandALCat character, a
54 # RandALCat character MUST be the first character of the
55 # string, and a RandALCat character MUST be the last
56 # character of the string.
57 if not RandAL[0]:
58 raise UnicodeEncodeError("idna", label, 0, 1,
59 "Violation of BIDI requirement 3")
60 if not RandAL[-1]:
61 raise UnicodeEncodeError("idna", label, len(label)-1, len(label),
62 "Violation of BIDI requirement 3")
63
64 return label
65
66 def ToASCII(label): # type: (str) -> bytes
67 try:
68 # Step 1: try ASCII
69 label_ascii = label.encode("ascii")
70 except UnicodeEncodeError:
71 pass
72 else:
73 # Skip to step 3: UseSTD3ASCIIRules is false, so
74 # Skip to step 8.
75 if 0 < len(label_ascii) < 64:
76 return label_ascii
77 if len(label) == 0:
78 raise UnicodeEncodeError("idna", label, 0, 1, "label empty")
79 else:
80 raise UnicodeEncodeError("idna", label, 0, len(label), "label too long")
81
82 # Step 2: nameprep
83 label = nameprep(label)
84
85 # Step 3: UseSTD3ASCIIRules is false
86 # Step 4: try ASCII
87 try:
88 label_ascii = label.encode("ascii")
89 except UnicodeEncodeError:
90 pass
91 else:
92 # Skip to step 8.
93 if 0 < len(label) < 64:
94 return label_ascii
95 if len(label) == 0:
96 raise UnicodeEncodeError("idna", label, 0, 1, "label empty")
97 else:
98 raise UnicodeEncodeError("idna", label, 0, len(label), "label too long")
99
100 # Step 5: Check ACE prefix
101 if label.lower().startswith(sace_prefix):
102 raise UnicodeEncodeError(
103 "idna", label, 0, len(sace_prefix), "Label starts with ACE prefix")
104
105 # Step 6: Encode with PUNYCODE
106 label_ascii = label.encode("punycode")
107
108 # Step 7: Prepend ACE prefix
109 label_ascii = ace_prefix + label_ascii
110
111 # Step 8: Check size
112 # do not check for empty as we prepend ace_prefix.
113 if len(label_ascii) < 64:
114 return label_ascii
115 raise UnicodeEncodeError("idna", label, 0, len(label), "label too long")
116
117 def ToUnicode(label):
118 if len(label) > 1024:
119 # Protection from https://github.com/python/cpython/issues/98433.
120 # https://datatracker.ietf.org/doc/html/rfc5894#section-6
121 # doesn't specify a label size limit prior to NAMEPREP. But having
122 # one makes practical sense.
123 # This leaves ample room for nameprep() to remove Nothing characters
124 # per https://www.rfc-editor.org/rfc/rfc3454#section-3.1 while still
125 # preventing us from wasting time decoding a big thing that'll just
126 # hit the actual <= 63 length limit in Step 6.
127 if isinstance(label, str):
128 label = label.encode("utf-8", errors="backslashreplace")
129 raise UnicodeDecodeError("idna", label, 0, len(label), "label way too long")
130 # Step 1: Check for ASCII
131 if isinstance(label, bytes):
132 pure_ascii = True
133 else:
134 try:
135 label = label.encode("ascii")
136 pure_ascii = True
137 except UnicodeEncodeError:
138 pure_ascii = False
139 if not pure_ascii:
140 assert isinstance(label, str)
141 # Step 2: Perform nameprep
142 label = nameprep(label)
143 # It doesn't say this, but apparently, it should be ASCII now
144 try:
145 label = label.encode("ascii")
146 except UnicodeEncodeError as exc:
147 raise UnicodeEncodeError("idna", label, exc.start, exc.end,
148 "Invalid character in IDN label")
149 # Step 3: Check for ACE prefix
150 assert isinstance(label, bytes)
151 if not label.lower().startswith(ace_prefix):
152 return str(label, "ascii")
153
154 # Step 4: Remove ACE prefix
155 label1 = label[len(ace_prefix):]
156
157 # Step 5: Decode using PUNYCODE
158 try:
159 result = label1.decode("punycode")
160 except UnicodeDecodeError as exc:
161 offset = len(ace_prefix)
162 raise UnicodeDecodeError("idna", label, offset+exc.start, offset+exc.end, exc.reason)
163
164 # Step 6: Apply ToASCII
165 label2 = ToASCII(result)
166
167 # Step 7: Compare the result of step 6 with the one of step 3
168 # label2 will already be in lower case.
169 if str(label, "ascii").lower() != str(label2, "ascii"):
170 raise UnicodeDecodeError("idna", label, 0, len(label),
171 f"IDNA does not round-trip, '{label!r}' != '{label2!r}'")
172
173 # Step 8: return the result of step 5
174 return result
175
176 ### Codec APIs
177
178 class Codec(codecs.Codec):
179 def encode(self, input, errors='strict'):
180
181 if errors != 'strict':
182 # IDNA is quite clear that implementations must be strict
183 raise UnicodeError(f"Unsupported error handling: {errors}")
184
185 if not input:
186 return b'', 0
187
188 try:
189 result = input.encode('ascii')
190 except UnicodeEncodeError:
191 pass
192 else:
193 # ASCII name: fast path
194 labels = result.split(b'.')
195 for i, label in enumerate(labels[:-1]):
196 if len(label) == 0:
197 offset = sum(len(l) for l in labels[:i]) + i
198 raise UnicodeEncodeError("idna", input, offset, offset+1,
199 "label empty")
200 for i, label in enumerate(labels):
201 if len(label) >= 64:
202 offset = sum(len(l) for l in labels[:i]) + i
203 raise UnicodeEncodeError("idna", input, offset, offset+len(label),
204 "label too long")
205 return result, len(input)
206
207 result = bytearray()
208 labels = dots.split(input)
209 if labels and not labels[-1]:
210 trailing_dot = b'.'
211 del labels[-1]
212 else:
213 trailing_dot = b''
214 for i, label in enumerate(labels):
215 if result:
216 # Join with U+002E
217 result.extend(b'.')
218 try:
219 result.extend(ToASCII(label))
220 except (UnicodeEncodeError, UnicodeDecodeError) as exc:
221 offset = sum(len(l) for l in labels[:i]) + i
222 raise UnicodeEncodeError(
223 "idna",
224 input,
225 offset + exc.start,
226 offset + exc.end,
227 exc.reason,
228 )
229 return bytes(result+trailing_dot), len(input)
230
231 def decode(self, input, errors='strict'):
232
233 if errors != 'strict':
234 raise UnicodeError(f"Unsupported error handling: {errors}")
235
236 if not input:
237 return "", 0
238
239 # IDNA allows decoding to operate on Unicode strings, too.
240 if not isinstance(input, bytes):
241 # XXX obviously wrong, see #3232
242 input = bytes(input)
243
244 if ace_prefix not in input.lower():
245 # Fast path
246 try:
247 return input.decode('ascii'), len(input)
248 except UnicodeDecodeError:
249 pass
250
251 labels = input.split(b".")
252
253 if labels and len(labels[-1]) == 0:
254 trailing_dot = '.'
255 del labels[-1]
256 else:
257 trailing_dot = ''
258
259 result = []
260 for i, label in enumerate(labels):
261 try:
262 u_label = ToUnicode(label)
263 except (UnicodeEncodeError, UnicodeDecodeError) as exc:
264 offset = sum(len(x) for x in labels[:i]) + len(labels[:i])
265 raise UnicodeDecodeError(
266 "idna", input, offset+exc.start, offset+exc.end, exc.reason)
267 else:
268 result.append(u_label)
269
270 return ".".join(result)+trailing_dot, len(input)
271
272 class IncrementalEncoder(codecs.BufferedIncrementalEncoder):
273 def _buffer_encode(self, input, errors, final):
274 if errors != 'strict':
275 # IDNA is quite clear that implementations must be strict
276 raise UnicodeError(f"Unsupported error handling: {errors}")
277
278 if not input:
279 return (b'', 0)
280
281 labels = dots.split(input)
282 trailing_dot = b''
283 if labels:
284 if not labels[-1]:
285 trailing_dot = b'.'
286 del labels[-1]
287 elif not final:
288 # Keep potentially unfinished label until the next call
289 del labels[-1]
290 if labels:
291 trailing_dot = b'.'
292
293 result = bytearray()
294 size = 0
295 for label in labels:
296 if size:
297 # Join with U+002E
298 result.extend(b'.')
299 size += 1
300 try:
301 result.extend(ToASCII(label))
302 except (UnicodeEncodeError, UnicodeDecodeError) as exc:
303 raise UnicodeEncodeError(
304 "idna",
305 input,
306 size + exc.start,
307 size + exc.end,
308 exc.reason,
309 )
310 size += len(label)
311
312 result += trailing_dot
313 size += len(trailing_dot)
314 return (bytes(result), size)
315
316 class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
317 def _buffer_decode(self, input, errors, final):
318 if errors != 'strict':
319 raise UnicodeError(f"Unsupported error handling: {errors}")
320
321 if not input:
322 return ("", 0)
323
324 # IDNA allows decoding to operate on Unicode strings, too.
325 if isinstance(input, str):
326 labels = dots.split(input)
327 else:
328 # Must be ASCII string
329 try:
330 input = str(input, "ascii")
331 except (UnicodeEncodeError, UnicodeDecodeError) as exc:
332 raise UnicodeDecodeError("idna", input,
333 exc.start, exc.end, exc.reason)
334 labels = input.split(".")
335
336 trailing_dot = ''
337 if labels:
338 if not labels[-1]:
339 trailing_dot = '.'
340 del labels[-1]
341 elif not final:
342 # Keep potentially unfinished label until the next call
343 del labels[-1]
344 if labels:
345 trailing_dot = '.'
346
347 result = []
348 size = 0
349 for label in labels:
350 try:
351 u_label = ToUnicode(label)
352 except (UnicodeEncodeError, UnicodeDecodeError) as exc:
353 raise UnicodeDecodeError(
354 "idna",
355 input.encode("ascii", errors="backslashreplace"),
356 size + exc.start,
357 size + exc.end,
358 exc.reason,
359 )
360 else:
361 result.append(u_label)
362 if size:
363 size += 1
364 size += len(label)
365
366 result = ".".join(result) + trailing_dot
367 size += len(trailing_dot)
368 return (result, size)
369
370 class StreamWriter(Codec,codecs.StreamWriter):
371 pass
372
373 class StreamReader(Codec,codecs.StreamReader):
374 pass
375
376 ### encodings module API
377
378 def getregentry():
379 return codecs.CodecInfo(
380 name='idna',
381 encode=Codec().encode,
382 decode=Codec().decode,
383 incrementalencoder=IncrementalEncoder,
384 incrementaldecoder=IncrementalDecoder,
385 streamwriter=StreamWriter,
386 streamreader=StreamReader,
387 )