Debian Codegraph @

Lib/encodings/punycode.py

raw
1 """ Codec for the Punycode encoding, as specified in RFC 3492
2
3 Written by Martin v. Löwis.
4 """
5
6 import codecs
7
8 ##################### Encoding #####################################
9
10 def segregate(str):
11 """3.1 Basic code point segregation"""
12 base = bytearray()
13 extended = set()
14 for c in str:
15 if ord(c) < 128:
16 base.append(ord(c))
17 else:
18 extended.add(c)
19 extended = sorted(extended)
20 return bytes(base), extended
21
22 def selective_len(str, max):
23 """Return the length of str, considering only characters below max."""
24 res = 0
25 for c in str:
26 if ord(c) < max:
27 res += 1
28 return res
29
30 def selective_find(str, char, index, pos):
31 """Return a pair (index, pos), indicating the next occurrence of
32 char in str. index is the position of the character considering
33 only ordinals up to and including char, and pos is the position in
34 the full string. index/pos is the starting position in the full
35 string."""
36
37 l = len(str)
38 while 1:
39 pos += 1
40 if pos == l:
41 return (-1, -1)
42 c = str[pos]
43 if c == char:
44 return index+1, pos
45 elif c < char:
46 index += 1
47
48 def insertion_unsort(str, extended):
49 """3.2 Insertion unsort coding"""
50 oldchar = 0x80
51 result = []
52 oldindex = -1
53 for c in extended:
54 index = pos = -1
55 char = ord(c)
56 curlen = selective_len(str, char)
57 delta = (curlen+1) * (char - oldchar)
58 while 1:
59 index,pos = selective_find(str,c,index,pos)
60 if index == -1:
61 break
62 delta += index - oldindex
63 result.append(delta-1)
64 oldindex = index
65 delta = 0
66 oldchar = char
67
68 return result
69
70 def T(j, bias):
71 # Punycode parameters: tmin = 1, tmax = 26, base = 36
72 res = 36 * (j + 1) - bias
73 if res < 1: return 1
74 if res > 26: return 26
75 return res
76
77 digits = b"abcdefghijklmnopqrstuvwxyz0123456789"
78 def generate_generalized_integer(N, bias):
79 """3.3 Generalized variable-length integers"""
80 result = bytearray()
81 j = 0
82 while 1:
83 t = T(j, bias)
84 if N < t:
85 result.append(digits[N])
86 return bytes(result)
87 result.append(digits[t + ((N - t) % (36 - t))])
88 N = (N - t) // (36 - t)
89 j += 1
90
91 def adapt(delta, first, numchars):
92 if first:
93 delta //= 700
94 else:
95 delta //= 2
96 delta += delta // numchars
97 # ((base - tmin) * tmax) // 2 == 455
98 divisions = 0
99 while delta > 455:
100 delta = delta // 35 # base - tmin
101 divisions += 36
102 bias = divisions + (36 * delta // (delta + 38))
103 return bias
104
105
106 def generate_integers(baselen, deltas):
107 """3.4 Bias adaptation"""
108 # Punycode parameters: initial bias = 72, damp = 700, skew = 38
109 result = bytearray()
110 bias = 72
111 for points, delta in enumerate(deltas):
112 s = generate_generalized_integer(delta, bias)
113 result.extend(s)
114 bias = adapt(delta, points==0, baselen+points+1)
115 return bytes(result)
116
117 def punycode_encode(text):
118 base, extended = segregate(text)
119 deltas = insertion_unsort(text, extended)
120 extended = generate_integers(len(base), deltas)
121 if base:
122 return base + b"-" + extended
123 return extended
124
125 ##################### Decoding #####################################
126
127 def decode_generalized_number(extended, extpos, bias, errors):
128 """3.3 Generalized variable-length integers"""
129 result = 0
130 w = 1
131 j = 0
132 while 1:
133 try:
134 char = extended[extpos]
135 except IndexError:
136 if errors == "strict":
137 raise UnicodeDecodeError("punycode", extended, extpos, extpos+1,
138 "incomplete punycode string")
139 return extpos + 1, None
140 extpos += 1
141 if 0x41 <= char <= 0x5A: # A-Z
142 digit = char - 0x41
143 elif 0x30 <= char <= 0x39:
144 digit = char - 22 # 0x30-26
145 elif errors == "strict":
146 raise UnicodeDecodeError("punycode", extended, extpos-1, extpos,
147 f"Invalid extended code point '{extended[extpos-1]}'")
148 else:
149 return extpos, None
150 t = T(j, bias)
151 result += digit * w
152 if digit < t:
153 return extpos, result
154 w = w * (36 - t)
155 j += 1
156
157
158 def insertion_sort(base, extended, errors):
159 """3.2 Insertion sort coding"""
160 # This function raises UnicodeDecodeError with position in the extended.
161 # Caller should add the offset.
162 char = 0x80
163 pos = -1
164 bias = 72
165 extpos = 0
166
167 while extpos < len(extended):
168 newpos, delta = decode_generalized_number(extended, extpos,
169 bias, errors)
170 if delta is None:
171 # There was an error in decoding. We can't continue because
172 # synchronization is lost.
173 return base
174 pos += delta+1
175 char += pos // (len(base) + 1)
176 if char > 0x10FFFF:
177 if errors == "strict":
178 raise UnicodeDecodeError(
179 "punycode", extended, pos-1, pos,
180 f"Invalid character U+{char:x}")
181 char = ord('?')
182 pos = pos % (len(base) + 1)
183 base = base[:pos] + chr(char) + base[pos:]
184 bias = adapt(delta, (extpos == 0), len(base))
185 extpos = newpos
186 return base
187
188 def punycode_decode(text, errors):
189 if isinstance(text, str):
190 text = text.encode("ascii")
191 if isinstance(text, memoryview):
192 text = bytes(text)
193 pos = text.rfind(b"-")
194 if pos == -1:
195 base = ""
196 extended = text.upper()
197 else:
198 try:
199 base = str(text[:pos], "ascii", errors)
200 except UnicodeDecodeError as exc:
201 raise UnicodeDecodeError("ascii", text, exc.start, exc.end,
202 exc.reason) from None
203 extended = text[pos+1:].upper()
204 try:
205 return insertion_sort(base, extended, errors)
206 except UnicodeDecodeError as exc:
207 offset = pos + 1
208 raise UnicodeDecodeError("punycode", text,
209 offset+exc.start, offset+exc.end,
210 exc.reason) from None
211
212 ### Codec APIs
213
214 class Codec(codecs.Codec):
215
216 def encode(self, input, errors='strict'):
217 res = punycode_encode(input)
218 return res, len(input)
219
220 def decode(self, input, errors='strict'):
221 if errors not in ('strict', 'replace', 'ignore'):
222 raise UnicodeError(f"Unsupported error handling: {errors}")
223 res = punycode_decode(input, errors)
224 return res, len(input)
225
226 class IncrementalEncoder(codecs.IncrementalEncoder):
227 def encode(self, input, final=False):
228 return punycode_encode(input)
229
230 class IncrementalDecoder(codecs.IncrementalDecoder):
231 def decode(self, input, final=False):
232 if self.errors not in ('strict', 'replace', 'ignore'):
233 raise UnicodeError(f"Unsupported error handling: {self.errors}")
234 return punycode_decode(input, self.errors)
235
236 class StreamWriter(Codec,codecs.StreamWriter):
237 pass
238
239 class StreamReader(Codec,codecs.StreamReader):
240 pass
241
242 ### encodings module API
243
244 def getregentry():
245 return codecs.CodecInfo(
246 name='punycode',
247 encode=Codec().encode,
248 decode=Codec().decode,
249 incrementalencoder=IncrementalEncoder,
250 incrementaldecoder=IncrementalDecoder,
251 streamwriter=StreamWriter,
252 streamreader=StreamReader,
253 )