| 1 |
""" Codec for the Punycode encoding, as specified in RFC 3492 |
| 2 |
|
| 3 |
Written by Martin v. Löwis. |
| 4 |
""" |
| 5 |
|
| 6 |
import codecs |
| 7 |
|
| 8 |
##################### Encoding ##################################### |
| 9 |
|
| 10 |
def segregate(str): |
| 11 |
"""3.1 Basic code point segregation""" |
| 12 |
base = bytearray() |
| 13 |
extended = set() |
| 14 |
for c in str: |
| 15 |
if ord(c) < 128: |
| 16 |
base.append(ord(c)) |
| 17 |
else: |
| 18 |
extended.add(c) |
| 19 |
extended = sorted(extended) |
| 20 |
return bytes(base), extended |
| 21 |
|
| 22 |
def selective_len(str, max): |
| 23 |
"""Return the length of str, considering only characters below max.""" |
| 24 |
res = 0 |
| 25 |
for c in str: |
| 26 |
if ord(c) < max: |
| 27 |
res += 1 |
| 28 |
return res |
| 29 |
|
| 30 |
def selective_find(str, char, index, pos): |
| 31 |
"""Return a pair (index, pos), indicating the next occurrence of |
| 32 |
char in str. index is the position of the character considering |
| 33 |
only ordinals up to and including char, and pos is the position in |
| 34 |
the full string. index/pos is the starting position in the full |
| 35 |
string.""" |
| 36 |
|
| 37 |
l = len(str) |
| 38 |
while 1: |
| 39 |
pos += 1 |
| 40 |
if pos == l: |
| 41 |
return (-1, -1) |
| 42 |
c = str[pos] |
| 43 |
if c == char: |
| 44 |
return index+1, pos |
| 45 |
elif c < char: |
| 46 |
index += 1 |
| 47 |
|
| 48 |
def insertion_unsort(str, extended): |
| 49 |
"""3.2 Insertion unsort coding""" |
| 50 |
oldchar = 0x80 |
| 51 |
result = [] |
| 52 |
oldindex = -1 |
| 53 |
for c in extended: |
| 54 |
index = pos = -1 |
| 55 |
char = ord(c) |
| 56 |
curlen = selective_len(str, char) |
| 57 |
delta = (curlen+1) * (char - oldchar) |
| 58 |
while 1: |
| 59 |
index,pos = selective_find(str,c,index,pos) |
| 60 |
if index == -1: |
| 61 |
break |
| 62 |
delta += index - oldindex |
| 63 |
result.append(delta-1) |
| 64 |
oldindex = index |
| 65 |
delta = 0 |
| 66 |
oldchar = char |
| 67 |
|
| 68 |
return result |
| 69 |
|
| 70 |
def T(j, bias): |
| 71 |
# Punycode parameters: tmin = 1, tmax = 26, base = 36 |
| 72 |
res = 36 * (j + 1) - bias |
| 73 |
if res < 1: return 1 |
| 74 |
if res > 26: return 26 |
| 75 |
return res |
| 76 |
|
| 77 |
digits = b"abcdefghijklmnopqrstuvwxyz0123456789" |
| 78 |
def generate_generalized_integer(N, bias): |
| 79 |
"""3.3 Generalized variable-length integers""" |
| 80 |
result = bytearray() |
| 81 |
j = 0 |
| 82 |
while 1: |
| 83 |
t = T(j, bias) |
| 84 |
if N < t: |
| 85 |
result.append(digits[N]) |
| 86 |
return bytes(result) |
| 87 |
result.append(digits[t + ((N - t) % (36 - t))]) |
| 88 |
N = (N - t) // (36 - t) |
| 89 |
j += 1 |
| 90 |
|
| 91 |
def adapt(delta, first, numchars): |
| 92 |
if first: |
| 93 |
delta //= 700 |
| 94 |
else: |
| 95 |
delta //= 2 |
| 96 |
delta += delta // numchars |
| 97 |
# ((base - tmin) * tmax) // 2 == 455 |
| 98 |
divisions = 0 |
| 99 |
while delta > 455: |
| 100 |
delta = delta // 35 # base - tmin |
| 101 |
divisions += 36 |
| 102 |
bias = divisions + (36 * delta // (delta + 38)) |
| 103 |
return bias |
| 104 |
|
| 105 |
|
| 106 |
def generate_integers(baselen, deltas): |
| 107 |
"""3.4 Bias adaptation""" |
| 108 |
# Punycode parameters: initial bias = 72, damp = 700, skew = 38 |
| 109 |
result = bytearray() |
| 110 |
bias = 72 |
| 111 |
for points, delta in enumerate(deltas): |
| 112 |
s = generate_generalized_integer(delta, bias) |
| 113 |
result.extend(s) |
| 114 |
bias = adapt(delta, points==0, baselen+points+1) |
| 115 |
return bytes(result) |
| 116 |
|
| 117 |
def punycode_encode(text): |
| 118 |
base, extended = segregate(text) |
| 119 |
deltas = insertion_unsort(text, extended) |
| 120 |
extended = generate_integers(len(base), deltas) |
| 121 |
if base: |
| 122 |
return base + b"-" + extended |
| 123 |
return extended |
| 124 |
|
| 125 |
##################### Decoding ##################################### |
| 126 |
|
| 127 |
def decode_generalized_number(extended, extpos, bias, errors): |
| 128 |
"""3.3 Generalized variable-length integers""" |
| 129 |
result = 0 |
| 130 |
w = 1 |
| 131 |
j = 0 |
| 132 |
while 1: |
| 133 |
try: |
| 134 |
char = extended[extpos] |
| 135 |
except IndexError: |
| 136 |
if errors == "strict": |
| 137 |
raise UnicodeDecodeError("punycode", extended, extpos, extpos+1, |
| 138 |
"incomplete punycode string") |
| 139 |
return extpos + 1, None |
| 140 |
extpos += 1 |
| 141 |
if 0x41 <= char <= 0x5A: # A-Z |
| 142 |
digit = char - 0x41 |
| 143 |
elif 0x30 <= char <= 0x39: |
| 144 |
digit = char - 22 # 0x30-26 |
| 145 |
elif errors == "strict": |
| 146 |
raise UnicodeDecodeError("punycode", extended, extpos-1, extpos, |
| 147 |
f"Invalid extended code point '{extended[extpos-1]}'") |
| 148 |
else: |
| 149 |
return extpos, None |
| 150 |
t = T(j, bias) |
| 151 |
result += digit * w |
| 152 |
if digit < t: |
| 153 |
return extpos, result |
| 154 |
w = w * (36 - t) |
| 155 |
j += 1 |
| 156 |
|
| 157 |
|
| 158 |
def insertion_sort(base, extended, errors): |
| 159 |
"""3.2 Insertion sort coding""" |
| 160 |
# This function raises UnicodeDecodeError with position in the extended. |
| 161 |
# Caller should add the offset. |
| 162 |
char = 0x80 |
| 163 |
pos = -1 |
| 164 |
bias = 72 |
| 165 |
extpos = 0 |
| 166 |
|
| 167 |
while extpos < len(extended): |
| 168 |
newpos, delta = decode_generalized_number(extended, extpos, |
| 169 |
bias, errors) |
| 170 |
if delta is None: |
| 171 |
# There was an error in decoding. We can't continue because |
| 172 |
# synchronization is lost. |
| 173 |
return base |
| 174 |
pos += delta+1 |
| 175 |
char += pos // (len(base) + 1) |
| 176 |
if char > 0x10FFFF: |
| 177 |
if errors == "strict": |
| 178 |
raise UnicodeDecodeError( |
| 179 |
"punycode", extended, pos-1, pos, |
| 180 |
f"Invalid character U+{char:x}") |
| 181 |
char = ord('?') |
| 182 |
pos = pos % (len(base) + 1) |
| 183 |
base = base[:pos] + chr(char) + base[pos:] |
| 184 |
bias = adapt(delta, (extpos == 0), len(base)) |
| 185 |
extpos = newpos |
| 186 |
return base |
| 187 |
|
| 188 |
def punycode_decode(text, errors): |
| 189 |
if isinstance(text, str): |
| 190 |
text = text.encode("ascii") |
| 191 |
if isinstance(text, memoryview): |
| 192 |
text = bytes(text) |
| 193 |
pos = text.rfind(b"-") |
| 194 |
if pos == -1: |
| 195 |
base = "" |
| 196 |
extended = text.upper() |
| 197 |
else: |
| 198 |
try: |
| 199 |
base = str(text[:pos], "ascii", errors) |
| 200 |
except UnicodeDecodeError as exc: |
| 201 |
raise UnicodeDecodeError("ascii", text, exc.start, exc.end, |
| 202 |
exc.reason) from None |
| 203 |
extended = text[pos+1:].upper() |
| 204 |
try: |
| 205 |
return insertion_sort(base, extended, errors) |
| 206 |
except UnicodeDecodeError as exc: |
| 207 |
offset = pos + 1 |
| 208 |
raise UnicodeDecodeError("punycode", text, |
| 209 |
offset+exc.start, offset+exc.end, |
| 210 |
exc.reason) from None |
| 211 |
|
| 212 |
### Codec APIs |
| 213 |
|
| 214 |
class Codec(codecs.Codec): |
| 215 |
|
| 216 |
def encode(self, input, errors='strict'): |
| 217 |
res = punycode_encode(input) |
| 218 |
return res, len(input) |
| 219 |
|
| 220 |
def decode(self, input, errors='strict'): |
| 221 |
if errors not in ('strict', 'replace', 'ignore'): |
| 222 |
raise UnicodeError(f"Unsupported error handling: {errors}") |
| 223 |
res = punycode_decode(input, errors) |
| 224 |
return res, len(input) |
| 225 |
|
| 226 |
class IncrementalEncoder(codecs.IncrementalEncoder): |
| 227 |
def encode(self, input, final=False): |
| 228 |
return punycode_encode(input) |
| 229 |
|
| 230 |
class IncrementalDecoder(codecs.IncrementalDecoder): |
| 231 |
def decode(self, input, final=False): |
| 232 |
if self.errors not in ('strict', 'replace', 'ignore'): |
| 233 |
raise UnicodeError(f"Unsupported error handling: {self.errors}") |
| 234 |
return punycode_decode(input, self.errors) |
| 235 |
|
| 236 |
class StreamWriter(Codec,codecs.StreamWriter): |
| 237 |
pass |
| 238 |
|
| 239 |
class StreamReader(Codec,codecs.StreamReader): |
| 240 |
pass |
| 241 |
|
| 242 |
### encodings module API |
| 243 |
|
| 244 |
def getregentry(): |
| 245 |
return codecs.CodecInfo( |
| 246 |
name='punycode', |
| 247 |
encode=Codec().encode, |
| 248 |
decode=Codec().decode, |
| 249 |
incrementalencoder=IncrementalEncoder, |
| 250 |
incrementaldecoder=IncrementalDecoder, |
| 251 |
streamwriter=StreamWriter, |
| 252 |
streamreader=StreamReader, |
| 253 |
) |