| 1 |
# Implementation of marshal.loads() in pure Python |
| 2 |
|
| 3 |
import ast |
| 4 |
from typing import Any |
| 5 |
|
| 6 |
|
| 7 |
class Type: |
| 8 |
# Adapted from marshal.c |
| 9 |
NULL = ord('0') |
| 10 |
NONE = ord('N') |
| 11 |
FALSE = ord('F') |
| 12 |
TRUE = ord('T') |
| 13 |
STOPITER = ord('S') |
| 14 |
ELLIPSIS = ord('.') |
| 15 |
INT = ord('i') |
| 16 |
INT64 = ord('I') |
| 17 |
FLOAT = ord('f') |
| 18 |
BINARY_FLOAT = ord('g') |
| 19 |
COMPLEX = ord('x') |
| 20 |
BINARY_COMPLEX = ord('y') |
| 21 |
LONG = ord('l') |
| 22 |
STRING = ord('s') |
| 23 |
INTERNED = ord('t') |
| 24 |
REF = ord('r') |
| 25 |
TUPLE = ord('(') |
| 26 |
LIST = ord('[') |
| 27 |
DICT = ord('{') |
| 28 |
CODE = ord('c') |
| 29 |
UNICODE = ord('u') |
| 30 |
UNKNOWN = ord('?') |
| 31 |
SET = ord('<') |
| 32 |
FROZENSET = ord('>') |
| 33 |
ASCII = ord('a') |
| 34 |
ASCII_INTERNED = ord('A') |
| 35 |
SMALL_TUPLE = ord(')') |
| 36 |
SHORT_ASCII = ord('z') |
| 37 |
SHORT_ASCII_INTERNED = ord('Z') |
| 38 |
|
| 39 |
|
| 40 |
FLAG_REF = 0x80 # with a type, add obj to index |
| 41 |
|
| 42 |
NULL = object() # marker |
| 43 |
|
| 44 |
# Cell kinds |
| 45 |
CO_FAST_LOCAL = 0x20 |
| 46 |
CO_FAST_CELL = 0x40 |
| 47 |
CO_FAST_FREE = 0x80 |
| 48 |
|
| 49 |
|
| 50 |
class Code: |
| 51 |
def __init__(self, **kwds: Any): |
| 52 |
self.__dict__.update(kwds) |
| 53 |
|
| 54 |
def __repr__(self) -> str: |
| 55 |
return f"Code(**{self.__dict__})" |
| 56 |
|
| 57 |
co_localsplusnames: tuple[str, ...] |
| 58 |
co_localspluskinds: tuple[int, ...] |
| 59 |
|
| 60 |
def get_localsplus_names(self, select_kind: int) -> tuple[str, ...]: |
| 61 |
varnames: list[str] = [] |
| 62 |
for name, kind in zip(self.co_localsplusnames, |
| 63 |
self.co_localspluskinds): |
| 64 |
if kind & select_kind: |
| 65 |
varnames.append(name) |
| 66 |
return tuple(varnames) |
| 67 |
|
| 68 |
@property |
| 69 |
def co_varnames(self) -> tuple[str, ...]: |
| 70 |
return self.get_localsplus_names(CO_FAST_LOCAL) |
| 71 |
|
| 72 |
@property |
| 73 |
def co_cellvars(self) -> tuple[str, ...]: |
| 74 |
return self.get_localsplus_names(CO_FAST_CELL) |
| 75 |
|
| 76 |
@property |
| 77 |
def co_freevars(self) -> tuple[str, ...]: |
| 78 |
return self.get_localsplus_names(CO_FAST_FREE) |
| 79 |
|
| 80 |
@property |
| 81 |
def co_nlocals(self) -> int: |
| 82 |
return len(self.co_varnames) |
| 83 |
|
| 84 |
|
| 85 |
class Reader: |
| 86 |
# A fairly literal translation of the marshal reader. |
| 87 |
|
| 88 |
def __init__(self, data: bytes): |
| 89 |
self.data: bytes = data |
| 90 |
self.end: int = len(self.data) |
| 91 |
self.pos: int = 0 |
| 92 |
self.refs: list[Any] = [] |
| 93 |
self.level: int = 0 |
| 94 |
|
| 95 |
def r_string(self, n: int) -> bytes: |
| 96 |
assert 0 <= n <= self.end - self.pos |
| 97 |
buf = self.data[self.pos : self.pos + n] |
| 98 |
self.pos += n |
| 99 |
return buf |
| 100 |
|
| 101 |
def r_byte(self) -> int: |
| 102 |
buf = self.r_string(1) |
| 103 |
return buf[0] |
| 104 |
|
| 105 |
def r_short(self) -> int: |
| 106 |
buf = self.r_string(2) |
| 107 |
x = buf[0] |
| 108 |
x |= buf[1] << 8 |
| 109 |
x |= -(x & (1<<15)) # Sign-extend |
| 110 |
return x |
| 111 |
|
| 112 |
def r_long(self) -> int: |
| 113 |
buf = self.r_string(4) |
| 114 |
x = buf[0] |
| 115 |
x |= buf[1] << 8 |
| 116 |
x |= buf[2] << 16 |
| 117 |
x |= buf[3] << 24 |
| 118 |
x |= -(x & (1<<31)) # Sign-extend |
| 119 |
return x |
| 120 |
|
| 121 |
def r_long64(self) -> int: |
| 122 |
buf = self.r_string(8) |
| 123 |
x = buf[0] |
| 124 |
x |= buf[1] << 8 |
| 125 |
x |= buf[2] << 16 |
| 126 |
x |= buf[3] << 24 |
| 127 |
x |= buf[4] << 32 |
| 128 |
x |= buf[5] << 40 |
| 129 |
x |= buf[6] << 48 |
| 130 |
x |= buf[7] << 56 |
| 131 |
x |= -(x & (1<<63)) # Sign-extend |
| 132 |
return x |
| 133 |
|
| 134 |
def r_PyLong(self) -> int: |
| 135 |
n = self.r_long() |
| 136 |
size = abs(n) |
| 137 |
x = 0 |
| 138 |
# Pray this is right |
| 139 |
for i in range(size): |
| 140 |
x |= self.r_short() << i*15 |
| 141 |
if n < 0: |
| 142 |
x = -x |
| 143 |
return x |
| 144 |
|
| 145 |
def r_float_bin(self) -> float: |
| 146 |
buf = self.r_string(8) |
| 147 |
import struct # Lazy import to avoid breaking UNIX build |
| 148 |
return struct.unpack("d", buf)[0] # type: ignore[no-any-return] |
| 149 |
|
| 150 |
def r_float_str(self) -> float: |
| 151 |
n = self.r_byte() |
| 152 |
buf = self.r_string(n) |
| 153 |
return ast.literal_eval(buf.decode("ascii")) # type: ignore[no-any-return] |
| 154 |
|
| 155 |
def r_ref_reserve(self, flag: int) -> int: |
| 156 |
if flag: |
| 157 |
idx = len(self.refs) |
| 158 |
self.refs.append(None) |
| 159 |
return idx |
| 160 |
else: |
| 161 |
return 0 |
| 162 |
|
| 163 |
def r_ref_insert(self, obj: Any, idx: int, flag: int) -> Any: |
| 164 |
if flag: |
| 165 |
self.refs[idx] = obj |
| 166 |
return obj |
| 167 |
|
| 168 |
def r_ref(self, obj: Any, flag: int) -> Any: |
| 169 |
assert flag & FLAG_REF |
| 170 |
self.refs.append(obj) |
| 171 |
return obj |
| 172 |
|
| 173 |
def r_object(self) -> Any: |
| 174 |
old_level = self.level |
| 175 |
try: |
| 176 |
return self._r_object() |
| 177 |
finally: |
| 178 |
self.level = old_level |
| 179 |
|
| 180 |
def _r_object(self) -> Any: |
| 181 |
code = self.r_byte() |
| 182 |
flag = code & FLAG_REF |
| 183 |
type = code & ~FLAG_REF |
| 184 |
# print(" "*self.level + f"{code} {flag} {type} {chr(type)!r}") |
| 185 |
self.level += 1 |
| 186 |
|
| 187 |
def R_REF(obj: Any) -> Any: |
| 188 |
if flag: |
| 189 |
obj = self.r_ref(obj, flag) |
| 190 |
return obj |
| 191 |
|
| 192 |
if type == Type.NULL: |
| 193 |
return NULL |
| 194 |
elif type == Type.NONE: |
| 195 |
return None |
| 196 |
elif type == Type.ELLIPSIS: |
| 197 |
return Ellipsis |
| 198 |
elif type == Type.FALSE: |
| 199 |
return False |
| 200 |
elif type == Type.TRUE: |
| 201 |
return True |
| 202 |
elif type == Type.INT: |
| 203 |
return R_REF(self.r_long()) |
| 204 |
elif type == Type.INT64: |
| 205 |
return R_REF(self.r_long64()) |
| 206 |
elif type == Type.LONG: |
| 207 |
return R_REF(self.r_PyLong()) |
| 208 |
elif type == Type.FLOAT: |
| 209 |
return R_REF(self.r_float_str()) |
| 210 |
elif type == Type.BINARY_FLOAT: |
| 211 |
return R_REF(self.r_float_bin()) |
| 212 |
elif type == Type.COMPLEX: |
| 213 |
return R_REF(complex(self.r_float_str(), |
| 214 |
self.r_float_str())) |
| 215 |
elif type == Type.BINARY_COMPLEX: |
| 216 |
return R_REF(complex(self.r_float_bin(), |
| 217 |
self.r_float_bin())) |
| 218 |
elif type == Type.STRING: |
| 219 |
n = self.r_long() |
| 220 |
return R_REF(self.r_string(n)) |
| 221 |
elif type == Type.ASCII_INTERNED or type == Type.ASCII: |
| 222 |
n = self.r_long() |
| 223 |
return R_REF(self.r_string(n).decode("ascii")) |
| 224 |
elif type == Type.SHORT_ASCII_INTERNED or type == Type.SHORT_ASCII: |
| 225 |
n = self.r_byte() |
| 226 |
return R_REF(self.r_string(n).decode("ascii")) |
| 227 |
elif type == Type.INTERNED or type == Type.UNICODE: |
| 228 |
n = self.r_long() |
| 229 |
return R_REF(self.r_string(n).decode("utf8", "surrogatepass")) |
| 230 |
elif type == Type.SMALL_TUPLE: |
| 231 |
n = self.r_byte() |
| 232 |
idx = self.r_ref_reserve(flag) |
| 233 |
retval: Any = tuple(self.r_object() for _ in range(n)) |
| 234 |
self.r_ref_insert(retval, idx, flag) |
| 235 |
return retval |
| 236 |
elif type == Type.TUPLE: |
| 237 |
n = self.r_long() |
| 238 |
idx = self.r_ref_reserve(flag) |
| 239 |
retval = tuple(self.r_object() for _ in range(n)) |
| 240 |
self.r_ref_insert(retval, idx, flag) |
| 241 |
return retval |
| 242 |
elif type == Type.LIST: |
| 243 |
n = self.r_long() |
| 244 |
retval = R_REF([]) |
| 245 |
for _ in range(n): |
| 246 |
retval.append(self.r_object()) |
| 247 |
return retval |
| 248 |
elif type == Type.DICT: |
| 249 |
retval = R_REF({}) |
| 250 |
while True: |
| 251 |
key = self.r_object() |
| 252 |
if key == NULL: |
| 253 |
break |
| 254 |
val = self.r_object() |
| 255 |
retval[key] = val |
| 256 |
return retval |
| 257 |
elif type == Type.SET: |
| 258 |
n = self.r_long() |
| 259 |
retval = R_REF(set()) |
| 260 |
for _ in range(n): |
| 261 |
v = self.r_object() |
| 262 |
retval.add(v) |
| 263 |
return retval |
| 264 |
elif type == Type.FROZENSET: |
| 265 |
n = self.r_long() |
| 266 |
s: set[Any] = set() |
| 267 |
idx = self.r_ref_reserve(flag) |
| 268 |
for _ in range(n): |
| 269 |
v = self.r_object() |
| 270 |
s.add(v) |
| 271 |
retval = frozenset(s) |
| 272 |
self.r_ref_insert(retval, idx, flag) |
| 273 |
return retval |
| 274 |
elif type == Type.CODE: |
| 275 |
retval = R_REF(Code()) |
| 276 |
retval.co_argcount = self.r_long() |
| 277 |
retval.co_posonlyargcount = self.r_long() |
| 278 |
retval.co_kwonlyargcount = self.r_long() |
| 279 |
retval.co_stacksize = self.r_long() |
| 280 |
retval.co_flags = self.r_long() |
| 281 |
retval.co_code = self.r_object() |
| 282 |
retval.co_consts = self.r_object() |
| 283 |
retval.co_names = self.r_object() |
| 284 |
retval.co_localsplusnames = self.r_object() |
| 285 |
retval.co_localspluskinds = self.r_object() |
| 286 |
retval.co_filename = self.r_object() |
| 287 |
retval.co_name = self.r_object() |
| 288 |
retval.co_qualname = self.r_object() |
| 289 |
retval.co_firstlineno = self.r_long() |
| 290 |
retval.co_linetable = self.r_object() |
| 291 |
retval.co_exceptiontable = self.r_object() |
| 292 |
return retval |
| 293 |
elif type == Type.REF: |
| 294 |
n = self.r_long() |
| 295 |
retval = self.refs[n] |
| 296 |
assert retval is not None |
| 297 |
return retval |
| 298 |
else: |
| 299 |
breakpoint() |
| 300 |
raise AssertionError(f"Unknown type {type} {chr(type)!r}") |
| 301 |
|
| 302 |
|
| 303 |
def loads(data: bytes) -> Any: |
| 304 |
assert isinstance(data, bytes) |
| 305 |
r = Reader(data) |
| 306 |
return r.r_object() |
| 307 |
|
| 308 |
|
| 309 |
def main() -> None: |
| 310 |
# Test |
| 311 |
import marshal |
| 312 |
import pprint |
| 313 |
sample = {'foo': {(42, "bar", 3.14)}} |
| 314 |
data = marshal.dumps(sample) |
| 315 |
retval = loads(data) |
| 316 |
assert retval == sample, retval |
| 317 |
|
| 318 |
sample2 = main.__code__ |
| 319 |
data = marshal.dumps(sample2) |
| 320 |
retval = loads(data) |
| 321 |
assert isinstance(retval, Code), retval |
| 322 |
pprint.pprint(retval.__dict__) |
| 323 |
|
| 324 |
|
| 325 |
if __name__ == "__main__": |
| 326 |
main() |