Debian Codegraph @

Tools/build/umarshal.py

raw
1 # Implementation of marshal.loads() in pure Python
2
3 import ast
4 from typing import Any
5
6
7 class Type:
8 # Adapted from marshal.c
9 NULL = ord('0')
10 NONE = ord('N')
11 FALSE = ord('F')
12 TRUE = ord('T')
13 STOPITER = ord('S')
14 ELLIPSIS = ord('.')
15 INT = ord('i')
16 INT64 = ord('I')
17 FLOAT = ord('f')
18 BINARY_FLOAT = ord('g')
19 COMPLEX = ord('x')
20 BINARY_COMPLEX = ord('y')
21 LONG = ord('l')
22 STRING = ord('s')
23 INTERNED = ord('t')
24 REF = ord('r')
25 TUPLE = ord('(')
26 LIST = ord('[')
27 DICT = ord('{')
28 CODE = ord('c')
29 UNICODE = ord('u')
30 UNKNOWN = ord('?')
31 SET = ord('<')
32 FROZENSET = ord('>')
33 ASCII = ord('a')
34 ASCII_INTERNED = ord('A')
35 SMALL_TUPLE = ord(')')
36 SHORT_ASCII = ord('z')
37 SHORT_ASCII_INTERNED = ord('Z')
38
39
40 FLAG_REF = 0x80 # with a type, add obj to index
41
42 NULL = object() # marker
43
44 # Cell kinds
45 CO_FAST_LOCAL = 0x20
46 CO_FAST_CELL = 0x40
47 CO_FAST_FREE = 0x80
48
49
50 class Code:
51 def __init__(self, **kwds: Any):
52 self.__dict__.update(kwds)
53
54 def __repr__(self) -> str:
55 return f"Code(**{self.__dict__})"
56
57 co_localsplusnames: tuple[str, ...]
58 co_localspluskinds: tuple[int, ...]
59
60 def get_localsplus_names(self, select_kind: int) -> tuple[str, ...]:
61 varnames: list[str] = []
62 for name, kind in zip(self.co_localsplusnames,
63 self.co_localspluskinds):
64 if kind & select_kind:
65 varnames.append(name)
66 return tuple(varnames)
67
68 @property
69 def co_varnames(self) -> tuple[str, ...]:
70 return self.get_localsplus_names(CO_FAST_LOCAL)
71
72 @property
73 def co_cellvars(self) -> tuple[str, ...]:
74 return self.get_localsplus_names(CO_FAST_CELL)
75
76 @property
77 def co_freevars(self) -> tuple[str, ...]:
78 return self.get_localsplus_names(CO_FAST_FREE)
79
80 @property
81 def co_nlocals(self) -> int:
82 return len(self.co_varnames)
83
84
85 class Reader:
86 # A fairly literal translation of the marshal reader.
87
88 def __init__(self, data: bytes):
89 self.data: bytes = data
90 self.end: int = len(self.data)
91 self.pos: int = 0
92 self.refs: list[Any] = []
93 self.level: int = 0
94
95 def r_string(self, n: int) -> bytes:
96 assert 0 <= n <= self.end - self.pos
97 buf = self.data[self.pos : self.pos + n]
98 self.pos += n
99 return buf
100
101 def r_byte(self) -> int:
102 buf = self.r_string(1)
103 return buf[0]
104
105 def r_short(self) -> int:
106 buf = self.r_string(2)
107 x = buf[0]
108 x |= buf[1] << 8
109 x |= -(x & (1<<15)) # Sign-extend
110 return x
111
112 def r_long(self) -> int:
113 buf = self.r_string(4)
114 x = buf[0]
115 x |= buf[1] << 8
116 x |= buf[2] << 16
117 x |= buf[3] << 24
118 x |= -(x & (1<<31)) # Sign-extend
119 return x
120
121 def r_long64(self) -> int:
122 buf = self.r_string(8)
123 x = buf[0]
124 x |= buf[1] << 8
125 x |= buf[2] << 16
126 x |= buf[3] << 24
127 x |= buf[4] << 32
128 x |= buf[5] << 40
129 x |= buf[6] << 48
130 x |= buf[7] << 56
131 x |= -(x & (1<<63)) # Sign-extend
132 return x
133
134 def r_PyLong(self) -> int:
135 n = self.r_long()
136 size = abs(n)
137 x = 0
138 # Pray this is right
139 for i in range(size):
140 x |= self.r_short() << i*15
141 if n < 0:
142 x = -x
143 return x
144
145 def r_float_bin(self) -> float:
146 buf = self.r_string(8)
147 import struct # Lazy import to avoid breaking UNIX build
148 return struct.unpack("d", buf)[0] # type: ignore[no-any-return]
149
150 def r_float_str(self) -> float:
151 n = self.r_byte()
152 buf = self.r_string(n)
153 return ast.literal_eval(buf.decode("ascii")) # type: ignore[no-any-return]
154
155 def r_ref_reserve(self, flag: int) -> int:
156 if flag:
157 idx = len(self.refs)
158 self.refs.append(None)
159 return idx
160 else:
161 return 0
162
163 def r_ref_insert(self, obj: Any, idx: int, flag: int) -> Any:
164 if flag:
165 self.refs[idx] = obj
166 return obj
167
168 def r_ref(self, obj: Any, flag: int) -> Any:
169 assert flag & FLAG_REF
170 self.refs.append(obj)
171 return obj
172
173 def r_object(self) -> Any:
174 old_level = self.level
175 try:
176 return self._r_object()
177 finally:
178 self.level = old_level
179
180 def _r_object(self) -> Any:
181 code = self.r_byte()
182 flag = code & FLAG_REF
183 type = code & ~FLAG_REF
184 # print(" "*self.level + f"{code} {flag} {type} {chr(type)!r}")
185 self.level += 1
186
187 def R_REF(obj: Any) -> Any:
188 if flag:
189 obj = self.r_ref(obj, flag)
190 return obj
191
192 if type == Type.NULL:
193 return NULL
194 elif type == Type.NONE:
195 return None
196 elif type == Type.ELLIPSIS:
197 return Ellipsis
198 elif type == Type.FALSE:
199 return False
200 elif type == Type.TRUE:
201 return True
202 elif type == Type.INT:
203 return R_REF(self.r_long())
204 elif type == Type.INT64:
205 return R_REF(self.r_long64())
206 elif type == Type.LONG:
207 return R_REF(self.r_PyLong())
208 elif type == Type.FLOAT:
209 return R_REF(self.r_float_str())
210 elif type == Type.BINARY_FLOAT:
211 return R_REF(self.r_float_bin())
212 elif type == Type.COMPLEX:
213 return R_REF(complex(self.r_float_str(),
214 self.r_float_str()))
215 elif type == Type.BINARY_COMPLEX:
216 return R_REF(complex(self.r_float_bin(),
217 self.r_float_bin()))
218 elif type == Type.STRING:
219 n = self.r_long()
220 return R_REF(self.r_string(n))
221 elif type == Type.ASCII_INTERNED or type == Type.ASCII:
222 n = self.r_long()
223 return R_REF(self.r_string(n).decode("ascii"))
224 elif type == Type.SHORT_ASCII_INTERNED or type == Type.SHORT_ASCII:
225 n = self.r_byte()
226 return R_REF(self.r_string(n).decode("ascii"))
227 elif type == Type.INTERNED or type == Type.UNICODE:
228 n = self.r_long()
229 return R_REF(self.r_string(n).decode("utf8", "surrogatepass"))
230 elif type == Type.SMALL_TUPLE:
231 n = self.r_byte()
232 idx = self.r_ref_reserve(flag)
233 retval: Any = tuple(self.r_object() for _ in range(n))
234 self.r_ref_insert(retval, idx, flag)
235 return retval
236 elif type == Type.TUPLE:
237 n = self.r_long()
238 idx = self.r_ref_reserve(flag)
239 retval = tuple(self.r_object() for _ in range(n))
240 self.r_ref_insert(retval, idx, flag)
241 return retval
242 elif type == Type.LIST:
243 n = self.r_long()
244 retval = R_REF([])
245 for _ in range(n):
246 retval.append(self.r_object())
247 return retval
248 elif type == Type.DICT:
249 retval = R_REF({})
250 while True:
251 key = self.r_object()
252 if key == NULL:
253 break
254 val = self.r_object()
255 retval[key] = val
256 return retval
257 elif type == Type.SET:
258 n = self.r_long()
259 retval = R_REF(set())
260 for _ in range(n):
261 v = self.r_object()
262 retval.add(v)
263 return retval
264 elif type == Type.FROZENSET:
265 n = self.r_long()
266 s: set[Any] = set()
267 idx = self.r_ref_reserve(flag)
268 for _ in range(n):
269 v = self.r_object()
270 s.add(v)
271 retval = frozenset(s)
272 self.r_ref_insert(retval, idx, flag)
273 return retval
274 elif type == Type.CODE:
275 retval = R_REF(Code())
276 retval.co_argcount = self.r_long()
277 retval.co_posonlyargcount = self.r_long()
278 retval.co_kwonlyargcount = self.r_long()
279 retval.co_stacksize = self.r_long()
280 retval.co_flags = self.r_long()
281 retval.co_code = self.r_object()
282 retval.co_consts = self.r_object()
283 retval.co_names = self.r_object()
284 retval.co_localsplusnames = self.r_object()
285 retval.co_localspluskinds = self.r_object()
286 retval.co_filename = self.r_object()
287 retval.co_name = self.r_object()
288 retval.co_qualname = self.r_object()
289 retval.co_firstlineno = self.r_long()
290 retval.co_linetable = self.r_object()
291 retval.co_exceptiontable = self.r_object()
292 return retval
293 elif type == Type.REF:
294 n = self.r_long()
295 retval = self.refs[n]
296 assert retval is not None
297 return retval
298 else:
299 breakpoint()
300 raise AssertionError(f"Unknown type {type} {chr(type)!r}")
301
302
303 def loads(data: bytes) -> Any:
304 assert isinstance(data, bytes)
305 r = Reader(data)
306 return r.r_object()
307
308
309 def main() -> None:
310 # Test
311 import marshal
312 import pprint
313 sample = {'foo': {(42, "bar", 3.14)}}
314 data = marshal.dumps(sample)
315 retval = loads(data)
316 assert retval == sample, retval
317
318 sample2 = main.__code__
319 data = marshal.dumps(sample2)
320 retval = loads(data)
321 assert isinstance(retval, Code), retval
322 pprint.pprint(retval.__dict__)
323
324
325 if __name__ == "__main__":
326 main()