Debian Codegraph @

deps/curl/tests/python_dependencies/impacket/structure.py

raw
1 from __future__ import print_function
2 # Copyright (c) 2003-2016 CORE Security Technologies
3 #
4 # This software is provided under under a slightly modified version
5 # of the Apache Software License. See the accompanying LICENSE file
6 # for more information.
7 #
8
9 from struct import pack, unpack, calcsize
10
11 class Structure:
12 """ sublcasses can define commonHdr and/or structure.
13 each of them is an tuple of either two: (fieldName, format) or three: (fieldName, ':', class) fields.
14 [it can't be a dictionary, because order is important]
15
16 where format specifies how the data in the field will be converted to/from bytes (string)
17 class is the class to use when unpacking ':' fields.
18
19 each field can only contain one value (or an array of values for *)
20 i.e. struct.pack('Hl',1,2) is valid, but format specifier 'Hl' is not (you must use 2 dfferent fields)
21
22 format specifiers:
23 specifiers from module pack can be used with the same format
24 see struct.__doc__ (pack/unpack is finally called)
25 x [padding byte]
26 c [character]
27 b [signed byte]
28 B [unsigned byte]
29 h [signed short]
30 H [unsigned short]
31 l [signed long]
32 L [unsigned long]
33 i [signed integer]
34 I [unsigned integer]
35 q [signed long long (quad)]
36 Q [unsigned long long (quad)]
37 s [string (array of chars), must be preceded with length in format specifier, padded with zeros]
38 p [pascal string (includes byte count), must be preceded with length in format specifier, padded with zeros]
39 f [float]
40 d [double]
41 = [native byte ordering, size and alignment]
42 @ [native byte ordering, standard size and alignment]
43 ! [network byte ordering]
44 < [little endian]
45 > [big endian]
46
47 usual printf like specifiers can be used (if started with %)
48 [not recommeneded, there is no why to unpack this]
49
50 %08x will output an 8 bytes hex
51 %s will output a string
52 %s\\x00 will output a NUL terminated string
53 %d%d will output 2 decimal digits (against the very same specification of Structure)
54 ...
55
56 some additional format specifiers:
57 : just copy the bytes from the field into the output string (input may be string, other structure, or anything responding to __str__()) (for unpacking, all what's left is returned)
58 z same as :, but adds a NUL byte at the end (asciiz) (for unpacking the first NUL byte is used as terminator) [asciiz string]
59 u same as z, but adds two NUL bytes at the end (after padding to an even size with NULs). (same for unpacking) [unicode string]
60 w DCE-RPC/NDR string (it's a macro for [ '<L=(len(field)+1)/2','"\\x00\\x00\\x00\\x00','<L=(len(field)+1)/2',':' ]
61 ?-field length of field named 'field', formated as specified with ? ('?' may be '!H' for example). The input value overrides the real length
62 ?1*?2 array of elements. Each formated as '?2', the number of elements in the array is stored as specified by '?1' (?1 is optional, or can also be a constant (number), for unpacking)
63 'xxxx literal xxxx (field's value doesn't change the output. quotes must not be closed or escaped)
64 "xxxx literal xxxx (field's value doesn't change the output. quotes must not be closed or escaped)
65 _ will not pack the field. Accepts a third argument, which is an unpack code. See _Test_UnpackCode for an example
66 ?=packcode will evaluate packcode in the context of the structure, and pack the result as specified by ?. Unpacking is made plain
67 ?&fieldname "Address of field fieldname".
68 For packing it will simply pack the id() of fieldname. Or use 0 if fieldname doesn't exists.
69 For unpacking, it's used to know weather fieldname has to be unpacked or not, i.e. by adding a & field you turn another field (fieldname) in an optional field.
70
71 """
72 commonHdr = ()
73 structure = ()
74 debug = 0
75
76 def __init__(self, data = None, alignment = 0):
77 if not hasattr(self, 'alignment'):
78 self.alignment = alignment
79
80 self.fields = {}
81 self.rawData = data
82 if data is not None:
83 self.fromString(data)
84 else:
85 self.data = None
86
87 @classmethod
88 def fromFile(self, file):
89 answer = self()
90 answer.fromString(file.read(len(answer)))
91 return answer
92
93 def setAlignment(self, alignment):
94 self.alignment = alignment
95
96 def setData(self, data):
97 self.data = data
98
99 def packField(self, fieldName, format = None):
100 if self.debug:
101 print("packField( %s | %s )" % (fieldName, format))
102
103 if format is None:
104 format = self.formatForField(fieldName)
105
106 if fieldName in self.fields:
107 ans = self.pack(format, self.fields[fieldName], field = fieldName)
108 else:
109 ans = self.pack(format, None, field = fieldName)
110
111 if self.debug:
112 print("\tanswer %r" % ans)
113
114 return ans
115
116 def getData(self):
117 if self.data is not None:
118 return self.data
119 data = ''
120 for field in self.commonHdr+self.structure:
121 try:
122 data += self.packField(field[0], field[1])
123 except Exception as e:
124 if field[0] in self.fields:
125 e.args += ("When packing field '%s | %s | %r' in %s" % (field[0], field[1], self[field[0]], self.__class__),)
126 else:
127 e.args += ("When packing field '%s | %s' in %s" % (field[0], field[1], self.__class__),)
128 raise
129 if self.alignment:
130 if len(data) % self.alignment:
131 data += ('\x00'*self.alignment)[:-(len(data) % self.alignment)]
132
133 #if len(data) % self.alignment: data += ('\x00'*self.alignment)[:-(len(data) % self.alignment)]
134 return data
135
136 def fromString(self, data):
137 self.rawData = data
138 for field in self.commonHdr+self.structure:
139 if self.debug:
140 print("fromString( %s | %s | %r )" % (field[0], field[1], data))
141 size = self.calcUnpackSize(field[1], data, field[0])
142 if self.debug:
143 print(" size = %d" % size)
144 dataClassOrCode = str
145 if len(field) > 2:
146 dataClassOrCode = field[2]
147 try:
148 self[field[0]] = self.unpack(field[1], data[:size], dataClassOrCode = dataClassOrCode, field = field[0])
149 except Exception as e:
150 e.args += ("When unpacking field '%s | %s | %r[:%d]'" % (field[0], field[1], data, size),)
151 raise
152
153 size = self.calcPackSize(field[1], self[field[0]], field[0])
154 if self.alignment and size % self.alignment:
155 size += self.alignment - (size % self.alignment)
156 data = data[size:]
157
158 return self
159
160 def __setitem__(self, key, value):
161 self.fields[key] = value
162 self.data = None # force recompute
163
164 def __getitem__(self, key):
165 return self.fields[key]
166
167 def __delitem__(self, key):
168 del self.fields[key]
169
170 def __str__(self):
171 return self.getData()
172
173 def __len__(self):
174 # XXX: improve
175 return len(self.getData())
176
177 def pack(self, format, data, field = None):
178 if self.debug:
179 print(" pack( %s | %r | %s)" % (format, data, field))
180
181 if field:
182 addressField = self.findAddressFieldFor(field)
183 if (addressField is not None) and (data is None):
184 return ''
185
186 # void specifier
187 if format[:1] == '_':
188 return ''
189
190 # quote specifier
191 if format[:1] == "'" or format[:1] == '"':
192 return format[1:]
193
194 # code specifier
195 two = format.split('=')
196 if len(two) >= 2:
197 try:
198 return self.pack(two[0], data)
199 except:
200 fields = {'self':self}
201 fields.update(self.fields)
202 return self.pack(two[0], eval(two[1], {}, fields))
203
204 # address specifier
205 two = format.split('&')
206 if len(two) == 2:
207 try:
208 return self.pack(two[0], data)
209 except:
210 if (two[1] in self.fields) and (self[two[1]] is not None):
211 return self.pack(two[0], id(self[two[1]]) & ((1<<(calcsize(two[0])*8))-1) )
212 else:
213 return self.pack(two[0], 0)
214
215 # length specifier
216 two = format.split('-')
217 if len(two) == 2:
218 try:
219 return self.pack(two[0],data)
220 except:
221 return self.pack(two[0], self.calcPackFieldSize(two[1]))
222
223 # array specifier
224 two = format.split('*')
225 if len(two) == 2:
226 answer = ''
227 for each in data:
228 answer += self.pack(two[1], each)
229 if two[0]:
230 if two[0].isdigit():
231 if int(two[0]) != len(data):
232 raise Exception("Array field has a constant size, and it doesn't match the actual value")
233 else:
234 return self.pack(two[0], len(data))+answer
235 return answer
236
237 # "printf" string specifier
238 if format[:1] == '%':
239 # format string like specifier
240 return format % data
241
242 # asciiz specifier
243 if format[:1] == 'z':
244 return str(data)+'\0'
245
246 # unicode specifier
247 if format[:1] == 'u':
248 return str(data)+'\0\0' + (len(data) & 1 and '\0' or '')
249
250 # DCE-RPC/NDR string specifier
251 if format[:1] == 'w':
252 if len(data) == 0:
253 data = '\0\0'
254 elif len(data) % 2:
255 data += '\0'
256 l = pack('<L', len(data)/2)
257 return '%s\0\0\0\0%s%s' % (l,l,data)
258
259 if data is None:
260 raise Exception("Trying to pack None")
261
262 # literal specifier
263 if format[:1] == ':':
264 return str(data)
265
266 # struct like specifier
267 return pack(format, data)
268
269 def unpack(self, format, data, dataClassOrCode = str, field = None):
270 if self.debug:
271 print(" unpack( %s | %r )" % (format, data))
272
273 if field:
274 addressField = self.findAddressFieldFor(field)
275 if addressField is not None:
276 if not self[addressField]:
277 return
278
279 # void specifier
280 if format[:1] == '_':
281 if dataClassOrCode != str:
282 fields = {'self':self, 'inputDataLeft':data}
283 fields.update(self.fields)
284 return eval(dataClassOrCode, {}, fields)
285 else:
286 return None
287
288 # quote specifier
289 if format[:1] == "'" or format[:1] == '"':
290 answer = format[1:]
291 if answer != data:
292 raise Exception("Unpacked data doesn't match constant value '%r' should be '%r'" % (data, answer))
293 return answer
294
295 # address specifier
296 two = format.split('&')
297 if len(two) == 2:
298 return self.unpack(two[0],data)
299
300 # code specifier
301 two = format.split('=')
302 if len(two) >= 2:
303 return self.unpack(two[0],data)
304
305 # length specifier
306 two = format.split('-')
307 if len(two) == 2:
308 return self.unpack(two[0],data)
309
310 # array specifier
311 two = format.split('*')
312 if len(two) == 2:
313 answer = []
314 sofar = 0
315 if two[0].isdigit():
316 number = int(two[0])
317 elif two[0]:
318 sofar += self.calcUnpackSize(two[0], data)
319 number = self.unpack(two[0], data[:sofar])
320 else:
321 number = -1
322
323 while number and sofar < len(data):
324 nsofar = sofar + self.calcUnpackSize(two[1],data[sofar:])
325 answer.append(self.unpack(two[1], data[sofar:nsofar], dataClassOrCode))
326 number -= 1
327 sofar = nsofar
328 return answer
329
330 # "printf" string specifier
331 if format[:1] == '%':
332 # format string like specifier
333 return format % data
334
335 # asciiz specifier
336 if format == 'z':
337 if data[-1] != '\x00':
338 raise Exception("%s 'z' field is not NUL terminated: %r" % (field, data))
339 return data[:-1] # remove trailing NUL
340
341 # unicode specifier
342 if format == 'u':
343 if data[-2:] != '\x00\x00':
344 raise Exception("%s 'u' field is not NUL-NUL terminated: %r" % (field, data))
345 return data[:-2] # remove trailing NUL
346
347 # DCE-RPC/NDR string specifier
348 if format == 'w':
349 l = unpack('<L', data[:4])[0]
350 return data[12:12+l*2]
351
352 # literal specifier
353 if format == ':':
354 return dataClassOrCode(data)
355
356 # struct like specifier
357 return unpack(format, data)[0]
358
359 def calcPackSize(self, format, data, field = None):
360 # # print " calcPackSize %s:%r" % (format, data)
361 if field:
362 addressField = self.findAddressFieldFor(field)
363 if addressField is not None:
364 if not self[addressField]:
365 return 0
366
367 # void specifier
368 if format[:1] == '_':
369 return 0
370
371 # quote specifier
372 if format[:1] == "'" or format[:1] == '"':
373 return len(format)-1
374
375 # address specifier
376 two = format.split('&')
377 if len(two) == 2:
378 return self.calcPackSize(two[0], data)
379
380 # code specifier
381 two = format.split('=')
382 if len(two) >= 2:
383 return self.calcPackSize(two[0], data)
384
385 # length specifier
386 two = format.split('-')
387 if len(two) == 2:
388 return self.calcPackSize(two[0], data)
389
390 # array specifier
391 two = format.split('*')
392 if len(two) == 2:
393 answer = 0
394 if two[0].isdigit():
395 if int(two[0]) != len(data):
396 raise Exception("Array field has a constant size, and it doesn't match the actual value")
397 elif two[0]:
398 answer += self.calcPackSize(two[0], len(data))
399
400 for each in data:
401 answer += self.calcPackSize(two[1], each)
402 return answer
403
404 # "printf" string specifier
405 if format[:1] == '%':
406 # format string like specifier
407 return len(format % data)
408
409 # asciiz specifier
410 if format[:1] == 'z':
411 return len(data)+1
412
413 # asciiz specifier
414 if format[:1] == 'u':
415 l = len(data)
416 return l + (l & 1 and 3 or 2)
417
418 # DCE-RPC/NDR string specifier
419 if format[:1] == 'w':
420 l = len(data)
421 return 12+l+l % 2
422
423 # literal specifier
424 if format[:1] == ':':
425 return len(data)
426
427 # struct like specifier
428 return calcsize(format)
429
430 def calcUnpackSize(self, format, data, field = None):
431 if self.debug:
432 print(" calcUnpackSize( %s | %s | %r)" % (field, format, data))
433
434 # void specifier
435 if format[:1] == '_':
436 return 0
437
438 addressField = self.findAddressFieldFor(field)
439 if addressField is not None:
440 if not self[addressField]:
441 return 0
442
443 try:
444 lengthField = self.findLengthFieldFor(field)
445 return self[lengthField]
446 except:
447 pass
448
449 # XXX: Try to match to actual values, raise if no match
450
451 # quote specifier
452 if format[:1] == "'" or format[:1] == '"':
453 return len(format)-1
454
455 # address specifier
456 two = format.split('&')
457 if len(two) == 2:
458 return self.calcUnpackSize(two[0], data)
459
460 # code specifier
461 two = format.split('=')
462 if len(two) >= 2:
463 return self.calcUnpackSize(two[0], data)
464
465 # length specifier
466 two = format.split('-')
467 if len(two) == 2:
468 return self.calcUnpackSize(two[0], data)
469
470 # array specifier
471 two = format.split('*')
472 if len(two) == 2:
473 answer = 0
474 if two[0]:
475 if two[0].isdigit():
476 number = int(two[0])
477 else:
478 answer += self.calcUnpackSize(two[0], data)
479 number = self.unpack(two[0], data[:answer])
480
481 while number:
482 number -= 1
483 answer += self.calcUnpackSize(two[1], data[answer:])
484 else:
485 while answer < len(data):
486 answer += self.calcUnpackSize(two[1], data[answer:])
487 return answer
488
489 # "printf" string specifier
490 if format[:1] == '%':
491 raise Exception("Can't guess the size of a printf like specifier for unpacking")
492
493 # asciiz specifier
494 if format[:1] == 'z':
495 return data.index('\x00')+1
496
497 # asciiz specifier
498 if format[:1] == 'u':
499 l = data.index('\x00\x00')
500 return l + (l & 1 and 3 or 2)
501
502 # DCE-RPC/NDR string specifier
503 if format[:1] == 'w':
504 l = unpack('<L', data[:4])[0]
505 return 12+l*2
506
507 # literal specifier
508 if format[:1] == ':':
509 return len(data)
510
511 # struct like specifier
512 return calcsize(format)
513
514 def calcPackFieldSize(self, fieldName, format = None):
515 if format is None:
516 format = self.formatForField(fieldName)
517
518 return self.calcPackSize(format, self[fieldName])
519
520 def formatForField(self, fieldName):
521 for field in self.commonHdr+self.structure:
522 if field[0] == fieldName:
523 return field[1]
524 raise Exception("Field %s not found" % fieldName)
525
526 def findAddressFieldFor(self, fieldName):
527 descriptor = '&%s' % fieldName
528 l = len(descriptor)
529 for field in self.commonHdr+self.structure:
530 if field[1][-l:] == descriptor:
531 return field[0]
532 return None
533
534 def findLengthFieldFor(self, fieldName):
535 descriptor = '-%s' % fieldName
536 l = len(descriptor)
537 for field in self.commonHdr+self.structure:
538 if field[1][-l:] == descriptor:
539 return field[0]
540 return None
541
542 def zeroValue(self, format):
543 two = format.split('*')
544 if len(two) == 2:
545 if two[0].isdigit():
546 return (self.zeroValue(two[1]),)*int(two[0])
547
548 if not format.find('*') == -1: return ()
549 if 's' in format: return ''
550 if format in ['z',':','u']: return ''
551 if format == 'w': return '\x00\x00'
552
553 return 0
554
555 def clear(self):
556 for field in self.commonHdr + self.structure:
557 self[field[0]] = self.zeroValue(field[1])
558
559 def dump(self, msg = None, indent = 0):
560 if msg is None: msg = self.__class__.__name__
561 ind = ' '*indent
562 print("\n%s" % msg)
563 fixedFields = []
564 for field in self.commonHdr+self.structure:
565 i = field[0]
566 if i in self.fields:
567 fixedFields.append(i)
568 if isinstance(self[i], Structure):
569 self[i].dump('%s%s:{' % (ind,i), indent = indent + 4)
570 print("%s}" % ind)
571 else:
572 print("%s%s: {%r}" % (ind,i,self[i]))
573 # Do we have remaining fields not defined in the structures? let's
574 # print them
575 remainingFields = list(set(self.fields) - set(fixedFields))
576 for i in remainingFields:
577 if isinstance(self[i], Structure):
578 self[i].dump('%s%s:{' % (ind,i), indent = indent + 4)
579 print("%s}" % ind)
580 else:
581 print("%s%s: {%r}" % (ind,i,self[i]))
582
583
584 class _StructureTest:
585 alignment = 0
586 def create(self,data = None):
587 if data is not None:
588 return self.theClass(data, alignment = self.alignment)
589 else:
590 return self.theClass(alignment = self.alignment)
591
592 def run(self):
593 print()
594 print("-"*70)
595 testName = self.__class__.__name__
596 print("starting test: %s....." % testName)
597 a = self.create()
598 self.populate(a)
599 a.dump("packing.....")
600 a_str = str(a)
601 print("packed: %r" % a_str)
602 print("unpacking.....")
603 b = self.create(a_str)
604 b.dump("unpacked.....")
605 print("repacking.....")
606 b_str = str(b)
607 if b_str != a_str:
608 print("ERROR: original packed and repacked don't match")
609 print("packed: %r" % b_str)
610
611 class _Test_simple(_StructureTest):
612 class theClass(Structure):
613 commonHdr = ()
614 structure = (
615 ('int1', '!L'),
616 ('len1','!L-z1'),
617 ('arr1','B*<L'),
618 ('z1', 'z'),
619 ('u1','u'),
620 ('', '"COCA'),
621 ('len2','!H-:1'),
622 ('', '"COCA'),
623 (':1', ':'),
624 ('int3','>L'),
625 ('code1','>L=len(arr1)*2+0x1000'),
626 )
627
628 def populate(self, a):
629 a['default'] = 'hola'
630 a['int1'] = 0x3131
631 a['int3'] = 0x45444342
632 a['z1'] = 'hola'
633 a['u1'] = 'hola'.encode('utf_16_le')
634 a[':1'] = ':1234:'
635 a['arr1'] = (0x12341234,0x88990077,0x41414141)
636 # a['len1'] = 0x42424242
637
638 class _Test_fixedLength(_Test_simple):
639 def populate(self, a):
640 _Test_simple.populate(self, a)
641 a['len1'] = 0x42424242
642
643 class _Test_simple_aligned4(_Test_simple):
644 alignment = 4
645
646 class _Test_nested(_StructureTest):
647 class theClass(Structure):
648 class _Inner(Structure):
649 structure = (('data', 'z'),)
650
651 structure = (
652 ('nest1', ':', _Inner),
653 ('nest2', ':', _Inner),
654 ('int', '<L'),
655 )
656
657 def populate(self, a):
658 a['nest1'] = _Test_nested.theClass._Inner()
659 a['nest2'] = _Test_nested.theClass._Inner()
660 a['nest1']['data'] = 'hola manola'
661 a['nest2']['data'] = 'chau loco'
662 a['int'] = 0x12345678
663
664 class _Test_Optional(_StructureTest):
665 class theClass(Structure):
666 structure = (
667 ('pName','<L&Name'),
668 ('pList','<L&List'),
669 ('Name','w'),
670 ('List','<H*<L'),
671 )
672
673 def populate(self, a):
674 a['Name'] = 'Optional test'
675 a['List'] = (1,2,3,4)
676
677 class _Test_Optional_sparse(_Test_Optional):
678 def populate(self, a):
679 _Test_Optional.populate(self, a)
680 del a['Name']
681
682 class _Test_AsciiZArray(_StructureTest):
683 class theClass(Structure):
684 structure = (
685 ('head','<L'),
686 ('array','B*z'),
687 ('tail','<L'),
688 )
689
690 def populate(self, a):
691 a['head'] = 0x1234
692 a['tail'] = 0xabcd
693 a['array'] = ('hola','manola','te traje')
694
695 class _Test_UnpackCode(_StructureTest):
696 class theClass(Structure):
697 structure = (
698 ('leni','<L=len(uno)*2'),
699 ('cuchi','_-uno','leni/2'),
700 ('uno',':'),
701 ('dos',':'),
702 )
703
704 def populate(self, a):
705 a['uno'] = 'soy un loco!'
706 a['dos'] = 'que haces fiera'
707
708 class _Test_AAA(_StructureTest):
709 class theClass(Structure):
710 commonHdr = ()
711 structure = (
712 ('iv', '!L=((init_vector & 0xFFFFFF) << 8) | ((pad & 0x3f) << 2) | (keyid & 3)'),
713 ('init_vector', '_','(iv >> 8)'),
714 ('pad', '_','((iv >>2) & 0x3F)'),
715 ('keyid', '_','( iv & 0x03 )'),
716 ('dataLen', '_-data', 'len(inputDataLeft)-4'),
717 ('data',':'),
718 ('icv','>L'),
719 )
720
721 def populate(self, a):
722 a['init_vector']=0x01020304
723 #a['pad']=int('01010101',2)
724 a['pad']=int('010101',2)
725 a['keyid']=0x07
726 a['data']="\xA0\xA1\xA2\xA3\xA4\xA5\xA6\xA7\xA8\xA9"
727 a['icv'] = 0x05060708
728 #a['iv'] = 0x01020304
729
730 if __name__ == '__main__':
731 _Test_simple().run()
732
733 try:
734 _Test_fixedLength().run()
735 except:
736 print("cannot repack because length is bogus")
737
738 _Test_simple_aligned4().run()
739 _Test_nested().run()
740 _Test_Optional().run()
741 _Test_Optional_sparse().run()
742 _Test_AsciiZArray().run()
743 _Test_UnpackCode().run()
744 _Test_AAA().run()