Debian Codegraph @

rts/python/values.py

raw
1 # Start of values.py.
2
3 # Hacky parser/reader/writer for values written in Futhark syntax.
4 # Used for reading stdin when compiling standalone programs with the
5 # Python code generator.
6
7 import numpy as np
8 import string
9 import struct
10 import sys
11
12
13 class ReaderInput:
14 def __init__(self, f):
15 self.f = f
16 self.lookahead_buffer = []
17
18 def get_char(self):
19 if len(self.lookahead_buffer) == 0:
20 return self.f.read(1)
21 else:
22 c = self.lookahead_buffer[0]
23 self.lookahead_buffer = self.lookahead_buffer[1:]
24 return c
25
26 def unget_char(self, c):
27 self.lookahead_buffer = [c] + self.lookahead_buffer
28
29 def get_chars(self, n):
30 n1 = min(n, len(self.lookahead_buffer))
31 s = b"".join(self.lookahead_buffer[:n1])
32 self.lookahead_buffer = self.lookahead_buffer[n1:]
33 n2 = n - n1
34 if n2 > 0:
35 s += self.f.read(n2)
36 return s
37
38 def peek_char(self):
39 c = self.get_char()
40 if c:
41 self.unget_char(c)
42 return c
43
44
45 def skip_spaces(f):
46 c = f.get_char()
47 while c != None:
48 if c.isspace():
49 c = f.get_char()
50 elif c == b"-":
51 # May be line comment.
52 if f.peek_char() == b"-":
53 # Yes, line comment. Skip to end of line.
54 while c != b"\n" and c != None:
55 c = f.get_char()
56 else:
57 break
58 else:
59 break
60 if c:
61 f.unget_char(c)
62
63
64 def parse_specific_char(f, expected):
65 got = f.get_char()
66 if got != expected:
67 f.unget_char(got)
68 raise ValueError
69 return True
70
71
72 def parse_specific_string(f, s):
73 # This funky mess is intended, and is caused by the fact that if `type(b) ==
74 # bytes` then `type(b[0]) == int`, but we need to match each element with a
75 # `bytes`, so therefore we make each character an array element
76 b = s.encode("utf8")
77 bs = [b[i : i + 1] for i in range(len(b))]
78 read = []
79 try:
80 for c in bs:
81 parse_specific_char(f, c)
82 read.append(c)
83 return True
84 except ValueError:
85 for c in read[::-1]:
86 f.unget_char(c)
87 raise
88
89
90 def optional(p, *args):
91 try:
92 return p(*args)
93 except ValueError:
94 return None
95
96
97 def optional_specific_string(f, s):
98 c = f.peek_char()
99 # This funky mess is intended, and is caused by the fact that if `type(b) ==
100 # bytes` then `type(b[0]) == int`, but we need to match each element with a
101 # `bytes`, so therefore we make each character an array element
102 b = s.encode("utf8")
103 bs = [b[i : i + 1] for i in range(len(b))]
104 if c == bs[0]:
105 return parse_specific_string(f, s)
106 else:
107 return False
108
109
110 def sepEndBy(p, sep, *args):
111 elems = []
112 x = optional(p, *args)
113 if x != None:
114 elems += [x]
115 while optional(sep, *args) != None:
116 x = optional(p, *args)
117 if x == None:
118 break
119 else:
120 elems += [x]
121 return elems
122
123
124 # Assumes '0x' has already been read
125 def parse_hex_int(f):
126 s = b""
127 c = f.get_char()
128 while c != None:
129 if c in b"01234556789ABCDEFabcdef":
130 s += c
131 c = f.get_char()
132 elif c == b"_":
133 c = f.get_char() # skip _
134 else:
135 f.unget_char(c)
136 break
137 return str(int(s, 16)).encode("utf8") # ugh
138
139
140 def parse_int(f):
141 s = b""
142 c = f.get_char()
143 if c == b"0" and f.peek_char() in b"xX":
144 c = f.get_char() # skip X
145 return parse_hex_int(f)
146 else:
147 while c != None:
148 if c.isdigit():
149 s += c
150 c = f.get_char()
151 elif c == b"_":
152 c = f.get_char() # skip _
153 else:
154 f.unget_char(c)
155 break
156 if len(s) == 0:
157 raise ValueError
158 return s
159
160
161 def parse_int_signed(f):
162 s = b""
163 c = f.get_char()
164
165 if c == b"-" and f.peek_char().isdigit():
166 return c + parse_int(f)
167 else:
168 if c != b"+":
169 f.unget_char(c)
170 return parse_int(f)
171
172
173 def read_str_comma(f):
174 skip_spaces(f)
175 parse_specific_char(f, b",")
176 return b","
177
178
179 def read_str_int(f, s):
180 skip_spaces(f)
181 x = int(parse_int_signed(f))
182 optional_specific_string(f, s)
183 return x
184
185
186 def read_str_uint(f, s):
187 skip_spaces(f)
188 x = int(parse_int(f))
189 optional_specific_string(f, s)
190 return x
191
192
193 def read_str_i8(f):
194 return np.int8(read_str_int(f, "i8"))
195
196
197 def read_str_i16(f):
198 return np.int16(read_str_int(f, "i16"))
199
200
201 def read_str_i32(f):
202 return np.int32(read_str_int(f, "i32"))
203
204
205 def read_str_i64(f):
206 return np.int64(read_str_int(f, "i64"))
207
208
209 def read_str_u8(f):
210 return np.uint8(read_str_int(f, "u8"))
211
212
213 def read_str_u16(f):
214 return np.uint16(read_str_int(f, "u16"))
215
216
217 def read_str_u32(f):
218 return np.uint32(read_str_int(f, "u32"))
219
220
221 def read_str_u64(f):
222 return np.uint64(read_str_int(f, "u64"))
223
224
225 def read_char(f):
226 skip_spaces(f)
227 parse_specific_char(f, b"'")
228 c = f.get_char()
229 parse_specific_char(f, b"'")
230 return c
231
232
233 def read_str_hex_float(f, sign):
234 int_part = parse_hex_int(f)
235 parse_specific_char(f, b".")
236 frac_part = parse_hex_int(f)
237 parse_specific_char(f, b"p")
238 exponent = parse_int(f)
239
240 int_val = int(int_part, 16)
241 frac_val = float(int(frac_part, 16)) / (16 ** len(frac_part))
242 exp_val = int(exponent)
243
244 total_val = (int_val + frac_val) * (2.0**exp_val)
245 if sign == b"-":
246 total_val = -1 * total_val
247
248 return float(total_val)
249
250
251 def read_str_decimal(f):
252 skip_spaces(f)
253 c = f.get_char()
254 if c == b"-":
255 sign = b"-"
256 else:
257 f.unget_char(c)
258 sign = b""
259
260 # Check for hexadecimal float
261 c = f.get_char()
262 if c == "0" and (f.peek_char() in ["x", "X"]):
263 f.get_char()
264 return read_str_hex_float(f, sign)
265 else:
266 f.unget_char(c)
267
268 bef = optional(parse_int, f)
269 if bef == None:
270 bef = b"0"
271 parse_specific_char(f, b".")
272 aft = parse_int(f)
273 elif optional(parse_specific_char, f, b"."):
274 aft = parse_int(f)
275 else:
276 aft = b"0"
277 if optional(parse_specific_char, f, b"E") or optional(
278 parse_specific_char, f, b"e"
279 ):
280 expt = parse_int_signed(f)
281 else:
282 expt = b"0"
283 return float(sign + bef + b"." + aft + b"E" + expt)
284
285
286 def read_str_f16(f):
287 skip_spaces(f)
288 try:
289 parse_specific_string(f, "f16.nan")
290 return np.float32(np.nan)
291 except ValueError:
292 try:
293 parse_specific_string(f, "f16.inf")
294 return np.float32(np.inf)
295 except ValueError:
296 try:
297 parse_specific_string(f, "-f16.inf")
298 return np.float32(-np.inf)
299 except ValueError:
300 x = read_str_decimal(f)
301 optional_specific_string(f, "f16")
302 return x
303
304
305 def read_str_f32(f):
306 skip_spaces(f)
307 try:
308 parse_specific_string(f, "f32.nan")
309 return np.float32(np.nan)
310 except ValueError:
311 try:
312 parse_specific_string(f, "f32.inf")
313 return np.float32(np.inf)
314 except ValueError:
315 try:
316 parse_specific_string(f, "-f32.inf")
317 return np.float32(-np.inf)
318 except ValueError:
319 x = read_str_decimal(f)
320 optional_specific_string(f, "f32")
321 return x
322
323
324 def read_str_f64(f):
325 skip_spaces(f)
326 try:
327 parse_specific_string(f, "f64.nan")
328 return np.float64(np.nan)
329 except ValueError:
330 try:
331 parse_specific_string(f, "f64.inf")
332 return np.float64(np.inf)
333 except ValueError:
334 try:
335 parse_specific_string(f, "-f64.inf")
336 return np.float64(-np.inf)
337 except ValueError:
338 x = read_str_decimal(f)
339 optional_specific_string(f, "f64")
340 return x
341
342
343 def read_str_bool(f):
344 skip_spaces(f)
345 if f.peek_char() == b"t":
346 parse_specific_string(f, "true")
347 return True
348 elif f.peek_char() == b"f":
349 parse_specific_string(f, "false")
350 return False
351 else:
352 raise ValueError
353
354
355 def read_str_empty_array(f, type_name, rank):
356 parse_specific_string(f, "empty")
357 parse_specific_char(f, b"(")
358 dims = []
359 for i in range(rank):
360 parse_specific_string(f, "[")
361 dims += [int(parse_int(f))]
362 parse_specific_string(f, "]")
363 if np.prod(dims) != 0:
364 raise ValueError
365 parse_specific_string(f, type_name)
366 parse_specific_char(f, b")")
367
368 return tuple(dims)
369
370
371 def read_str_array_elems(f, elem_reader, type_name, rank):
372 skip_spaces(f)
373 try:
374 parse_specific_char(f, b"[")
375 except ValueError:
376 return read_str_empty_array(f, type_name, rank)
377 else:
378 xs = sepEndBy(elem_reader, read_str_comma, f)
379 skip_spaces(f)
380 parse_specific_char(f, b"]")
381 return xs
382
383
384 def read_str_array_helper(f, elem_reader, type_name, rank):
385 def nested_row_reader(_):
386 return read_str_array_helper(f, elem_reader, type_name, rank - 1)
387
388 if rank == 1:
389 row_reader = elem_reader
390 else:
391 row_reader = nested_row_reader
392 return read_str_array_elems(f, row_reader, type_name, rank)
393
394
395 def expected_array_dims(l, rank):
396 if rank > 1:
397 n = len(l)
398 if n == 0:
399 elem = []
400 else:
401 elem = l[0]
402 return [n] + expected_array_dims(elem, rank - 1)
403 else:
404 return [len(l)]
405
406
407 def verify_array_dims(l, dims):
408 if dims[0] != len(l):
409 raise ValueError
410 if len(dims) > 1:
411 for x in l:
412 verify_array_dims(x, dims[1:])
413
414
415 def read_str_array(f, elem_reader, type_name, rank, bt):
416 elems = read_str_array_helper(f, elem_reader, type_name, rank)
417 if type(elems) == tuple:
418 # Empty array
419 return np.empty(elems, dtype=bt)
420 else:
421 dims = expected_array_dims(elems, rank)
422 verify_array_dims(elems, dims)
423 return np.array(elems, dtype=bt)
424
425
426 ################################################################################
427
428 READ_BINARY_VERSION = 2
429
430 # struct format specified at
431 # https://docs.python.org/2/library/struct.html#format-characters
432
433
434 def mk_bin_scalar_reader(t):
435 def bin_reader(f):
436 fmt = FUTHARK_PRIMTYPES[t]["bin_format"]
437 size = FUTHARK_PRIMTYPES[t]["size"]
438 tf = FUTHARK_PRIMTYPES[t]["numpy_type"]
439 return tf(struct.unpack("<" + fmt, f.get_chars(size))[0])
440
441 return bin_reader
442
443
444 read_bin_i8 = mk_bin_scalar_reader("i8")
445 read_bin_i16 = mk_bin_scalar_reader("i16")
446 read_bin_i32 = mk_bin_scalar_reader("i32")
447 read_bin_i64 = mk_bin_scalar_reader("i64")
448
449 read_bin_u8 = mk_bin_scalar_reader("u8")
450 read_bin_u16 = mk_bin_scalar_reader("u16")
451 read_bin_u32 = mk_bin_scalar_reader("u32")
452 read_bin_u64 = mk_bin_scalar_reader("u64")
453
454 read_bin_f16 = mk_bin_scalar_reader("f16")
455 read_bin_f32 = mk_bin_scalar_reader("f32")
456 read_bin_f64 = mk_bin_scalar_reader("f64")
457
458 read_bin_bool = mk_bin_scalar_reader("bool")
459
460
461 def read_is_binary(f):
462 skip_spaces(f)
463 c = f.get_char()
464 if c == b"b":
465 bin_version = read_bin_u8(f)
466 if bin_version != READ_BINARY_VERSION:
467 panic(
468 1,
469 "binary-input: File uses version %i, but I only understand version %i.\n",
470 bin_version,
471 READ_BINARY_VERSION,
472 )
473 return True
474 else:
475 f.unget_char(c)
476 return False
477
478
479 FUTHARK_PRIMTYPES = {
480 "i8": {
481 "binname": b" i8",
482 "size": 1,
483 "bin_reader": read_bin_i8,
484 "str_reader": read_str_i8,
485 "bin_format": "b",
486 "numpy_type": np.int8,
487 },
488 "i16": {
489 "binname": b" i16",
490 "size": 2,
491 "bin_reader": read_bin_i16,
492 "str_reader": read_str_i16,
493 "bin_format": "h",
494 "numpy_type": np.int16,
495 },
496 "i32": {
497 "binname": b" i32",
498 "size": 4,
499 "bin_reader": read_bin_i32,
500 "str_reader": read_str_i32,
501 "bin_format": "i",
502 "numpy_type": np.int32,
503 },
504 "i64": {
505 "binname": b" i64",
506 "size": 8,
507 "bin_reader": read_bin_i64,
508 "str_reader": read_str_i64,
509 "bin_format": "q",
510 "numpy_type": np.int64,
511 },
512 "u8": {
513 "binname": b" u8",
514 "size": 1,
515 "bin_reader": read_bin_u8,
516 "str_reader": read_str_u8,
517 "bin_format": "B",
518 "numpy_type": np.uint8,
519 },
520 "u16": {
521 "binname": b" u16",
522 "size": 2,
523 "bin_reader": read_bin_u16,
524 "str_reader": read_str_u16,
525 "bin_format": "H",
526 "numpy_type": np.uint16,
527 },
528 "u32": {
529 "binname": b" u32",
530 "size": 4,
531 "bin_reader": read_bin_u32,
532 "str_reader": read_str_u32,
533 "bin_format": "I",
534 "numpy_type": np.uint32,
535 },
536 "u64": {
537 "binname": b" u64",
538 "size": 8,
539 "bin_reader": read_bin_u64,
540 "str_reader": read_str_u64,
541 "bin_format": "Q",
542 "numpy_type": np.uint64,
543 },
544 "f16": {
545 "binname": b" f16",
546 "size": 2,
547 "bin_reader": read_bin_f16,
548 "str_reader": read_str_f16,
549 "bin_format": "e",
550 "numpy_type": np.float16,
551 },
552 "f32": {
553 "binname": b" f32",
554 "size": 4,
555 "bin_reader": read_bin_f32,
556 "str_reader": read_str_f32,
557 "bin_format": "f",
558 "numpy_type": np.float32,
559 },
560 "f64": {
561 "binname": b" f64",
562 "size": 8,
563 "bin_reader": read_bin_f64,
564 "str_reader": read_str_f64,
565 "bin_format": "d",
566 "numpy_type": np.float64,
567 },
568 "bool": {
569 "binname": b"bool",
570 "size": 1,
571 "bin_reader": read_bin_bool,
572 "str_reader": read_str_bool,
573 "bin_format": "b",
574 "numpy_type": bool,
575 },
576 }
577
578
579 def read_bin_read_type(f):
580 read_binname = f.get_chars(4)
581
582 for k, v in FUTHARK_PRIMTYPES.items():
583 if v["binname"] == read_binname:
584 return k
585 panic(1, "binary-input: Did not recognize the type '%s'.\n", read_binname)
586
587
588 def numpy_type_to_type_name(t):
589 for k, v in FUTHARK_PRIMTYPES.items():
590 if v["numpy_type"] == t:
591 return k
592 raise Exception("Unknown Numpy type: {}".format(t))
593
594
595 def read_bin_ensure_scalar(f, expected_type):
596 dims = read_bin_i8(f)
597
598 if dims != 0:
599 panic(
600 1,
601 "binary-input: Expected scalar (0 dimensions), but got array with %i dimensions.\n",
602 dims,
603 )
604
605 bin_type = read_bin_read_type(f)
606 if bin_type != expected_type:
607 panic(
608 1,
609 "binary-input: Expected scalar of type %s but got scalar of type %s.\n",
610 expected_type,
611 bin_type,
612 )
613
614
615 # ------------------------------------------------------------------------------
616 # General interface for reading Primitive Futhark Values
617 # ------------------------------------------------------------------------------
618
619
620 def read_scalar(f, ty):
621 if read_is_binary(f):
622 read_bin_ensure_scalar(f, ty)
623 return FUTHARK_PRIMTYPES[ty]["bin_reader"](f)
624 return FUTHARK_PRIMTYPES[ty]["str_reader"](f)
625
626
627 def read_array(f, expected_type, rank):
628 if not read_is_binary(f):
629 str_reader = FUTHARK_PRIMTYPES[expected_type]["str_reader"]
630 return read_str_array(
631 f,
632 str_reader,
633 expected_type,
634 rank,
635 FUTHARK_PRIMTYPES[expected_type]["numpy_type"],
636 )
637
638 bin_rank = read_bin_u8(f)
639
640 if bin_rank != rank:
641 panic(
642 1,
643 "binary-input: Expected %i dimensions, but got array with %i dimensions.\n",
644 rank,
645 bin_rank,
646 )
647
648 bin_type_enum = read_bin_read_type(f)
649 if expected_type != bin_type_enum:
650 panic(
651 1,
652 "binary-input: Expected %iD-array with element type '%s' but got %iD-array with element type '%s'.\n",
653 rank,
654 expected_type,
655 bin_rank,
656 bin_type_enum,
657 )
658
659 shape = []
660 elem_count = 1
661 for i in range(rank):
662 bin_size = read_bin_i64(f)
663 elem_count *= bin_size
664 shape.append(bin_size)
665
666 bin_fmt = FUTHARK_PRIMTYPES[bin_type_enum]["bin_format"]
667
668 # We first read the expected number of types into a bytestring,
669 # then use np.frombuffer. This is because np.fromfile does not
670 # work on things that are insufficiently file-like, like a network
671 # stream.
672 bytes = f.get_chars(elem_count * FUTHARK_PRIMTYPES[expected_type]["size"])
673 arr = np.frombuffer(
674 bytes, dtype=FUTHARK_PRIMTYPES[bin_type_enum]["numpy_type"]
675 )
676 arr.shape = shape
677
678 return arr.copy() # To ensure it is writeable.
679
680
681 if sys.version_info >= (3, 0):
682 input_reader = ReaderInput(sys.stdin.buffer)
683 else:
684 input_reader = ReaderInput(sys.stdin)
685
686 import re
687
688
689 def read_value(type_desc, reader=input_reader):
690 """Read a value of the given type. The type is a string
691 representation of the Futhark type."""
692 m = re.match(r"((?:\[\])*)([a-z0-9]+)$", type_desc)
693 if m:
694 dims = int(len(m.group(1)) / 2)
695 basetype = m.group(2)
696 assert m and basetype in FUTHARK_PRIMTYPES, "Unknown type: {}".format(
697 type_desc
698 )
699 if dims > 0:
700 return read_array(reader, basetype, dims)
701 else:
702 return read_scalar(reader, basetype)
703
704
705 def end_of_input(entry, f=input_reader):
706 skip_spaces(f)
707 if f.get_char() != b"":
708 panic(1, 'Expected EOF on stdin after reading input for "%s".', entry)
709
710
711 def write_value_text(v, out=sys.stdout):
712 if type(v) == np.uint8:
713 out.write("%uu8" % v)
714 elif type(v) == np.uint16:
715 out.write("%uu16" % v)
716 elif type(v) == np.uint32:
717 out.write("%uu32" % v)
718 elif type(v) == np.uint64:
719 out.write("%uu64" % v)
720 elif type(v) == np.int8:
721 out.write("%di8" % v)
722 elif type(v) == np.int16:
723 out.write("%di16" % v)
724 elif type(v) == np.int32:
725 out.write("%di32" % v)
726 elif type(v) == np.int64:
727 out.write("%di64" % v)
728 elif type(v) in [bool, np.bool_]:
729 if v:
730 out.write("true")
731 else:
732 out.write("false")
733 elif type(v) == np.float16:
734 if np.isnan(v):
735 out.write("f16.nan")
736 elif np.isinf(v):
737 if v >= 0:
738 out.write("f16.inf")
739 else:
740 out.write("-f16.inf")
741 else:
742 out.write("%.6ff16" % v)
743 elif type(v) == np.float32:
744 if np.isnan(v):
745 out.write("f32.nan")
746 elif np.isinf(v):
747 if v >= 0:
748 out.write("f32.inf")
749 else:
750 out.write("-f32.inf")
751 else:
752 out.write("%.6ff32" % v)
753 elif type(v) == np.float64:
754 if np.isnan(v):
755 out.write("f64.nan")
756 elif np.isinf(v):
757 if v >= 0:
758 out.write("f64.inf")
759 else:
760 out.write("-f64.inf")
761 else:
762 out.write("%.6ff64" % v)
763 elif type(v) == np.ndarray:
764 if np.prod(v.shape) == 0:
765 tname = numpy_type_to_type_name(v.dtype)
766 out.write(
767 "empty({}{})".format(
768 "".join(["[{}]".format(d) for d in v.shape]), tname
769 )
770 )
771 else:
772 first = True
773 out.write("[")
774 for x in v:
775 if not first:
776 out.write(", ")
777 first = False
778 write_value(x, out=out)
779 out.write("]")
780 else:
781 raise Exception("Cannot print value of type {}: {}".format(type(v), v))
782
783
784 type_strs = {
785 np.dtype("int8"): b" i8",
786 np.dtype("int16"): b" i16",
787 np.dtype("int32"): b" i32",
788 np.dtype("int64"): b" i64",
789 np.dtype("uint8"): b" u8",
790 np.dtype("uint16"): b" u16",
791 np.dtype("uint32"): b" u32",
792 np.dtype("uint64"): b" u64",
793 np.dtype("float16"): b" f16",
794 np.dtype("float32"): b" f32",
795 np.dtype("float64"): b" f64",
796 np.dtype("bool"): b"bool",
797 }
798
799
800 def construct_binary_value(v):
801 t = v.dtype
802 shape = v.shape
803
804 elems = 1
805 for d in shape:
806 elems *= d
807
808 num_bytes = 1 + 1 + 1 + 4 + len(shape) * 8 + elems * t.itemsize
809 bytes = bytearray(num_bytes)
810 bytes[0] = np.int8(ord("b"))
811 bytes[1] = 2
812 bytes[2] = np.int8(len(shape))
813 bytes[3:7] = type_strs[t]
814
815 for i in range(len(shape)):
816 bytes[7 + i * 8 : 7 + (i + 1) * 8] = np.int64(shape[i]).tobytes()
817
818 bytes[7 + len(shape) * 8 :] = np.ascontiguousarray(v).tobytes()
819
820 return bytes
821
822
823 def write_value_binary(v, out=sys.stdout):
824 if sys.version_info >= (3, 0):
825 out = out.buffer
826 out.write(construct_binary_value(v))
827
828
829 def write_value(v, out=sys.stdout, binary=False):
830 if binary:
831 return write_value_binary(v, out=out)
832 else:
833 return write_value_text(v, out=out)
834
835
836 # End of values.py.