Debian Codegraph @

autoray/autoray.py

raw
1 """
2 AUTORAY - backend agnostic array operations.
3
4
5 Copyright 2019-2023 Johnnie Gray
6
7 Licensed under the Apache License, Version 2.0 (the "License");
8 you may not use this file except in compliance with the License.
9 You may obtain a copy of the License at
10
11 http://www.apache.org/licenses/LICENSE-2.0
12
13 Unless required by applicable law or agreed to in writing, software
14 distributed under the License is distributed on an "AS IS" BASIS,
15 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 See the License for the specific language governing permissions and
17 limitations under the License.
18 """
19
20 import contextlib
21 import functools
22 import importlib
23 import itertools
24 import math
25 import threading
26 from collections import OrderedDict, defaultdict
27 from inspect import signature
28
29
30 def do(fn, *args, like=None, **kwargs):
31 """Do function named ``fn`` on ``(*args, **kwargs)``, peforming single
32 dispatch to retrieve ``fn`` based on whichever library defines the class of
33 the ``args[0]``, or the ``like`` keyword argument if specified.
34
35 Examples
36 --------
37
38 Works on numpy arrays:
39
40 >>> import numpy as np
41 >>> x_np = np.random.uniform(size=[5])
42 >>> y_np = do('sqrt', x_np)
43 >>> y_np
44 array([0.32464973, 0.90379787, 0.85037325, 0.88729814, 0.46768083])
45
46 >>> type(y_np)
47 numpy.ndarray
48
49 Works on cupy arrays:
50
51 >>> import cupy as cp
52 >>> x_cp = cp.random.uniform(size=[5])
53 >>> y_cp = do('sqrt', x_cp)
54 >>> y_cp
55 array([0.44541656, 0.88713113, 0.92626237, 0.64080557, 0.69620767])
56
57 >>> type(y_cp)
58 cupy.core.core.ndarray
59
60 Works on tensorflow arrays:
61
62 >>> import tensorflow as tf
63 >>> x_tf = tf.random.uniform(shape=[5])
64 >>> y_tf = do('sqrt', x_tf)
65 >>> y_tf
66 <tf.Tensor 'Sqrt_1:0' shape=(5,) dtype=float32>
67
68 >>> type(y_tf)
69 tensorflow.python.framework.ops.Tensor
70
71 You get the idea.
72
73 For functions that don't dispatch on the first argument you can use the
74 ``like`` keyword:
75
76 >>> do('eye', 3, like=x_tf)
77 <tf.Tensor: id=91, shape=(3, 3), dtype=float32>
78 """
79 backend = _choose_backend(fn, args, kwargs, like=like)
80 func = get_lib_fn(backend, fn)
81 return func(*args, **kwargs)
82
83
84 # ------------------------- efficiently dispatching ------------------------- #
85
86
87 def _default_infer_from_sig(fn, *args, **kwargs):
88 """This is the default backend dispatcher, used if no global backend has
89 been set. Hot swapping this function out as below avoids having to check
90 manually for a global backend or worse, a thread aware global backend, on
91 every call to ``do``.
92 """
93 return _DISPATCHERS[fn](*args, **kwargs)
94
95
96 _global_backend = None
97 _inferrer_global = _default_infer_from_sig
98
99 # this is the function that autoray uses when `do` is called without an
100 # explicit like/backend argument. It is set to `_default_infer_from_sig` by
101 # default, but can be set to `_always_the_same` if a global backend is set e.g.
102 _infer_auto = _inferrer_global
103
104 # if a thread that isn't the 'importing' thread tries to set a backend, this
105 # by default turns on thread aware dispatching, but once such custom sub
106 # backends have been reset, the global values above are used again.
107 _global_backends_threadaware = {}
108 _inferrers_threadaware = {}
109 _importing_thrid = threading.get_ident()
110 _backend_lock = threading.Lock()
111
112
113 def _default_infer_from_sig_threadaware(fn, args, kwargs):
114 # check for a thread aware inferrer, default to the global inferrer
115 thrid = threading.get_ident()
116 return _inferrers_threadaware.get(thrid, _inferrer_global)(
117 fn, args, kwargs
118 )
119
120
121 def _always_the_same(fn, args, kwargs, backend):
122 return backend
123
124
125 def get_backend(get_globally="auto"):
126 """Return the universally set backend, if any.
127
128 Parameters
129 ----------
130 get_globally : {"auto", False, True}, optional
131 Which backend to return:
132
133 - True: return the globally set backend, if any.
134 - False: return the backend set for the current thread, if any.
135 - "auto": return the globally set backend, if this thread is the thread
136 that imported autoray. Otherwise return the backend set for the
137 current thread, if any.
138
139 Returns
140 -------
141 backend : str or None
142 The name of the backend, or None if no backend is set.
143 """
144 if get_globally == "auto":
145 get_globally = threading.get_ident() == _importing_thrid
146
147 if get_globally:
148 backend = _global_backend
149 else:
150 thrid = threading.get_ident()
151 backend = _global_backends_threadaware.get(thrid, None)
152
153 return backend
154
155
156 def set_backend(like, set_globally="auto"):
157 """Set a default global backend. The argument ``like`` can be an explicit
158 backend name or an ``array``.
159
160 Parameters
161 ----------
162 like : str or array
163 The backend to set. If an array, the backend of the array's class will
164 be set.
165 set_globally : {"auto", False, True}, optional
166 Whether to set the backend globally or for the current thread:
167
168 - True: set the backend globally.
169 - False: set the backend for the current thread.
170 - "auto": set the backend globally if this thread is the thread that
171 imported autoray. Otherwise set the backend for the current thread.
172
173 Only one thread should ever call this function with
174 ``set_globally=True``, (by default this is importing thread).
175 """
176 global _global_backend
177 global _infer_auto
178 global _inferrer_global
179
180 if like is None:
181 backend = None
182 inferrer = _default_infer_from_sig
183 elif isinstance(like, str):
184 backend = like
185 inferrer = functools.partial(_always_the_same, backend=backend)
186 else:
187 backend = _infer_class_backend_cached(like.__class__)
188 inferrer = functools.partial(_always_the_same, backend=backend)
189
190 if set_globally == "auto":
191 set_globally = threading.get_ident() == _importing_thrid
192
193 if set_globally:
194 _global_backend = backend
195 _inferrer_global = inferrer
196 if not _inferrers_threadaware:
197 # only revert the actual function if no subthread backends set
198 _infer_auto = inferrer
199 else:
200 thrid = threading.get_ident()
201 _backend_lock.acquire()
202 if backend is None:
203 _global_backends_threadaware.pop(thrid)
204 _inferrers_threadaware.pop(thrid)
205 else:
206 _global_backends_threadaware[thrid] = backend
207 _inferrers_threadaware[thrid] = inferrer
208
209 if _inferrers_threadaware:
210 # a subthread backend has been set, so we need to be thread aware
211 _infer_auto = _default_infer_from_sig_threadaware
212 else:
213 # no subthread backend has been set anymore, so we can ignore
214 # threads and just use the global inferrer
215 _infer_auto = _inferrer_global
216 _backend_lock.release()
217
218
219 @contextlib.contextmanager
220 def backend_like(like, set_globally="auto"):
221 """Context manager for setting a default backend. The argument ``like`` can
222 be an explicit backend name or an ``array`` to infer it from.
223
224 Parameters
225 ----------
226 like : str or array
227 The backend to set. If an array, the backend of the array's class will
228 be set.
229 set_globally : {"auto", False, True}, optional
230 Whether to set the backend globally or for the current thread:
231
232 - True: set the backend globally.
233 - False: set the backend for the current thread.
234 - "auto": set the backend globally if this thread is the thread that
235 imported autoray. Otherwise set the backend for the current thread.
236
237 Only one thread should ever call this function with
238 ``set_globally=True``, (by default this is importing thread).
239 """
240 if set_globally == "auto":
241 set_globally = threading.get_ident() == _importing_thrid
242
243 old_backend = get_backend(get_globally=set_globally)
244 try:
245 set_backend(like, set_globally)
246 yield
247 finally:
248 set_backend(old_backend, set_globally)
249
250
251 _CUSTOM_BACKENDS = {}
252
253
254 def register_backend(cls, name):
255 """Register the name (and by default the module or submodule) of a custom
256 array class.
257
258 Parameters
259 ----------
260 cls : type
261 The array class itself.
262 name : str
263 The name of the backend that should be used for this class. By default
264 this wil be assumed to be the location of the relevant functions for
265 this class, but this can be overridden.
266 """
267 if not isinstance(cls, type):
268 raise TypeError("The array class itself should be supplied.")
269
270 global _CUSTOM_BACKENDS
271 _CUSTOM_BACKENDS[cls] = name
272
273
274 @functools.lru_cache(None)
275 def _infer_class_backend_cached(cls):
276 try:
277 import numpy as _numpy
278
279 if issubclass(cls, _numpy.ndarray):
280 return "numpy"
281 except ImportError:
282 # numpy not installed
283 pass
284
285 if cls in _CUSTOM_BACKENDS:
286 return _CUSTOM_BACKENDS[cls]
287
288 lib = cls.__module__.split(".")[0]
289
290 # check if lib should mapped entirely to another lib
291 backend = _BACKEND_ALIASES.get(lib, lib)
292
293 return backend
294
295
296 def infer_backend(array):
297 """Get the name of the library that defined the class of ``array`` - unless
298 ``array`` is directly a subclass of ``numpy.ndarray``, in which case assume
299 ``numpy`` is the desired backend.
300 """
301 return _infer_class_backend_cached(array.__class__)
302
303
304 multi_class_priorities = {
305 "builtins": -2,
306 "numpy": -1,
307 "autoray.lazy": 1,
308 }
309
310
311 @functools.lru_cache(None)
312 def _infer_class_backend_multi_cached(classes):
313 return max(
314 map(_infer_class_backend_cached, classes),
315 key=lambda n: multi_class_priorities.get(n, 0),
316 )
317
318
319 def infer_backend_multi(*arrays):
320 """Infer which backend should be used for a function that takes multiple
321 arguments. This assigns a priority to each backend, and returns the backend
322 with the highest priority. By default, the priority is:
323
324 - ``builtins``: -2
325 - ``numpy``: -1
326 - other backends: 0
327 - ``autoray.lazy``: 1
328
329 I.e. when mixing with ``numpy``, other array libraries are preferred, when
330 mixing with ``autoray.lazy``, ``autoray.lazy`` is preferred. This has quite
331 low overhead due to caching.
332 """
333 return _infer_class_backend_multi_cached(
334 tuple(array.__class__ for array in arrays)
335 )
336
337
338 # the set of functions that create new arrays, with `dtype` and possibly
339 # `device` kwargs, that should be inferred from the like argument
340 _CREATION_ROUTINES = {
341 "empty",
342 "eye",
343 "full",
344 "identity",
345 "ones",
346 "zeros",
347 # TODO: should these be included?
348 # "arange",
349 # "geomspace",
350 # "linspace",
351 # "logspace",
352 }
353
354 # cache for whether backends have a device attribute
355 _CREATION_INJECT = {}
356
357
358 def register_creation_routine(
359 backend, fn, inject_dtype=True, inject_device=False
360 ):
361 """Register a function that creates a new array, with `dtype` and possibly
362 `device` kwargs, that should be inferred from the like argument. This is
363 not necessary for array creation routines that don't accept either.
364
365 Parameters
366 ----------
367 backend : str
368 The backend to register the function for.
369 fn : str
370 The name of the function to register.
371 inject_dtype : bool, optional
372 Whether to inject a `dtype` argument based on the `like` argument.
373 inject_device : bool, optional
374 Whether to inject a `device` argument based on the `like` argument.
375 """
376 _CREATION_INJECT[backend, fn] = (inject_dtype, inject_device)
377
378
379 def _choose_backend(fn, args, kwargs, like=None):
380 """Private function to choose a backend based on function name and
381 signature, which passes args and kwargs by reference for performance and
382 also to allow injection of dtype and device arguments for array creation
383 routines.
384 """
385 if like is None:
386 # infer from function call (or global backend)
387 return _infer_auto(fn, args, kwargs)
388 elif isinstance(like, str):
389 # explicit backend
390 return like
391 else:
392 # explicit example array
393 backend = _infer_class_backend_cached(like.__class__)
394
395 # check if we should set some extra defaults based on the example array
396 if fn in _CREATION_ROUTINES:
397 try:
398 inject_dtype, inject_device = _CREATION_INJECT[backend, fn]
399 except KeyError:
400 # default to just dtype (e.g. for numpy)
401 inject_dtype = True
402 inject_device = False
403 _CREATION_INJECT[backend, fn] = (inject_dtype, inject_device)
404
405 if inject_dtype:
406 kwargs.setdefault("dtype", getattr(like, "dtype", type(like)))
407 if inject_device:
408 kwargs.setdefault("device", like.device)
409
410 return backend
411
412
413 def choose_backend(fn, *args, like=None, **kwargs):
414 """Choose a backend based on function name, arguments, and the ``like``
415 keyword argument. The default, if ``like`` is not specified, is to infer
416 the backend from the function call, the default of which is simply to use
417 the first argument, if no custom dispatcher is found. Otherwise the
418 backend is chosen based on the ``like`` argument - which can be an explicit
419 backend name or an arbitrary object.
420 """
421 return _choose_backend(fn, args, kwargs, like=like)
422
423
424 # ------------------- importing and caching the function -------------------- #
425
426 # lookup for mapping entire lib to another
427 _BACKEND_ALIASES = {}
428
429 # global (non function specific) aliases
430 _MODULE_ALIASES = {}
431
432 # lookup for when functions are elsewhere than the expected location
433 _SUBMODULE_ALIASES = {}
434
435 # lookup for when functions are simply called something else
436 _FUNC_ALIASES = {}
437
438 # custom wrappers for when functions don't just have different location or
439 # name. For example, when kwargs need to be translated or results modified
440 _CUSTOM_WRAPPERS = {}
441
442 # actual cache of funtions to use - this is populated lazily and can be used
443 # to directly set an implementation of a function for a specific backend
444 _FUNCS = {}
445
446 # these are functions where a default implementation can be constructed
447 # (composed of other functions), but this is only done lazily
448 _COMPOSED_FUNCTION_GENERATORS = {}
449
450
451 def import_lib_fn(backend, fn):
452 # first check explicitly composed functions -> if the function hasn't been
453 # called directly yet, it won't have been loaded into the cache, and needs
454 # generating before e.g. the ``do`` verrsion will work
455 if fn in _COMPOSED_FUNCTION_GENERATORS:
456 return _COMPOSED_FUNCTION_GENERATORS[fn](backend)
457
458 try:
459 # submodule where function is found for backend,
460 # e.g. ['tensorflow', trace'] -> 'tensorflow.linalg'
461 try:
462 full_location = _SUBMODULE_ALIASES[backend, fn]
463
464 # if explicit submodule alias given, don't use prepended location
465 # for example, ('torch', 'linalg.svd') -> torch.svd
466 only_fn = fn.split(".")[-1]
467
468 except KeyError:
469 full_location = backend
470
471 # move any prepended location into the full module path
472 # e.g. 'fn=linalg.eigh' -> ['linalg', 'eigh']
473 split_fn = fn.split(".")
474 full_location = ".".join([full_location] + split_fn[:-1])
475 only_fn = split_fn[-1]
476
477 # try aliases for global (not function specific) modules and
478 # submodules:
479 # e.g. 'decimal' -> 'math'
480 # e.g. 'cupy.scipy' -> 'cupyx.scipy'
481 # we don't do this if the function location has been explicitly
482 # give in _SUBMODULE_ALIASES, as that is already a full path
483 for k, v in _MODULE_ALIASES.items():
484 if full_location[: len(k)] == k:
485 full_location = full_location.replace(k, v, 1)
486 break
487
488 # cached lookup of custom name function might take
489 # e.g. ['tensorflow', 'sum'] -> 'reduce_sum'
490 fn_name = _FUNC_ALIASES.get((backend, fn), only_fn)
491
492 # import the function into the cache
493 try:
494 lib = importlib.import_module(full_location)
495 except ImportError:
496 if "." in full_location:
497 # sometimes libraries hack an attribute to look like submodule
498 mod, *submods = full_location.split(".")
499 lib = importlib.import_module(mod)
500 # also need to handle nested submodules
501 for submod in submods:
502 lib = getattr(lib, submod)
503 else:
504 # failed to import library at all -> catch + raise ImportError
505 raise AttributeError
506
507 # check for a custom wrapper but default to identity
508 wrapper = _CUSTOM_WRAPPERS.get((backend, fn), lambda fn: fn)
509
510 # store the function!
511 lib_fn = _FUNCS[backend, fn] = wrapper(getattr(lib, fn_name))
512
513 except AttributeError:
514 # check if there is a backup function (e.g. for older library version)
515 backend_alt = backend + "[alt]"
516 if backend_alt in _MODULE_ALIASES:
517 return import_lib_fn(backend_alt, fn)
518
519 raise ImportError(
520 f"autoray couldn't find function '{fn}' for "
521 f"backend '{backend.replace('[alt]', '')}'."
522 )
523
524 return lib_fn
525
526
527 def get_lib_fn(backend, fn):
528 """Cached retrieval of correct function for backend, all the logic for
529 finding the correct funtion only runs the first time.
530
531 Parameters
532 ----------
533 backend : str
534 The module defining the array class to dispatch on.
535 fn : str
536 The function to retrieve.
537
538 Returns
539 -------
540 callable
541 """
542
543 try:
544 lib_fn = _FUNCS[backend, fn]
545 except KeyError:
546 lib_fn = import_lib_fn(backend, fn)
547 return lib_fn
548
549
550 # --------------------------- register your own! ---------------------------- #
551
552
553 def register_function(backend, name, fn, wrap=False):
554 """Directly provide your own function.
555
556 Parameters
557 ----------
558 backend : str
559 The name of the backend to register the function for.
560 name : str
561 Name of the function, e.g. `'sum'` or `'linalg.svd'`.
562 fn : callable
563 The function to register.
564 wrap : bool, optional
565 Whether to wrap the old function like ``fn(old_fn)`` rather than
566 directly supply the entire new function.
567 """
568 if wrap:
569 old = get_lib_fn(backend, name)
570 _FUNCS[backend, name] = fn(old)
571 else:
572 _FUNCS[backend, name] = fn
573
574
575 # ------------------------------- tree utils -------------------------------- #
576
577 TREE_MAP_REGISTRY = {}
578 TREE_APPLY_REGISTRY = {}
579 TREE_ITER_REGISTRY = {}
580
581
582 def tree_register_container(cls, mapper, iterator, applier):
583 """Register a new container type for use with ``tree_map`` and
584 ``tree_apply``.
585
586 Parameters
587 ----------
588 cls : type
589 The container type to register.
590 mapper : callable
591 A function that takes ``f``, ``tree`` and ``is_leaf`` and returns a new
592 tree of type ``cls`` with ``f`` applied to all leaves.
593 applier : callable
594 A function that takes ``f``, ``tree`` and ``is_leaf`` and applies ``f``
595 to all leaves in ``tree``.
596 """
597 TREE_MAP_REGISTRY[cls] = mapper
598 TREE_ITER_REGISTRY[cls] = iterator
599 TREE_APPLY_REGISTRY[cls] = applier
600
601
602 IS_CONTAINER_CACHE = {}
603
604
605 def is_not_container(x):
606 """The default function to determine if an object is a leaf. This simply
607 checks if the object is an instance of any of the registered container
608 types.
609 """
610 try:
611 return IS_CONTAINER_CACHE[x.__class__]
612 except KeyError:
613 isleaf = not any(isinstance(x, cls) for cls in TREE_MAP_REGISTRY)
614 IS_CONTAINER_CACHE[x.__class__] = isleaf
615 return isleaf
616
617
618 def is_array(x):
619 """An alternative leaf tester for addressing only arrays within trees."""
620 return hasattr(x, "shape")
621
622
623 def identity(f, tree, is_leaf):
624 return tree
625
626
627 TREE_MAPPER_CACHE = {}
628
629
630 def tree_map(f, tree, is_leaf=is_not_container):
631 """Map ``f`` over all leaves in ``tree``, returning a new pytree.
632
633 Parameters
634 ----------
635 f : callable
636 A function to apply to all leaves in ``tree``.
637 tree : pytree
638 A nested sequence of tuples, lists, dicts and other objects.
639 is_leaf : callable
640 A function to determine if an object is a leaf, ``f`` is only applied
641 to objects for which ``is_leaf(x)`` returns ``True``.
642
643 Returns
644 -------
645 pytree
646 """
647 if is_leaf(tree):
648 return f(tree)
649
650 try:
651 return TREE_MAPPER_CACHE[tree.__class__](f, tree, is_leaf)
652 except KeyError:
653 # reverse so later registered classes take precedence
654 for cls, mapper in reversed(TREE_MAP_REGISTRY.items()):
655 if isinstance(tree, cls):
656 break
657 else:
658 # neither leaf nor container -> simply return it
659 mapper = identity
660 TREE_MAPPER_CACHE[tree.__class__] = mapper
661 return mapper(f, tree, is_leaf)
662
663
664 def empty(tree, is_leaf):
665 return iter(())
666
667
668 TREE_ITER_CACHE = {}
669
670
671 def tree_iter(tree, is_leaf=is_not_container):
672 """Iterate over all leaves in ``tree``.
673
674 Parameters
675 ----------
676 f : callable
677 A function to apply to all leaves in ``tree``.
678 tree : pytree
679 A nested sequence of tuples, lists, dicts and other objects.
680 is_leaf : callable
681 A function to determine if an object is a leaf, ``f`` is only applied
682 to objects for which ``is_leaf(x)`` returns ``True``.
683 """
684 if is_leaf(tree):
685 yield tree
686 return
687
688 try:
689 yield from TREE_ITER_CACHE[tree.__class__](tree, is_leaf)
690 except KeyError:
691 # reverse so later registered classes take precedence
692 for cls, iterator in reversed(TREE_ITER_REGISTRY.items()):
693 if isinstance(tree, cls):
694 break
695 else:
696 # neither leaf nor container -> simply ignore it
697 iterator = empty
698 TREE_ITER_CACHE[tree.__class__] = iterator
699 yield from iterator(tree, is_leaf)
700
701
702 def nothing(f, tree, is_leaf):
703 pass
704
705
706 TREE_APPLIER_CACHE = {}
707
708
709 def tree_apply(f, tree, is_leaf=is_not_container):
710 """Apply ``f`` to all leaves in ``tree``, no new pytree is built.
711
712 Parameters
713 ----------
714 f : callable
715 A function to apply to all leaves in ``tree``.
716 tree : pytree
717 A nested sequence of tuples, lists, dicts and other objects.
718 is_leaf : callable
719 A function to determine if an object is a leaf, ``f`` is only applied
720 to objects for which ``is_leaf(x)`` returns ``True``.
721 """
722 if is_leaf(tree):
723 f(tree)
724 return
725
726 try:
727 TREE_APPLIER_CACHE[tree.__class__](f, tree, is_leaf)
728 except KeyError:
729 # reverse so later registered classes take precedence
730 for cls, applier in reversed(TREE_APPLY_REGISTRY.items()):
731 if isinstance(tree, cls):
732 break
733 else:
734 # neither leaf nor container -> simply ignore it
735 applier = nothing
736 TREE_APPLIER_CACHE[tree.__class__] = applier
737 applier(f, tree, is_leaf)
738
739
740 class Leaf:
741 """A singleton object to use as a placeholder in a pytree, for
742 unflattening.
743 """
744
745 __slots__ = ()
746
747 def __repr__(self):
748 return "Leaf"
749
750
751 LEAF = Leaf()
752
753
754 def is_leaf_placeholder(x):
755 # don't do `x is LEAF` to allow pickling / unpickling
756 return x.__class__ is Leaf
757
758
759 def tree_flatten(tree, is_leaf=is_not_container, get_ref=False):
760 """Flatten ``tree`` into a list of leaves.
761
762 Parameters
763 ----------
764 tree : pytree
765 A nested sequence of tuples, lists, dicts and other objects.
766 is_leaf : callable
767 A function to determine if an object is a leaf, only objects for which
768 ``is_leaf(x)`` returns ``True`` are returned in the flattened list.
769 get_ref : bool
770 If ``True``, a reference tree is also returned which can be used to
771 reconstruct the original tree from a flattened list.
772
773 Returns
774 -------
775 objs : list
776 The flattened list of leaf objects.
777 (ref_tree) : pytree
778 If ``get_ref`` is ``True``, a reference tree, with leaves of ``Leaf``,
779 is returned which can be used to reconstruct the original tree.
780 """
781 objs = []
782 if get_ref:
783 # return a new tree with Leaf leaves, as well as the flattened list
784
785 def f(x):
786 objs.append(x)
787 return LEAF
788
789 ref_tree = tree_map(f, tree, is_leaf)
790 return objs, ref_tree
791 else:
792 tree_apply(objs.append, tree, is_leaf)
793 return objs
794
795
796 def tree_unflatten(objs, tree, is_leaf=is_leaf_placeholder):
797 """Unflatten ``objs`` into a pytree of the same structure as ``tree``.
798
799 Parameters
800 ----------
801 objs : sequence
802 A sequence of objects to be unflattened into a pytree.
803 tree : pytree
804 A nested sequence of tuples, lists, dicts and other objects, the objs
805 will be inserted into a new pytree of the same structure.
806 is_leaf : callable
807 A function to determine if an object is a leaf, only objects for which
808 ``is_leaf(x)`` returns ``True`` will have the next item from ``objs``
809 inserted. By default checks for the ``Leaf`` object inserted by
810 ``tree_flatten(..., get_ref=True)``.
811
812 Returns
813 -------
814 pytree
815 """
816 objs = iter(objs)
817 return tree_map(lambda _: next(objs), tree, is_leaf)
818
819
820 def tree_map_tuple(f, tree, is_leaf):
821 return tuple(tree_map(f, x, is_leaf) for x in tree)
822
823
824 def tree_iter_tuple(tree, is_leaf):
825 for x in tree:
826 yield from tree_iter(x, is_leaf)
827
828
829 def tree_apply_tuple(f, tree, is_leaf):
830 for x in tree:
831 tree_apply(f, x, is_leaf)
832
833
834 tree_register_container(
835 tuple, tree_map_tuple, tree_iter_tuple, tree_apply_tuple
836 )
837
838
839 def tree_map_list(f, tree, is_leaf):
840 return [tree_map(f, x, is_leaf) for x in tree]
841
842
843 def tree_iter_list(tree, is_leaf):
844 for x in tree:
845 yield from tree_iter(x, is_leaf)
846
847
848 def tree_apply_list(f, tree, is_leaf):
849 for x in tree:
850 tree_apply(f, x, is_leaf)
851
852
853 tree_register_container(list, tree_map_list, tree_iter_list, tree_apply_list)
854
855
856 def tree_map_dict(f, tree, is_leaf):
857 return {k: tree_map(f, v, is_leaf) for k, v in tree.items()}
858
859
860 def tree_iter_dict(tree, is_leaf):
861 for v in tree.values():
862 yield from tree_iter(v, is_leaf)
863
864
865 def tree_apply_dict(f, tree, is_leaf):
866 for v in tree.values():
867 tree_apply(f, v, is_leaf)
868
869
870 tree_register_container(dict, tree_map_dict, tree_iter_dict, tree_apply_dict)
871
872
873 # --------------------------- composed functions ---------------------------- #
874
875
876 class Composed:
877 """Compose an ``autoray.do`` using function. See the main wrapper
878 ``compose``.
879 """
880
881 def __init__(self, fn, name=None):
882 self._default_fn = fn
883 if name is None:
884 name = fn.__name__
885 self._name = name
886 self._supply_backend = "backend" in signature(fn).parameters
887
888 # this registers the fact that when `get_lib_fn` is called, the
889 # function can be created even if it doesn't exist for a specific
890 # backend yet.
891 _COMPOSED_FUNCTION_GENERATORS[self._name] = self.make_function
892
893 def register(self, backend, fn=None):
894 """Register a different implementation for ``backend``."""
895 if fn is not None:
896 register_function(backend, self._name, fn)
897 else:
898 # wrapper form
899 def wrapper(fn):
900 register_function(backend, self._name, fn)
901 return fn
902
903 return wrapper
904
905 def make_function(self, backend):
906 """Make a new function for the specific ``backend``."""
907 if self._supply_backend:
908 # make sure it inherits __name__ etc
909 fn = functools.wraps(self._default_fn)(
910 functools.partial(self._default_fn, backend=backend)
911 )
912 else:
913 fn = self._default_fn
914 self.register(backend, fn)
915 return fn
916
917 def __call__(self, *args, like=None, **kwargs):
918 backend = _choose_backend(self._name, args, kwargs, like=like)
919 # `get_lib_fn` will call `make_function` if the function doesn't exist
920 fn = get_lib_fn(backend, self._name)
921 return fn(*args, **kwargs)
922
923 def __repr__(self):
924 return f"Composed('{self._name}')"
925
926
927 def compose(fn, *, name=None):
928 """Take a function consisting of multiple ``autoray.do`` calls and compose
929 it into a new, single, named function, registered with ``autoray.do``.
930
931 This creates a default implementation of this function for each new backend
932 encountered without explicitly having to write each out, but also allows
933 for specific implementations to be overridden for specific backends.
934
935 If the function takes a ``backend`` argument, it will be supplied with the
936 backend name, to save having to re-choose the backend.
937
938 Specific implementations can be provided by calling the ``register`` method
939 of the composed function, or it can itself be used like a decorator::
940
941 @compose
942 def foo(x):
943 ...
944
945 @foo.register("numpy")
946 @numba.njit
947 def foo_numba(x):
948 ...
949
950 Parameters
951 ----------
952 fn : callable
953 The funtion to compose, and its default implementation.
954 name : str, optional
955 The name of the composed function. If not provided, the name of the
956 function will be used.
957 """
958 if fn is None:
959 return functools.partial(compose, name=name)
960 return functools.wraps(fn)(Composed(fn, name))
961
962
963 # ---------------------- special top level functions ------------------------ #
964
965
966 @compose
967 def shape(x):
968 """Get the shape of an array as a tuple of int. This should be preferred
969 to calling `x.shape` directly, as it:
970
971 1. Allows customization (e.g. for torch and aesara which return
972 different types for shape - use `@shape.register(backend)` to
973 customize the behavior from this default implementation).
974 2. Can be used on nested lists and tuples, without calling numpy.
975
976 Parameters
977 ----------
978 x : array_like
979 The array to get the shape of. It can be an arbitrary nested list or
980 tuple of arrays and scalars, but is assumed not to be ragged.
981
982 Returns
983 -------
984 shape : tuple of int
985 The size of each dimension of the array.
986 """
987 try:
988 return x.shape
989 except AttributeError:
990 # want to handle builtins / nested stuff
991 if isinstance(x, (list, tuple)):
992 d = len(x)
993 if d != 0:
994 # NB: slightly different from np.shape, as we do not check for
995 # ragged arrays, but that behavior is seemingly deprecated
996 return (d,) + shape(x[0])
997 return (d,)
998 return ()
999
1000
1001 @compose
1002 def ndim(x):
1003 """Get the number of dimensions of an array. This should be preferred to
1004 calling `x.ndim`, since not all backends implement that, and it can also be
1005 called on nested lists and tuples.
1006
1007 Parameters
1008 ----------
1009 x : array_like
1010 The array to get the number of dimensions of. It can be an arbitrary
1011 nested list or tuple of arrays and scalars.
1012
1013 Returns
1014 -------
1015 ndim : int
1016 """
1017 try:
1018 return x.ndim
1019 except AttributeError:
1020 return len(shape(x))
1021
1022
1023 @compose
1024 def size(x):
1025 """Get the size, or number of elements, of an array. This should be
1026 preferred to calling `x.size`, since not all backends implement that, and
1027 it can also be called on nested lists and tuples.
1028
1029 Parameters
1030 ----------
1031 x : array_like
1032 The array to get the size of. It can be an arbitrary nested list or
1033 tuple of arrays and scalars.
1034
1035 Returns
1036 -------
1037 size : int
1038 """
1039 try:
1040 return x.size
1041 except AttributeError:
1042 return math.prod(shape(x))
1043
1044
1045 def conj(x):
1046 """Array conjugate."""
1047 return do("conj", x)
1048
1049
1050 def transpose(x, *args):
1051 """Array transpose."""
1052 return do("transpose", x, *args)
1053
1054
1055 def dag(x):
1056 """Array Hermitian transpose."""
1057 try:
1058 return x.H
1059 except AttributeError:
1060 backend = _infer_class_backend_cached(x.__class__)
1061 return do("conj", do("transpose", x, like=backend), like=backend)
1062
1063
1064 def real(x):
1065 """Array real part."""
1066 return do("real", x)
1067
1068
1069 def imag(x):
1070 """Array imaginary part."""
1071 return do("imag", x)
1072
1073
1074 def reshape(x, shape):
1075 """Array reshaped."""
1076 try:
1077 return x.reshape(shape)
1078 except AttributeError:
1079 return do("reshape", x, shape)
1080
1081
1082 def to_backend_dtype(dtype_name, like):
1083 """Turn string specifier ``dtype_name`` into dtype of backend ``like``."""
1084 if not isinstance(like, str):
1085 like = _infer_class_backend_cached(like.__class__)
1086
1087 try:
1088 return get_lib_fn(like, dtype_name)
1089 except ImportError:
1090 return dtype_name
1091
1092
1093 @compose
1094 def get_dtype_name(x):
1095 """Find string specifier ``dtype_name`` of array ``x``."""
1096 dtype = x.dtype
1097 try:
1098 return dtype.name
1099 except AttributeError:
1100 return str(dtype)
1101
1102
1103 _COMPLEX_DTYPES = {"complex64", "complex128"}
1104 _DOUBLE_DTYPES = {"float64", "complex128"}
1105 _DTYPE_MAP = {
1106 (False, False): "float32",
1107 (False, True): "float64",
1108 (True, False): "complex64",
1109 (True, True): "complex128",
1110 }
1111
1112
1113 def get_common_dtype(*arrays):
1114 """Compute the minimal dtype sufficient for ``arrays``."""
1115 dtypes = set(map(get_dtype_name, arrays))
1116 has_complex = not _COMPLEX_DTYPES.isdisjoint(dtypes)
1117 has_double = not _DOUBLE_DTYPES.isdisjoint(dtypes)
1118 return _DTYPE_MAP[has_complex, has_double]
1119
1120
1121 def astype(x, dtype_name, **kwargs):
1122 """Cast array as type ``dtype_name`` - tries ``x.astype`` first."""
1123 dtype = to_backend_dtype(dtype_name, like=x)
1124 try:
1125 return x.astype(dtype, **kwargs)
1126 except AttributeError:
1127 return do("astype", x, dtype, **kwargs)
1128
1129
1130 def to_numpy(x):
1131 """Get a numpy version of array ``x``."""
1132 return do("to_numpy", x)
1133
1134
1135 # -------------------------- some common wrappers --------------------------- #
1136
1137
1138 def svd_not_full_matrices_wrapper(fn):
1139 @functools.wraps(fn)
1140 def default_not_full_matrices(*args, **kwargs):
1141 kwargs.setdefault("full_matrices", False)
1142 return fn(*args, **kwargs)
1143
1144 return default_not_full_matrices
1145
1146
1147 def svd_sUV_to_UsVH_wrapper(fn):
1148 @functools.wraps(fn)
1149 def numpy_like(*args, **kwargs):
1150 s, U, V = fn(*args, **kwargs)
1151 return U, s, dag(V)
1152
1153 return numpy_like
1154
1155
1156 def svd_UsV_to_UsVH_wrapper(fn):
1157 @functools.wraps(fn)
1158 def numpy_like(*args, **kwargs):
1159 U, s, V = fn(*args, **kwargs)
1160 return U, s, dag(V)
1161
1162 return numpy_like
1163
1164
1165 def svd_manual_full_matrices_kwarg(fn):
1166 @functools.wraps(fn)
1167 def numpy_like(*args, full_matrices=False, **kwargs):
1168 U, s, VH = fn(*args, **kwargs)
1169
1170 if not full_matrices:
1171 U, VH = U[:, : s.size], VH[: s.size, :]
1172
1173 return U, s, VH
1174
1175 return numpy_like
1176
1177
1178 def qr_allow_fat(fn):
1179 @functools.wraps(fn)
1180 def numpy_like(a, **kwargs):
1181 m, n = shape(a)
1182
1183 if m >= n:
1184 # square or thin
1185 return fn(a, **kwargs)
1186
1187 Q, R_sq = fn(a[:, :m])
1188 R_r = dag(Q) @ a[:, m:]
1189 R = do("concatenate", (R_sq, R_r), axis=1, like=a)
1190
1191 return Q, R
1192
1193 return numpy_like
1194
1195
1196 def tril_to_band_part(fn):
1197 @functools.wraps(fn)
1198 def numpy_like(x, k=0):
1199 if k < 0:
1200 raise ValueError(
1201 "'k' must be positive to recreate 'numpy.tril' "
1202 "behaviour with 'tensorflow.matrix_band_part'."
1203 )
1204
1205 return fn(x, -1, k)
1206
1207 return numpy_like
1208
1209
1210 def triu_to_band_part(fn):
1211 @functools.wraps(fn)
1212 def numpy_like(x, k=0):
1213 if k > 0:
1214 raise ValueError(
1215 "'k' must be negative to recreate 'numpy.triu' "
1216 "behaviour with 'tensorflow.matrix_band_part'."
1217 )
1218
1219 return fn(x, -k, -1)
1220
1221 return numpy_like
1222
1223
1224 def cholesky_lower(fn):
1225 @functools.wraps(fn)
1226 def cholesky_numpy_like(a):
1227 return fn(a, lower=True)
1228
1229 return cholesky_numpy_like
1230
1231
1232 def binary_allow_1d_rhs_wrap(fn):
1233 @functools.wraps(fn)
1234 def allow_1d_rhs(a, b):
1235 need_to_convert = ndim(a) != ndim(b)
1236 if need_to_convert:
1237 b = reshape(b, (*shape(b), 1))
1238 x = fn(a, b)
1239 if need_to_convert:
1240 x = reshape(x, shape(x)[:-1])
1241 return x
1242
1243 return allow_1d_rhs
1244
1245
1246 def scale_random_uniform_manually(fn):
1247 @functools.wraps(fn)
1248 def numpy_like(low=0.0, high=1.0, size=None, dtype=None, **kwargs):
1249 if size is None:
1250 size = ()
1251
1252 x = fn(size, **kwargs)
1253
1254 if (low != 0.0) or (high != 1.0):
1255 x = (high - low) * x + low
1256
1257 if (dtype is not None) and get_dtype_name(x) != dtype:
1258 x = astype(x, dtype)
1259 return x
1260
1261 return numpy_like
1262
1263
1264 def scale_random_normal_manually(fn):
1265 @functools.wraps(fn)
1266 def numpy_like(loc=0.0, scale=1.0, size=None, dtype=None, **kwargs):
1267 if size is None:
1268 size = ()
1269
1270 x = fn(size, **kwargs)
1271
1272 if (loc != 0.0) or (scale != 1.0):
1273 x = scale * x + loc
1274
1275 if (dtype is not None) and get_dtype_name(x) != dtype:
1276 x = astype(x, dtype)
1277 return x
1278
1279 return numpy_like
1280
1281
1282 def with_dtype_wrapper(fn):
1283 """Add ability to handle `dtype` keyword.
1284 If not None, `dtype` should be specified as a string, otherwise conversion
1285 will happen regardless.
1286 """
1287
1288 @functools.wraps(fn)
1289 def with_dtype(*args, dtype=None, **kwargs):
1290 A = fn(*args, **kwargs)
1291 if (dtype is not None) and (dtype != get_dtype_name(A)):
1292 A = astype(A, dtype)
1293 return A
1294
1295 return with_dtype
1296
1297
1298 def translate_wrapper(fn, translator):
1299 """Wrap a function to match the api of another according to a translation.
1300 The ``translator`` entries in the form of an ordered dict should have
1301 entries like:
1302
1303 (desired_kwarg: (backend_kwarg, default_value))
1304
1305 with the order defining the args of the function.
1306 """
1307
1308 @functools.wraps(fn)
1309 def translated_function(*args, **kwargs):
1310 new_kwargs = {}
1311 translation = translator.copy()
1312
1313 # convert args, pairing them off with kwargs
1314 for arg_value in args:
1315 new_arg_name = translation.popitem(last=False)[1][0]
1316 new_kwargs[new_arg_name] = arg_value
1317
1318 # convert kwargs - but only those in the translation
1319 for key, value in kwargs.items():
1320 try:
1321 new_kwargs[translation.pop(key)[0]] = value
1322 except KeyError:
1323 new_kwargs[key] = value
1324
1325 # set remaining default kwargs
1326 for opt in translation.values():
1327 if len(opt) == 2:
1328 # backend_name, default_value
1329 new_kwargs[opt[0]] = opt[1]
1330 # else, no default value -> don't inject
1331
1332 return fn(**new_kwargs)
1333
1334 return translated_function
1335
1336
1337 def make_translator(t):
1338 return functools.partial(translate_wrapper, translator=OrderedDict(t))
1339
1340
1341 def complex_add_re_im(re, im):
1342 return re + 1j * im
1343
1344
1345 def allclose(x, y, rtol=1e-05, atol=1e-08):
1346 return do("all", do("abs", x - y) <= atol + rtol * do("abs", y))
1347
1348
1349 def _handle_size_to_shape(size=None):
1350 if size is None:
1351 return ()
1352 try:
1353 return tuple(size)
1354 except TypeError:
1355 return (size,)
1356
1357
1358 # ----------------------------- Custom dispatchers -------------------------- #
1359
1360
1361 def wrap_args_kwargs_from_raw(fn):
1362 """Take a function with signature ``(*args, **kwargs)`` and wrap it to
1363 accept a single tuple of args and a dict of kwargs.
1364 """
1365
1366 @functools.wraps(fn)
1367 def wrapped(args, kwargs):
1368 return fn(*args, **kwargs)
1369
1370 return wrapped
1371
1372
1373 def register_dispatch(fun, dispatcher, raw_signature=True):
1374 """Register a new dispatcher, a function that takes the arguments and
1375 keyword arguments of a function and returns the backend to use, when the
1376 backend is not explicitly given.
1377
1378 This is useful in case the backend to be used by a function cannot be
1379 inferred from the first argument.
1380
1381 Parameters
1382 ----------
1383 fun : str
1384 The name of the function to register the dispatcher for.
1385 dispatcher : callable
1386 The dispatcher function to use. This should take the arguments and
1387 keyword arguments of the function and return the backend to use.
1388 raw_signature : bool, optional
1389 The ``dispatcher`` has signature ``(*args, **kwargs)`` if ``True``,
1390 otherwise it has signature ``(args, kwargs)``.
1391 """
1392 if raw_signature:
1393 dispatcher = wrap_args_kwargs_from_raw(dispatcher)
1394
1395 _DISPATCHERS[fun] = dispatcher
1396
1397
1398 def default_dispatcher(args, kwargs):
1399 """Try to infer backend from first argument passed to function."""
1400 try:
1401 return _infer_class_backend_cached(args[0].__class__)
1402 except IndexError:
1403 raise TypeError("No args to infer backend from.")
1404
1405
1406 # lookup of custom dispatcher methods, for cases when backend cannot be
1407 # inferred accurately from first argument.
1408 _DISPATCHERS = defaultdict(lambda: default_dispatcher)
1409
1410
1411 def join_array_dispatcher(args, kwargs):
1412 """Dispatcher for functions where first argument is a sequence."""
1413 try:
1414 return _infer_class_backend_cached(args[0][0].__class__)
1415 except (TypeError, ValueError):
1416 # user passed an empty sequence, or something non-iterable
1417 # try to infer backend from first argument as fallback
1418 return _infer_class_backend_cached(args[0].__class__)
1419
1420
1421 # List of functions listed in numpy API as array joining operations
1422 register_dispatch("concatenate", join_array_dispatcher, raw_signature=False)
1423 register_dispatch("stack", join_array_dispatcher, raw_signature=False)
1424 register_dispatch("block", join_array_dispatcher, raw_signature=False)
1425 register_dispatch("vstack", join_array_dispatcher, raw_signature=False)
1426 register_dispatch("hstack", join_array_dispatcher, raw_signature=False)
1427 register_dispatch("dstack", join_array_dispatcher, raw_signature=False)
1428 register_dispatch("column_stack", join_array_dispatcher, raw_signature=False)
1429 register_dispatch("row_stack", join_array_dispatcher, raw_signature=False)
1430
1431
1432 def einsum_dispatcher(args, kwargs):
1433 """Dispatcher for handling einsum.
1434
1435 einsum can be called with a str equation as the first argument, or with
1436 'interleaved' inputs. This dispatcher handles both cases and also takes
1437 into account all arrays.
1438 """
1439 return infer_backend_multi(*args)
1440
1441
1442 register_dispatch("einsum", einsum_dispatcher, raw_signature=False)
1443
1444
1445 def binary_dispatcher(args, kwargs):
1446 """There are cases when we want to take into account both backends of two
1447 arguments, e.g. a lazy variable and a constant array.
1448 """
1449 return infer_backend_multi(*args[:2])
1450
1451
1452 register_dispatch("tensordot", binary_dispatcher, raw_signature=False)
1453 register_dispatch("matmul", binary_dispatcher, raw_signature=False)
1454 register_dispatch("multiply", binary_dispatcher, raw_signature=False)
1455 register_dispatch("divide", binary_dispatcher, raw_signature=False)
1456 register_dispatch("true_divide", binary_dispatcher, raw_signature=False)
1457 register_dispatch("add", binary_dispatcher, raw_signature=False)
1458 register_dispatch("subtract", binary_dispatcher, raw_signature=False)
1459
1460 # TODO: register other binary functions?
1461
1462 # --------------- object to act as drop-in replace for numpy ---------------- #
1463
1464
1465 class InjectDtypeDevice:
1466 """Wrapper that injects defaultdtype and device arguments into a function"""
1467
1468 __slots__ = ("_fn", "_device", "_dtype")
1469
1470 def __init__(self, fn, device=None, dtype=None):
1471 self._fn = fn
1472 self._device = device
1473 self._dtype = dtype
1474
1475 def __call__(self, *args, **kwargs):
1476 if self._device is not None and "device" not in kwargs:
1477 kwargs["device"] = self._device
1478 if self._dtype is not None and "dtype" not in kwargs:
1479 kwargs["dtype"] = self._dtype
1480 return self._fn(*args, **kwargs)
1481
1482 def __repr__(self):
1483 return (
1484 f"InjectDtypeDevice(fn={self._fn}, "
1485 f"device={self._device}, "
1486 f"dtype={self._dtype})"
1487 )
1488
1489
1490 class AutoNamespace:
1491 """Mimics a namespace, optionally for a specific backend, device, and
1492 dtype, caching the lookup of functions, and injecting default device and
1493 dtype arguments for certain creation routines.
1494
1495 Parameters
1496 ----------
1497 like : array_like, str, or None
1498 The backend to use, or an object to infer the backend from. If None,
1499 the default behavior is to use `autoray.do` and auto dispatch backend
1500 at function call time. If given, the functions are cached at first
1501 call.
1502 device : str, optional
1503 The device to use for the backend. If None, it will be inferred from
1504 the `like` paramater is that is array-like or set to None.
1505 dtype : str, optional
1506 The dtype to use for the backend. If None, it will be inferred from
1507 the `like` parameter if that is array-like or set to None.
1508 submodule : str, optional
1509 This is used internally when nesting attribute lookups, e.g.
1510 `xp.random.normal`, `xp.linalg.eigh`.
1511 """
1512
1513 def __init__(
1514 self,
1515 like=None,
1516 device=None,
1517 dtype=None,
1518 submodule=None,
1519 ):
1520 if like is None:
1521 # use autoray.do and auto dispatch
1522 self._backend = None
1523 elif isinstance(like, str):
1524 # commit to a specific given backend
1525 self._backend = like
1526 else:
1527 # commit to a specific backend inferred from array-like
1528 self._backend = _infer_class_backend_cached(like.__class__)
1529
1530 if device is None:
1531 if like is not None:
1532 if hasattr(like, "device"):
1533 device = like.device
1534 self._device = device
1535
1536 if dtype is None:
1537 if like is not None:
1538 if hasattr(like, "dtype"):
1539 dtype = like.dtype
1540 self._dtype = dtype
1541 self._submodule = submodule
1542
1543 def _get_submodule(self, name):
1544 new = object.__new__(type(self))
1545 new._backend = self._backend
1546 new._device = self._device
1547 new._dtype = self._dtype
1548 new._submodule = name
1549 return new
1550
1551 def _get_fn(self, name):
1552 if self._submodule is not None:
1553 # prepend the submodule name
1554 name = f"{self._submodule}.{name}"
1555
1556 if name in ("random", "linalg"):
1557 # note that other submodules can be accessed, these are just the
1558 # ones with functions that we possibly want to translate
1559 return self._get_submodule(name)
1560
1561 if self._backend is None:
1562 # use autoray.do and auto dispatch
1563 return functools.partial(do, name)
1564
1565 fn = get_lib_fn(self._backend, name)
1566
1567 # possibly wrap for dtype and device injection
1568 if name in _CREATION_ROUTINES:
1569 inject_dtype, inject_device = _CREATION_INJECT.get(
1570 (self._backend, fn), (True, False)
1571 )
1572
1573 if not inject_dtype:
1574 # this is not a function accepts dtype
1575 dtype_to_inject = None
1576 else:
1577 dtype_to_inject = self._dtype
1578
1579 if not inject_device:
1580 # this is not a function accepts device
1581 device_to_inject = None
1582 else:
1583 device_to_inject = self._device
1584
1585 if (dtype_to_inject is not None) or (device_to_inject is not None):
1586 # only wrap if we actually inject something
1587 fn = InjectDtypeDevice(fn, device_to_inject, dtype_to_inject)
1588
1589 return fn
1590
1591 def __getattribute__(self, name):
1592 try:
1593 return object.__getattribute__(self, name)
1594 except AttributeError:
1595 x = self._get_fn(name)
1596 super().__getattribute__("__dict__")[name] = x
1597 return x
1598
1599 def __repr__(self):
1600 return (
1601 f"{self.__class__.__name__}("
1602 f"backend={self._backend}, "
1603 f"device={self._device}, "
1604 f"dtype={self._dtype}, "
1605 f"submodule={self._submodule}"
1606 ")"
1607 )
1608
1609
1610 numpy = AutoNamespace()
1611
1612
1613 def get_namespace(like=None, device=None, dtype=None, submodule=None):
1614 """Get an automatic namespace object.
1615
1616 If `like` is None, the namespace essentially provides an alternative syntax
1617 to `do`, dispatching each function at calltime, and allowing the backend
1618 and function implementations to be dynamically updated.
1619
1620 If `like` is supplied however, the backend is eagerly dispatched and
1621 functions are loaded and cached specifically for that backend. In this
1622 case, default `device` and `dtype` can also be specified for various array
1623 creation routines, or if `like` is an array, inferred from that.
1624
1625 Parameters
1626 ----------
1627 like : array-like, str or None, optional
1628 An array-like object to dispatch on, an explicit backend name, or None.
1629 device : str or None, optional
1630 The device to use for array creation, or None to infer from `like`.
1631 dtype : str or None, optional
1632 The data type to use for array creation, or None to infer from `like`.
1633
1634 Returns
1635 -------
1636 AutoNamespace
1637 An automatic namespace object.
1638 """
1639 return AutoNamespace(
1640 like=like,
1641 device=device,
1642 dtype=dtype,
1643 submodule=submodule,
1644 )
1645
1646
1647 # --------------------------------------------------------------------------- #
1648 # specific functions #
1649 # --------------------------------------------------------------------------- #
1650
1651 # ------------------------------ standard-lib ------------------------------- #
1652
1653 _MODULE_ALIASES["decimal"] = "math"
1654 _MODULE_ALIASES["builtins"] = "numpy"
1655
1656
1657 _builtin_dtype_lookup = {
1658 int: "int64",
1659 float: "float64",
1660 complex: "complex128",
1661 }
1662
1663
1664 @get_dtype_name.register("builtins")
1665 def builtins_get_dtype_name(x):
1666 return _builtin_dtype_lookup[x.__class__]
1667
1668
1669 _FUNCS["builtins", "complex"] = complex
1670
1671 # ---------------------------------- numpy ---------------------------------- #
1672
1673
1674 def numpy_to_numpy(x):
1675 return do("asarray", x, like="numpy")
1676
1677
1678 _MODULE_ALIASES["numpy.scipy"] = "scipy"
1679 _FUNCS["numpy", "to_numpy"] = numpy_to_numpy
1680 _FUNCS["numpy", "complex"] = complex_add_re_im
1681 _FUNCS["builtins", "to_numpy"] = numpy_to_numpy
1682 _SUBMODULE_ALIASES["numpy", "linalg.lu"] = "scipy.linalg"
1683 _SUBMODULE_ALIASES["numpy", "linalg.expm"] = "scipy.linalg"
1684 _CUSTOM_WRAPPERS["numpy", "linalg.svd"] = svd_not_full_matrices_wrapper
1685 _CUSTOM_WRAPPERS["numpy", "random.normal"] = with_dtype_wrapper
1686 _CUSTOM_WRAPPERS["numpy", "random.uniform"] = with_dtype_wrapper
1687
1688 # ---------------------------------- cupy ----------------------------------- #
1689
1690
1691 def cupy_to_numpy(x): # pragma: no cover
1692 return x.get()
1693
1694
1695 _MODULE_ALIASES["cupy.scipy"] = "cupyx.scipy"
1696 _FUNCS["cupy", "to_numpy"] = cupy_to_numpy
1697 _FUNCS["cupy", "complex"] = complex_add_re_im
1698 _CUSTOM_WRAPPERS["cupy", "linalg.svd"] = svd_not_full_matrices_wrapper
1699
1700 # ----------------------------------- jax ----------------------------------- #
1701
1702
1703 def jax_to_numpy(x):
1704 return do("asarray", x, like="numpy")
1705
1706
1707 class JaxDefaultRNG:
1708 """Stateful but deterministic random number generator for JAX following
1709 numpy's Generator API, compatible with `jax.jit`.
1710 """
1711
1712 def __init__(self, seed, **kwargs):
1713 import jax
1714
1715 self.jax = jax
1716 self.key = jax.random.key(seed, **kwargs)
1717
1718 def binomial(self, n, p, size=None, **kwargs):
1719 self.key, subkey = self.jax.random.split(self.key)
1720 shape = _handle_size_to_shape(size)
1721 return self.jax.random.binomial(
1722 subkey, n=n, p=p, shape=shape, **kwargs
1723 )
1724
1725 def choice(self, a, size=None, replace=True, p=None, axis=0, **kwargs):
1726 self.key, subkey = self.jax.random.split(self.key)
1727 shape = _handle_size_to_shape(size)
1728 return self.jax.random.choice(
1729 subkey,
1730 a,
1731 shape=shape,
1732 replace=replace,
1733 p=p,
1734 axis=axis,
1735 **kwargs,
1736 )
1737
1738 def exponential(self, scale=1.0, size=None, **kwargs):
1739 self.key, subkey = self.jax.random.split(self.key)
1740 shape = _handle_size_to_shape(size)
1741 x = self.jax.random.exponential(subkey, shape=shape, **kwargs)
1742 if scale != 1.0:
1743 x *= scale
1744 return x
1745
1746 def gumbel(self, loc=0.0, scale=1.0, size=None, **kwargs):
1747 self.key, subkey = self.jax.random.split(self.key)
1748 shape = _handle_size_to_shape(size)
1749 x = self.jax.random.gumbel(subkey, shape=shape, **kwargs)
1750 if scale != 1.0:
1751 x *= scale
1752 if loc != 0.0:
1753 x += loc
1754 return x
1755
1756 def integers(self, low, high=None, size=None, **kwargs):
1757 self.key, subkey = self.jax.random.split(self.key)
1758 shape = _handle_size_to_shape(size)
1759 if high is None:
1760 high = low
1761 low = 0
1762 return self.jax.random.randint(
1763 subkey,
1764 shape=shape,
1765 minval=low,
1766 maxval=high,
1767 **kwargs,
1768 )
1769
1770 def normal(self, loc=0.0, scale=1.0, size=None, **kwargs):
1771 self.key, subkey = self.jax.random.split(self.key)
1772 shape = _handle_size_to_shape(size)
1773 x = self.jax.random.normal(subkey, shape=shape, **kwargs)
1774 if scale != 1.0:
1775 x *= scale
1776 if loc != 0.0:
1777 x += loc
1778 return x
1779
1780 def permutation(self, x, **kwargs):
1781 self.key, subkey = self.jax.random.split(self.key)
1782 return self.jax.random.permutation(subkey, x, **kwargs)
1783
1784 def poisson(self, lam=1.0, size=None, **kwargs):
1785 self.key, subkey = self.jax.random.split(self.key)
1786 shape = _handle_size_to_shape(size)
1787 x = self.jax.random.poisson(subkey, lam, shape=shape, **kwargs)
1788 return x
1789
1790 def random(self, size=None, **kwargs):
1791 return self.uniform(size=size, **kwargs)
1792
1793 def uniform(self, low=0.0, high=1.0, size=None, **kwargs):
1794 self.key, subkey = self.jax.random.split(self.key)
1795 shape = _handle_size_to_shape(size)
1796 return self.jax.random.uniform(
1797 subkey, shape=shape, minval=low, maxval=high, **kwargs
1798 )
1799
1800
1801 def jax_default_rng(seed, **kwargs):
1802 if isinstance(seed, JaxDefaultRNG):
1803 return seed
1804 return JaxDefaultRNG(seed, **kwargs)
1805
1806
1807 register_function("jax", "random.default_rng", jax_default_rng)
1808 register_backend(JaxDefaultRNG, "jax")
1809
1810
1811 _JAX_RANDOM_KEY = None
1812
1813
1814 def jax_random_seed(seed=None):
1815 from jax.random import PRNGKey
1816
1817 global _JAX_RANDOM_KEY
1818 if seed is None:
1819 from random import SystemRandom
1820
1821 seed = SystemRandom().randint(-(2**63), 2**63 - 1) # inclusive high
1822 _JAX_RANDOM_KEY = PRNGKey(seed)
1823
1824
1825 def jax_random_get_key():
1826 from jax.random import split
1827
1828 global _JAX_RANDOM_KEY
1829 if _JAX_RANDOM_KEY is None:
1830 jax_random_seed()
1831 _JAX_RANDOM_KEY, subkey = split(_JAX_RANDOM_KEY)
1832 return subkey
1833
1834
1835 def jax_random_uniform(low=0.0, high=1.0, size=None, **kwargs):
1836 from jax.random import uniform
1837
1838 if size is None:
1839 size = ()
1840 return uniform(
1841 jax_random_get_key(), shape=size, minval=low, maxval=high, **kwargs
1842 )
1843
1844
1845 def jax_random_normal(loc=0.0, scale=1.0, size=None, **kwargs):
1846 from jax.random import normal
1847
1848 if size is None:
1849 size = ()
1850 x = normal(jax_random_get_key(), shape=size, **kwargs)
1851 if scale != 1.0:
1852 x *= scale
1853 if loc != 0.0:
1854 x += loc
1855 return x
1856
1857
1858 _BACKEND_ALIASES["jaxlib"] = "jax"
1859 _MODULE_ALIASES["jax.scipy"] = "jax.scipy"
1860 _MODULE_ALIASES["jax"] = "jax.numpy"
1861 _SUBMODULE_ALIASES["jax", "complex"] = "jax.lax"
1862 _SUBMODULE_ALIASES["jax", "linalg.expm"] = "jax.scipy.linalg"
1863 _SUBMODULE_ALIASES["jax", "linalg.householder_product"] = "jax.lax.linalg"
1864 # n.b. jax supports fat QR but not when computing gradients
1865 _CUSTOM_WRAPPERS["jax", "linalg.qr"] = qr_allow_fat
1866 _CUSTOM_WRAPPERS["jax", "linalg.svd"] = svd_not_full_matrices_wrapper
1867 _FUNCS["jax", "to_numpy"] = jax_to_numpy
1868 _FUNCS["jax", "random.seed"] = jax_random_seed
1869 _FUNCS["jax", "random.uniform"] = jax_random_uniform
1870 _FUNCS["jax", "random.normal"] = jax_random_normal
1871
1872
1873 # --------------------------------- aesara ---------------------------------- #
1874
1875
1876 @shape.register("aesara")
1877 def aesara_shape(x):
1878 return x.type.shape
1879
1880
1881 _MODULE_ALIASES["aesara"] = "aesara.tensor"
1882 _FUNCS["aesara", "shape"] = aesara_shape
1883
1884
1885 # -------------------------------- autograd --------------------------------- #
1886
1887 _MODULE_ALIASES["autograd"] = "autograd.numpy"
1888 _CUSTOM_WRAPPERS["autograd", "linalg.svd"] = svd_not_full_matrices_wrapper
1889 _FUNCS["autograd", "complex"] = complex_add_re_im
1890
1891
1892 # ---------------------------------- dask ----------------------------------- #
1893
1894
1895 def dask_to_numpy(x):
1896 return x.compute()
1897
1898
1899 def dask_eye_wrapper(fn):
1900 # Make M work as positional argument
1901 @functools.wraps(fn)
1902 def numpy_like(N, M=None, **kwargs):
1903 return fn(N, M=M, **kwargs)
1904
1905 return numpy_like
1906
1907
1908 _FUNCS["dask", "to_numpy"] = dask_to_numpy
1909 _FUNCS["dask", "complex"] = complex_add_re_im
1910 _FUNC_ALIASES["dask", "abs"] = "absolute"
1911 _FUNC_ALIASES["dask", "identity"] = "eye"
1912 _MODULE_ALIASES["dask"] = "dask.array"
1913 _CUSTOM_WRAPPERS["dask", "linalg.svd"] = svd_manual_full_matrices_kwarg
1914 _CUSTOM_WRAPPERS["dask", "linalg.cholesky"] = cholesky_lower
1915 _CUSTOM_WRAPPERS["dask", "random.normal"] = with_dtype_wrapper
1916 _CUSTOM_WRAPPERS["dask", "random.uniform"] = with_dtype_wrapper
1917 _CUSTOM_WRAPPERS["dask", "eye"] = dask_eye_wrapper
1918
1919 # ---------------------------------- mars ----------------------------------- #
1920
1921
1922 def mars_to_numpy(x):
1923 return x.to_numpy()
1924
1925
1926 _FUNCS["mars", "to_numpy"] = mars_to_numpy
1927 _FUNCS["mars", "complex"] = complex_add_re_im
1928 _MODULE_ALIASES["mars"] = "mars.tensor"
1929 _CUSTOM_WRAPPERS["mars", "linalg.cholesky"] = cholesky_lower
1930
1931
1932 # ----------------------------------- ctf ----------------------------------- #
1933
1934
1935 def ctf_array(x):
1936 return do("astensor", x, like="ctf")
1937
1938
1939 def ctf_to_numpy(x):
1940 return x.to_nparray()
1941
1942
1943 def ctf_count_nonzero(x):
1944 return (x != 0).astype(int).sum()
1945
1946
1947 @get_dtype_name.register("ctf")
1948 def ctf_get_dtype_name(x):
1949 return x.dtype.__name__
1950
1951
1952 _FUNCS["ctf", "array"] = ctf_array
1953 _FUNCS["ctf", "complex"] = complex_add_re_im
1954 _FUNCS["ctf", "allclose"] = allclose
1955 _FUNCS["ctf", "to_numpy"] = ctf_to_numpy
1956 _FUNCS["ctf", "count_nonzero"] = ctf_count_nonzero
1957
1958 _SUBMODULE_ALIASES["ctf", "float32"] = "numpy"
1959 _SUBMODULE_ALIASES["ctf", "float64"] = "numpy"
1960 _SUBMODULE_ALIASES["ctf", "complex64"] = "numpy"
1961 _SUBMODULE_ALIASES["ctf", "complex128"] = "numpy"
1962 _SUBMODULE_ALIASES["ctf", "linalg.svd"] = "ctf"
1963 _SUBMODULE_ALIASES["ctf", "linalg.eigh"] = "ctf"
1964 _SUBMODULE_ALIASES["ctf", "linalg.qr"] = "ctf"
1965 _SUBMODULE_ALIASES["ctf", "linalg.norm"] = "ctf"
1966
1967 _FUNC_ALIASES["ctf", "random.uniform"] = "random"
1968
1969 _CUSTOM_WRAPPERS["ctf", "random.uniform"] = scale_random_uniform_manually
1970
1971
1972 # ------------------------------- sparse------------------------------------- #
1973
1974
1975 def sparse_array(x):
1976 return do("COO.from_numpy", x, like="sparse")
1977
1978
1979 def sparse_to_numpy(x):
1980 return x.todense()
1981
1982
1983 def sparse_transpose(x, axes=None):
1984 return x.transpose(axes)
1985
1986
1987 def sparse_reshape(x, shape):
1988 return x.reshape(shape)
1989
1990
1991 def sparse_sum(x, axis=None, keepdims=False, dtype=None, out=None):
1992 return x.sum(axis=axis, keepdims=keepdims, dtype=dtype, out=out)
1993
1994
1995 def sparse_prod(x, axis=None, keepdims=False, dtype=None, out=None):
1996 return x.prod(axis=axis, keepdims=keepdims, dtype=dtype, out=out)
1997
1998
1999 def sparse_conj(x):
2000 return x.conj()
2001
2002
2003 def sparse_real(x):
2004 return x.real
2005
2006
2007 def sparse_imag(x):
2008 return x.imag
2009
2010
2011 def sparse_count_nonzero(x):
2012 return x.nnz
2013
2014
2015 def sparse_random_uniform(low=0.0, high=1.0, size=None, dtype=None, **kwargs):
2016 def rvs(nnz):
2017 return do(
2018 "random.uniform", low, high, (nnz,), dtype=dtype, like="numpy"
2019 )
2020
2021 return do("random", size, data_rvs=rvs, **kwargs, like="sparse")
2022
2023
2024 def sparse_random_normal(loc=0.0, scale=1.0, size=None, dtype=None, **kwargs):
2025 def rvs(nnz):
2026 return do(
2027 "random.normal", loc, scale, (nnz,), dtype=dtype, like="numpy"
2028 )
2029
2030 return do("random", size, data_rvs=rvs, **kwargs, like="sparse")
2031
2032
2033 _FUNCS["sparse", "array"] = sparse_array
2034 _FUNCS["sparse", "to_numpy"] = sparse_to_numpy
2035 _FUNCS["sparse", "transpose"] = sparse_transpose
2036 _FUNCS["sparse", "reshape"] = sparse_reshape
2037 _FUNCS["sparse", "sum"] = sparse_sum
2038 _FUNCS["sparse", "prod"] = sparse_prod
2039 _FUNCS["sparse", "conj"] = sparse_conj
2040 _FUNCS["sparse", "real"] = sparse_real
2041 _FUNCS["sparse", "real"] = sparse_real
2042 _FUNCS["sparse", "imag"] = sparse_imag
2043 _FUNCS["sparse", "complex"] = complex_add_re_im
2044 _FUNCS["sparse", "count_nonzero"] = sparse_count_nonzero
2045 _FUNCS["sparse", "random.uniform"] = sparse_random_uniform
2046 _FUNCS["sparse", "random.normal"] = sparse_random_normal
2047
2048 _FUNC_ALIASES["sparse", "identity"] = "eye"
2049
2050 # sparse uses numpys __array_func__ interface
2051 for f in (
2052 "log",
2053 "log2",
2054 "log10",
2055 "exp",
2056 "sqrt",
2057 "sign",
2058 "sin",
2059 "cos",
2060 "tan",
2061 "arcsin",
2062 "arccos",
2063 "arctan",
2064 "sinh",
2065 "cosh",
2066 "tanh",
2067 "arcsinh",
2068 "arccosh",
2069 "arctanh",
2070 "tensordot",
2071 # NB put tensordot here, as sparse.tensordot can produce dense (numpy)
2072 # arrays but errors when both inputs are dense - we want nested calls to
2073 # tensordot to handle this
2074 ):
2075 _SUBMODULE_ALIASES["sparse", f] = "numpy"
2076
2077
2078 # ------------------------------- tensorflow -------------------------------- #
2079
2080
2081 def tensorflow_to_numpy(x):
2082 return x.numpy()
2083
2084
2085 def tensorflow_indices(dimensions):
2086 _meshgrid = get_lib_fn("tensorflow", "meshgrid")
2087 _arange = get_lib_fn("tensorflow", "arange")
2088 return _meshgrid(*map(_arange, dimensions), indexing="ij")
2089
2090
2091 _MODULE_ALIASES["tensorflow.linalg"] = "tensorflow.linalg"
2092 _MODULE_ALIASES["tensorflow.random"] = "tensorflow.random"
2093 _MODULE_ALIASES["tensorflow"] = "tensorflow.experimental.numpy"
2094
2095 _FUNCS["tensorflow", "to_numpy"] = tensorflow_to_numpy
2096 _FUNCS["tensorflow", "indices"] = tensorflow_indices
2097
2098 _FUNC_ALIASES["tensorflow", "astype"] = "cast"
2099 _SUBMODULE_ALIASES["tensorflow", "cast"] = "tensorflow"
2100 _SUBMODULE_ALIASES["tensorflow", "astype"] = "tensorflow"
2101 _SUBMODULE_ALIASES["tensorflow", "complex"] = "tensorflow"
2102
2103 _CUSTOM_WRAPPERS["tensorflow", "linalg.svd"] = svd_sUV_to_UsVH_wrapper
2104 _CUSTOM_WRAPPERS["tensorflow", "linalg.solve"] = binary_allow_1d_rhs_wrap
2105 _CUSTOM_WRAPPERS["tensorflow", "random.uniform"] = make_translator(
2106 [
2107 ("low", ("minval", 0.0)),
2108 ("high", ("maxval", 1.0)),
2109 ("size", ("shape", ())),
2110 ]
2111 )
2112 _CUSTOM_WRAPPERS["tensorflow", "random.normal"] = make_translator(
2113 [
2114 ("loc", ("mean", 0.0)),
2115 ("scale", ("stddev", 1.0)),
2116 ("size", ("shape", ())),
2117 ]
2118 )
2119
2120
2121 def tensorflow_pad_wrap(tf_pad):
2122 def numpy_like(array, pad_width, mode="constant", constant_values=0):
2123 if mode != "constant":
2124 raise NotImplementedError
2125
2126 try:
2127 if len(pad_width) == 1:
2128 pad_width = pad_width * ndim(array)
2129 except TypeError:
2130 pad_width = ((pad_width, pad_width),) * ndim(array)
2131
2132 return tf_pad(
2133 array, pad_width, mode="CONSTANT", constant_values=constant_values
2134 )
2135
2136 return numpy_like
2137
2138
2139 _CUSTOM_WRAPPERS["tensorflow", "pad"] = tensorflow_pad_wrap
2140 _SUBMODULE_ALIASES["tensorflow", "pad"] = "tensorflow"
2141
2142
2143 register_creation_routine("tensorflow", "linspace", inject_dtype=False)
2144
2145
2146 # ---------------------------------- torch ---------------------------------- #
2147
2148
2149 @shape.register("torch")
2150 def torch_shape(x):
2151 # torch returns a Size object, we want tuple[int]
2152 return tuple(map(int, x.shape))
2153
2154
2155 @size.register("torch")
2156 def torch_size(x):
2157 return x.numel()
2158
2159
2160 def torch_to_numpy(x):
2161 return x.detach().cpu().numpy()
2162
2163
2164 def torch_copy(x):
2165 return x.detach().clone()
2166
2167
2168 def torch_transpose(x, axes=None):
2169 if axes is None:
2170 axes = reversed(range(0, x.ndimension()))
2171 return x.permute(*axes)
2172
2173
2174 def torch_count_nonzero(x):
2175 return do("sum", x != 0, like="torch")
2176
2177
2178 def torch_astype(x, dtype):
2179 return x.to(dtype=to_backend_dtype(dtype, like=x))
2180
2181
2182 @functools.lru_cache(None)
2183 def _torch_get_dtype_name(dtype):
2184 return str(dtype).split(".")[-1]
2185
2186
2187 @get_dtype_name.register("torch")
2188 def torch_get_dtype_name(x):
2189 return _torch_get_dtype_name(x.dtype)
2190
2191
2192 def torch_real(x):
2193 # torch doesn't support calling real on real arrays
2194 try:
2195 if x.is_complex():
2196 return x.real
2197 except AttributeError:
2198 pass
2199 return x
2200
2201
2202 def torch_imag(x):
2203 # torch doesn't support calling imag on real arrays
2204 try:
2205 if x.is_complex():
2206 return x.imag
2207 except AttributeError:
2208 pass
2209 return do("zeros_like", x)
2210
2211
2212 def torch_linalg_solve_wrap(fn):
2213 @binary_allow_1d_rhs_wrap
2214 def numpy_like(a, b):
2215 return fn(b, a)[0]
2216
2217 return numpy_like
2218
2219
2220 def torch_linalg_eigh(x):
2221 return tuple(do("symeig", x, eigenvectors=True, like="torch"))
2222
2223
2224 def torch_linalg_eigvalsh(x):
2225 return do("symeig", x, eigenvectors=False, like="torch")[0]
2226
2227
2228 def torch_tensordot_wrap(fn):
2229 @functools.wraps(fn)
2230 def numpy_like(a, b, axes=2):
2231 return fn(a, b, dims=axes)
2232
2233 return numpy_like
2234
2235
2236 def torch_pad(array, pad_width, mode="constant", constant_values=0):
2237 if mode != "constant":
2238 raise NotImplementedError
2239
2240 try:
2241 # numpy takes pads like ((0, 0), (1, 1), ... (n-1, n-1))
2242 # torch takes pads like (n-1, n-1, n-2, n-2, n-3, n-3, ...)
2243 pad = tuple(itertools.chain.from_iterable(pad_width))[::-1]
2244
2245 # a single tuple was specified ((a, b),) - use for all axes
2246 if len(pad) == 2:
2247 pad = pad * array.ndimension()
2248
2249 except TypeError:
2250 # assume int
2251 pad = (pad_width,) * 2 * array.ndimension()
2252
2253 return do(
2254 "nn.functional.pad",
2255 array,
2256 pad=pad,
2257 mode=mode,
2258 value=constant_values,
2259 like="torch",
2260 )
2261
2262
2263 def torch_split_wrap(fn):
2264 # for torch >=1.8 we can use tensor_split instead, but in current stable
2265 # release this function has not been added
2266 @functools.wraps(fn)
2267 def numpy_like(ary, indices_or_sections, axis=0, **kwargs):
2268 if isinstance(indices_or_sections, int):
2269 split_size = shape(ary)[axis] // indices_or_sections
2270 return fn(ary, split_size, dim=axis, **kwargs)
2271 else:
2272 # torch.split doesn't support empty splits
2273 if len(indices_or_sections) == 0:
2274 return (ary,)
2275
2276 diff = do(
2277 "diff",
2278 indices_or_sections,
2279 prepend=0,
2280 append=shape(ary)[axis],
2281 like="numpy",
2282 )
2283 diff = list(diff)
2284 return fn(ary, diff, dim=axis)
2285
2286 return numpy_like
2287
2288
2289 def torch_zeros_ones_wrap(fn):
2290 @functools.wraps(fn)
2291 def numpy_like(shape, dtype=None, **kwargs):
2292 if dtype is not None:
2293 dtype = to_backend_dtype(dtype, like="torch")
2294 return fn(shape, dtype=dtype, **kwargs)
2295
2296 return numpy_like
2297
2298
2299 def torch_eye_wrap(fn):
2300 @functools.wraps(fn)
2301 def numpy_like(N, M=None, dtype=None, **kwargs):
2302 if dtype is not None:
2303 dtype = to_backend_dtype(dtype, like="torch")
2304 if M is not None:
2305 return fn(N, m=M, dtype=dtype, **kwargs)
2306 else:
2307 return fn(N, dtype=dtype, **kwargs)
2308
2309 return numpy_like
2310
2311
2312 def torch_sort_wrap(fn):
2313 @functools.wraps(fn)
2314 def numpy_like(a, axis=-1):
2315 return fn(a, dim=axis)[0]
2316
2317 return numpy_like
2318
2319
2320 def torch_indices(dimensions):
2321 _meshgrid = get_lib_fn("torch", "meshgrid")
2322 _arange = get_lib_fn("torch", "arange")
2323 return _meshgrid(*map(_arange, dimensions), indexing="ij")
2324
2325
2326 def torch_flip_wrap(torch_flip):
2327 def numpy_like(x, axis=None):
2328 if axis is None:
2329 dims = tuple(range(x.ndimension()))
2330 elif isinstance(axis, int):
2331 dims = (axis,)
2332 else:
2333 # already tuple/list
2334 dims = axis
2335 return torch_flip(x, dims)
2336
2337 return numpy_like
2338
2339
2340 class TorchDefaultRNG:
2341 def __init__(self, seed, device=None):
2342 import torch
2343
2344 self._torch = torch
2345 self._generator = torch.Generator(device=device)
2346 self._generator.manual_seed(seed)
2347
2348 # def binomial(self, n, p, size=None, **kwargs):
2349 # raise NotImplementedError()
2350
2351 # def choice(self, a, size=None, replace=True, p=None, axis=0, **kwargs):
2352 # raise NotImplementedError()
2353
2354 # def exponential(self, scale=1.0, size=None, **kwargs):
2355 # raise NotImplementedError()
2356
2357 # def gumbel(self, loc=0.0, scale=1.0, size=None, **kwargs):
2358 # raise NotImplementedError()
2359
2360 def integers(self, low, high=None, size=None, **kwargs):
2361 if high is None:
2362 high = low
2363 low = 0
2364 size = _handle_size_to_shape(size)
2365 return self._torch.randint(
2366 low, high, size, generator=self._generator, **kwargs
2367 )
2368
2369 # def poisson(self, lam=1.0, size=None, **kwargs):
2370 # raise NotImplementedError()
2371
2372 def normal(self, loc=0.0, scale=1.0, size=None, **kwargs):
2373 size = _handle_size_to_shape(size)
2374 x = self._torch.randn(size, generator=self._generator, **kwargs)
2375 if scale != 1.0:
2376 x = x * scale
2377 if loc != 0.0:
2378 x = x + loc
2379 return x
2380
2381 def random(self, size=None, **kwargs):
2382 size = _handle_size_to_shape(size)
2383 return self._torch.rand(size, generator=self._generator, **kwargs)
2384
2385 def permutation(self, x, **kwargs):
2386 if isinstance(x, int):
2387 return self._torch.randperm(x, generator=self._generator, **kwargs)
2388
2389 axis = kwargs.get("axis", 0)
2390 n = x.shape[axis]
2391 perm_indices = self._torch.randperm(
2392 n, generator=self._generator, device=x.device
2393 )
2394 return self._torch.index_select(x, axis, perm_indices)
2395
2396 def uniform(self, low=0.0, high=1.0, size=None, **kwargs):
2397 size = _handle_size_to_shape(size)
2398 x = self._torch.rand(size, generator=self._generator, **kwargs)
2399 if low != 0.0 or high != 1.0:
2400 x = x * (high - low) + low
2401 return x
2402
2403
2404 def torch_default_rng(seed, **kwargs):
2405 if isinstance(seed, TorchDefaultRNG):
2406 return seed
2407 return TorchDefaultRNG(seed, **kwargs)
2408
2409
2410 register_function("torch", "random.default_rng", torch_default_rng)
2411 register_backend(TorchDefaultRNG, "torch")
2412
2413
2414 _FUNCS["torch", "pad"] = torch_pad
2415 _FUNCS["torch", "real"] = torch_real
2416 _FUNCS["torch", "imag"] = torch_imag
2417 _FUNCS["torch", "astype"] = torch_astype
2418 _FUNCS["torch", "copy"] = torch_copy
2419 _FUNCS["torch", "to_numpy"] = torch_to_numpy
2420 _FUNCS["torch", "complex"] = complex_add_re_im
2421 _FUNCS["torch", "transpose"] = torch_transpose
2422 _FUNCS["torch", "count_nonzero"] = torch_count_nonzero
2423 _FUNCS["torch", "indices"] = torch_indices
2424
2425 _FUNC_ALIASES["torch", "array"] = "tensor"
2426 _FUNC_ALIASES["torch", "asarray"] = "as_tensor"
2427 _FUNC_ALIASES["torch", "clip"] = "clamp"
2428 _FUNC_ALIASES["torch", "concatenate"] = "cat"
2429 _FUNC_ALIASES["torch", "conjugate"] = "conj"
2430 _FUNC_ALIASES["torch", "equal"] = "eq"
2431 _FUNC_ALIASES["torch", "expand_dims"] = "unsqueeze"
2432 _FUNC_ALIASES["torch", "identity"] = "eye"
2433 _FUNC_ALIASES["torch", "linalg.expm"] = "matrix_exp"
2434 _FUNC_ALIASES["torch", "max"] = "amax"
2435 _FUNC_ALIASES["torch", "min"] = "amin"
2436 _FUNC_ALIASES["torch", "power"] = "pow"
2437 _FUNC_ALIASES["torch", "random.normal"] = "randn"
2438 _FUNC_ALIASES["torch", "random.uniform"] = "rand"
2439 _FUNC_ALIASES["torch", "scipy.linalg.expm"] = "matrix_exp"
2440 _FUNC_ALIASES["torch", "split"] = "tensor_split"
2441 _FUNC_ALIASES["torch", "take"] = "index_select"
2442 _FUNC_ALIASES["torch", "take_along_axis"] = "take_along_dim"
2443
2444 _SUBMODULE_ALIASES["torch", "linalg.expm"] = "torch"
2445 _SUBMODULE_ALIASES["torch", "scipy.linalg.expm"] = "torch"
2446 _SUBMODULE_ALIASES["torch", "random.normal"] = "torch"
2447 _SUBMODULE_ALIASES["torch", "random.uniform"] = "torch"
2448
2449 _CUSTOM_WRAPPERS["torch", "clip"] = make_translator(
2450 [("a", ("input",)), ("a_min", ("min",)), ("a_max", ("max",))]
2451 )
2452 _CUSTOM_WRAPPERS["torch", "concatenate"] = make_translator(
2453 [("arrays", ("tensors",)), ("axis", ("dim", 0))]
2454 )
2455 _CUSTOM_WRAPPERS["torch", "diagonal"] = make_translator(
2456 [("a", ("input",)), ("axis1", ("dim1", 0)), ("axis2", ("dim2", 1))]
2457 )
2458 _CUSTOM_WRAPPERS["torch", "empty"] = make_translator([("shape", ("size",))])
2459 _CUSTOM_WRAPPERS["torch", "expand_dims"] = make_translator(
2460 [("a", ("input",)), ("axis", ("dim",))]
2461 )
2462 _CUSTOM_WRAPPERS["torch", "eye"] = torch_eye_wrap
2463 _CUSTOM_WRAPPERS["torch", "flip"] = torch_flip_wrap
2464 _CUSTOM_WRAPPERS["torch", "linalg.svd"] = svd_not_full_matrices_wrapper
2465 _CUSTOM_WRAPPERS["torch", "ones"] = torch_zeros_ones_wrap
2466 _CUSTOM_WRAPPERS["torch", "random.normal"] = scale_random_normal_manually
2467 _CUSTOM_WRAPPERS["torch", "random.uniform"] = scale_random_uniform_manually
2468 _CUSTOM_WRAPPERS["torch", "sort"] = torch_sort_wrap
2469 _CUSTOM_WRAPPERS["torch", "stack"] = make_translator(
2470 [("arrays", ("tensors",)), ("axis", ("dim", 0))]
2471 )
2472 _CUSTOM_WRAPPERS["torch", "take"] = make_translator(
2473 [("a", ("input",)), ("indices", ("index",)), ("axis", ("dim",))]
2474 )
2475 _CUSTOM_WRAPPERS["torch", "tensordot"] = torch_tensordot_wrap
2476 _CUSTOM_WRAPPERS["torch", "tril"] = make_translator(
2477 [("m", ("input",)), ("k", ("diagonal", 0))]
2478 )
2479 _CUSTOM_WRAPPERS["torch", "triu"] = make_translator(
2480 [("m", ("input",)), ("k", ("diagonal", 0))]
2481 )
2482 _CUSTOM_WRAPPERS["torch", "zeros"] = torch_zeros_ones_wrap
2483 _CUSTOM_WRAPPERS["torch", "take_along_axis"] = make_translator(
2484 [("arr", ("input",)), ("indices", ("indices",)), ("axis", ("dim", -1))]
2485 )
2486
2487 _torch_reduce_translation = [
2488 ("a", ("input",)),
2489 ("axis", ("dim",)),
2490 ("keepdims", ("keepdim",)),
2491 ]
2492 for f in ("sum", "max", "min", "prod", "mean", "median", "std", "var"):
2493 # TODO: search "keepdim" in torch docs to find more
2494 _CUSTOM_WRAPPERS["torch", f] = make_translator(_torch_reduce_translation)
2495
2496 # for older versions of torch, can provide some alternative implementations
2497 _MODULE_ALIASES["torch[alt]"] = "torch"
2498
2499 _FUNCS["torch[alt]", "linalg.eigh"] = torch_linalg_eigh
2500 _FUNCS["torch[alt]", "linalg.eigvalsh"] = torch_linalg_eigvalsh
2501
2502 _SUBMODULE_ALIASES["torch[alt]", "linalg.norm"] = "torch"
2503 _SUBMODULE_ALIASES["torch[alt]", "linalg.qr"] = "torch"
2504 _SUBMODULE_ALIASES["torch[alt]", "linalg.solve"] = "torch"
2505 _SUBMODULE_ALIASES["torch[alt]", "linalg.svd"] = "torch"
2506
2507 _CUSTOM_WRAPPERS["torch[alt]", "linalg.qr"] = qr_allow_fat
2508 _CUSTOM_WRAPPERS["torch[alt]", "linalg.solve"] = torch_linalg_solve_wrap
2509 _CUSTOM_WRAPPERS["torch[alt]", "linalg.svd"] = svd_UsV_to_UsVH_wrapper
2510 _CUSTOM_WRAPPERS["torch[alt]", "split"] = torch_split_wrap
2511
2512 for f in _CREATION_ROUTINES:
2513 register_creation_routine("torch", f, inject_device=True)
2514
2515 # ---------------------------------- mxnet ---------------------------------- #
2516
2517
2518 def mxnet_to_numpy(x):
2519 return x.asnumpy()
2520
2521
2522 _MODULE_ALIASES["mxnet"] = "mxnet.numpy"
2523 _FUNCS["mxnet", "to_numpy"] = mxnet_to_numpy
2524
2525
2526 # --------------------------------- paddle ---------------------------------- #
2527
2528 _paddle_dtype_name_conversion = {
2529 "BOOL": "bool",
2530 "INT8": "int8",
2531 "INT16": "int16",
2532 "INT32": "int32",
2533 "INT64": "int64",
2534 "FP16": "float16",
2535 "FP32": "float32",
2536 "FP64": "float64",
2537 "COMPLEX64": "complex64",
2538 "COMPLEX128": "complex128",
2539 }
2540
2541
2542 @get_dtype_name.register("paddle")
2543 def paddle_get_dtype_name(x):
2544 return _paddle_dtype_name_conversion[x.dtype.name]
2545
2546
2547 @shape.register("paddle")
2548 def paddle_shape(x):
2549 # convert from list
2550 return tuple(x.shape)
2551
2552
2553 def paddle_to_numpy(x):
2554 return x.numpy()
2555
2556
2557 def paddle_transpose(a, axes=None):
2558 if axes is None:
2559 axes = tuple(range(a.ndim - 1, -1, -1))
2560 return a.transpose(perm=axes)
2561
2562
2563 def paddle_real(x):
2564 # paddle doesn't support calling real on real arrays
2565 try:
2566 if x.is_complex():
2567 return x.real()
2568 except AttributeError:
2569 pass
2570 return x
2571
2572
2573 def paddle_imag(x):
2574 # paddle doesn't support calling imag on real arrays
2575 try:
2576 if x.is_complex():
2577 return x.imag()
2578 except AttributeError:
2579 pass
2580 return do("zeros_like", x)
2581
2582
2583 def paddle_indices(dimensions):
2584 _meshgrid = get_lib_fn("paddle", "meshgrid")
2585 _arange = get_lib_fn("paddle", "arange")
2586 return _meshgrid(*map(_arange, dimensions), indexing="ij")
2587
2588
2589 def paddle_ravel(x):
2590 return x.reshape((-1,))
2591
2592
2593 def paddle_pad(array, pad_width, mode="constant", constant_values=0):
2594 if mode != "constant":
2595 raise NotImplementedError
2596
2597 try:
2598 # numpy takes pads like ((0, 0), (1, 1), ... (n-1, n-1))
2599 # paddle takes pads like (0, 0, 1, 1, 2, 2, ...)
2600 pad = tuple(itertools.chain.from_iterable(pad_width))
2601
2602 # a single tuple was specified ((a, b),) - use for all axes
2603 if len(pad) == 2:
2604 pad = pad * array.ndim
2605
2606 except TypeError:
2607 # assume int
2608 pad = (pad_width,) * 2 * array.ndim
2609
2610 return do(
2611 "nn.functional.pad",
2612 array,
2613 pad=pad,
2614 mode=mode,
2615 value=constant_values,
2616 like="paddle",
2617 )
2618
2619
2620 def paddle_wrap_reduction(fn):
2621 def numpy_like(*args, **kwargs):
2622 keepdims = kwargs.pop("keepdims", None)
2623 if keepdims is not None:
2624 kwargs["keepdim"] = keepdims
2625 return fn(*args, **kwargs)
2626
2627 return numpy_like
2628
2629
2630 def paddle_split_wrap(fn):
2631 # paddle doesn't seem to have `tensor_split always`
2632
2633 @functools.wraps(fn)
2634 def numpy_like(ary, indices_or_sections, axis=0, **kwargs):
2635 if isinstance(indices_or_sections, int):
2636 return fn(ary, indices_or_sections, axis=axis, **kwargs)
2637 else:
2638 diff = do(
2639 "diff",
2640 indices_or_sections,
2641 prepend=0,
2642 append=shape(ary)[axis],
2643 like="numpy",
2644 )
2645 diff = list(diff)
2646 return fn(ary, diff, axis=axis)
2647
2648 return numpy_like
2649
2650
2651 _MODULE_ALIASES["paddle[alt]"] = "paddle"
2652
2653 _FUNCS["paddle", "to_numpy"] = paddle_to_numpy
2654 _FUNCS["paddle", "transpose"] = paddle_transpose
2655 _FUNCS["paddle", "real"] = paddle_real
2656 _FUNCS["paddle", "imag"] = paddle_imag
2657 _FUNCS["paddle", "indices"] = paddle_indices
2658 _FUNCS["paddle", "ravel"] = paddle_ravel
2659 _FUNCS["paddle", "pad"] = paddle_pad
2660
2661 _FUNC_ALIASES["paddle", "random.normal"] = "randn"
2662 _FUNC_ALIASES["paddle", "random.uniform"] = "rand"
2663 _FUNC_ALIASES["paddle", "asarray"] = "to_tensor"
2664 _FUNC_ALIASES["paddle", "concatenate"] = "concat"
2665 _FUNC_ALIASES["paddle", "power"] = "pow"
2666 _FUNC_ALIASES["paddle", "identity"] = "eye"
2667 _FUNC_ALIASES["paddle", "split"] = "tensor_split"
2668
2669 _SUBMODULE_ALIASES["paddle", "random.normal"] = "paddle"
2670 _SUBMODULE_ALIASES["paddle", "random.uniform"] = "paddle"
2671
2672 _CUSTOM_WRAPPERS["paddle", "random.normal"] = scale_random_normal_manually
2673 _CUSTOM_WRAPPERS["paddle", "random.uniform"] = scale_random_uniform_manually
2674 _CUSTOM_WRAPPERS["paddle[alt]", "split"] = paddle_split_wrap
2675 _CUSTOM_WRAPPERS["paddle", "tril"] = make_translator(
2676 [("m", ("x",)), ("k", ("diagonal", 0))]
2677 )
2678 _CUSTOM_WRAPPERS["paddle", "triu"] = make_translator(
2679 [("m", ("x",)), ("k", ("diagonal", 0))]
2680 )
2681 for f in ("sum", "max", "min", "prod", "mean", "std", "var"):
2682 _CUSTOM_WRAPPERS["paddle", f] = paddle_wrap_reduction
2683
2684
2685 # -------------------------------- pytensor --------------------------------- #
2686
2687
2688 @shape.register("pytensor")
2689 def pytensor_shape(x):
2690 return x.type.shape
2691
2692
2693 def pytensor_wrap_qr_with_shapes(fn):
2694 import pytensor.tensor as pt
2695
2696 @functools.wraps(fn)
2697 def qr_shaped(x, **kwargs):
2698 *b, m, n = x.type.shape
2699 k = min(m, n)
2700 q, r = fn(x, mode="economic", **kwargs)
2701 q = pt.specify_shape(q, (*b, m, k))
2702 r = pt.specify_shape(r, (*b, k, n))
2703 return q, r
2704
2705 return qr_shaped
2706
2707
2708 def pytensor_wrap_svd_with_shapes(fn):
2709 import pytensor.tensor as pt
2710
2711 @functools.wraps(fn)
2712 def svd_shaped(x, full_matrices=False, **kwargs):
2713 *b, m, n = x.type.shape
2714
2715 k = min(m, n)
2716 if full_matrices:
2717 u_shape = (*b, m, m)
2718 vh_shape = (*b, n, n)
2719 else:
2720 u_shape = (*b, m, k)
2721 vh_shape = (*b, k, n)
2722
2723 u, s, vh = fn(x, full_matrices=full_matrices, **kwargs)
2724 u = pt.specify_shape(u, u_shape)
2725 s = pt.specify_shape(s, (*b, k))
2726 vh = pt.specify_shape(vh, vh_shape)
2727 return u, s, vh
2728
2729 return svd_shaped
2730
2731
2732 _MODULE_ALIASES["pytensor"] = "pytensor.tensor"
2733 _CUSTOM_WRAPPERS["pytensor", "linalg.qr"] = pytensor_wrap_qr_with_shapes
2734 _CUSTOM_WRAPPERS["pytensor", "linalg.svd"] = pytensor_wrap_svd_with_shapes