Debian Codegraph @

.cargo/vendor/half/src/binary16/arch/aarch64.rs

raw
1 use core::{
2 arch::{
3 aarch64::{float32x4_t, float64x2_t, uint16x4_t},
4 asm,
5 },
6 mem::MaybeUninit,
7 ptr,
8 };
9
10 #[target_feature(enable = "fp16")]
11 #[inline]
12 pub(super) unsafe fn f16_to_f32_fp16(i: u16) -> f32 {
13 let result: f32;
14 asm!(
15 "fcvt {0:s}, {1:h}",
16 out(vreg) result,
17 in(vreg) i,
18 options(pure, nomem, nostack, preserves_flags));
19 result
20 }
21
22 #[target_feature(enable = "fp16")]
23 #[inline]
24 pub(super) unsafe fn f16_to_f64_fp16(i: u16) -> f64 {
25 let result: f64;
26 asm!(
27 "fcvt {0:d}, {1:h}",
28 out(vreg) result,
29 in(vreg) i,
30 options(pure, nomem, nostack, preserves_flags));
31 result
32 }
33
34 #[target_feature(enable = "fp16")]
35 #[inline]
36 pub(super) unsafe fn f32_to_f16_fp16(f: f32) -> u16 {
37 let result: u16;
38 asm!(
39 "fcvt {0:h}, {1:s}",
40 out(vreg) result,
41 in(vreg) f,
42 options(pure, nomem, nostack, preserves_flags));
43 result
44 }
45
46 #[target_feature(enable = "fp16")]
47 #[inline]
48 pub(super) unsafe fn f64_to_f16_fp16(f: f64) -> u16 {
49 let result: u16;
50 asm!(
51 "fcvt {0:h}, {1:d}",
52 out(vreg) result,
53 in(vreg) f,
54 options(pure, nomem, nostack, preserves_flags));
55 result
56 }
57
58 #[target_feature(enable = "fp16")]
59 #[inline]
60 pub(super) unsafe fn f16x4_to_f32x4_fp16(v: &[u16; 4]) -> [f32; 4] {
61 let mut vec = MaybeUninit::<uint16x4_t>::uninit();
62 ptr::copy_nonoverlapping(v.as_ptr(), vec.as_mut_ptr().cast(), 4);
63 let result: float32x4_t;
64 asm!(
65 "fcvtl {0:v}.4s, {1:v}.4h",
66 out(vreg) result,
67 in(vreg) vec.assume_init(),
68 options(pure, nomem, nostack));
69 *(&result as *const float32x4_t).cast()
70 }
71
72 #[target_feature(enable = "fp16")]
73 #[inline]
74 pub(super) unsafe fn f32x4_to_f16x4_fp16(v: &[f32; 4]) -> [u16; 4] {
75 let mut vec = MaybeUninit::<float32x4_t>::uninit();
76 ptr::copy_nonoverlapping(v.as_ptr(), vec.as_mut_ptr().cast(), 4);
77 let result: uint16x4_t;
78 asm!(
79 "fcvtn {0:v}.4h, {1:v}.4s",
80 out(vreg) result,
81 in(vreg) vec.assume_init(),
82 options(pure, nomem, nostack));
83 *(&result as *const uint16x4_t).cast()
84 }
85
86 #[target_feature(enable = "fp16")]
87 #[inline]
88 pub(super) unsafe fn f16x4_to_f64x4_fp16(v: &[u16; 4]) -> [f64; 4] {
89 let mut vec = MaybeUninit::<uint16x4_t>::uninit();
90 ptr::copy_nonoverlapping(v.as_ptr(), vec.as_mut_ptr().cast(), 4);
91 let low: float64x2_t;
92 let high: float64x2_t;
93 asm!(
94 "fcvtl {2:v}.4s, {3:v}.4h", // Convert to f32
95 "fcvtl {0:v}.2d, {2:v}.2s", // Convert low part to f64
96 "fcvtl2 {1:v}.2d, {2:v}.4s", // Convert high part to f64
97 lateout(vreg) low,
98 lateout(vreg) high,
99 out(vreg) _,
100 in(vreg) vec.assume_init(),
101 options(pure, nomem, nostack));
102 *[low, high].as_ptr().cast()
103 }
104
105 #[target_feature(enable = "fp16")]
106 #[inline]
107 pub(super) unsafe fn f64x4_to_f16x4_fp16(v: &[f64; 4]) -> [u16; 4] {
108 let mut low = MaybeUninit::<float64x2_t>::uninit();
109 let mut high = MaybeUninit::<float64x2_t>::uninit();
110 ptr::copy_nonoverlapping(v.as_ptr(), low.as_mut_ptr().cast(), 2);
111 ptr::copy_nonoverlapping(v[2..].as_ptr(), high.as_mut_ptr().cast(), 2);
112 let result: uint16x4_t;
113 asm!(
114 "fcvtn {1:v}.2s, {2:v}.2d", // Convert low to f32
115 "fcvtn2 {1:v}.4s, {3:v}.2d", // Convert high to f32
116 "fcvtn {0:v}.4h, {1:v}.4s", // Convert to f16
117 lateout(vreg) result,
118 out(vreg) _,
119 in(vreg) low.assume_init(),
120 in(vreg) high.assume_init(),
121 options(pure, nomem, nostack));
122 *(&result as *const uint16x4_t).cast()
123 }
124
125 #[target_feature(enable = "fp16")]
126 #[inline]
127 pub(super) unsafe fn add_f16_fp16(a: u16, b: u16) -> u16 {
128 let result: u16;
129 asm!(
130 "fadd {0:h}, {1:h}, {2:h}",
131 out(vreg) result,
132 in(vreg) a,
133 in(vreg) b,
134 options(pure, nomem, nostack));
135 result
136 }
137
138 #[target_feature(enable = "fp16")]
139 #[inline]
140 pub(super) unsafe fn subtract_f16_fp16(a: u16, b: u16) -> u16 {
141 let result: u16;
142 asm!(
143 "fsub {0:h}, {1:h}, {2:h}",
144 out(vreg) result,
145 in(vreg) a,
146 in(vreg) b,
147 options(pure, nomem, nostack));
148 result
149 }
150
151 #[target_feature(enable = "fp16")]
152 #[inline]
153 pub(super) unsafe fn multiply_f16_fp16(a: u16, b: u16) -> u16 {
154 let result: u16;
155 asm!(
156 "fmul {0:h}, {1:h}, {2:h}",
157 out(vreg) result,
158 in(vreg) a,
159 in(vreg) b,
160 options(pure, nomem, nostack));
161 result
162 }
163
164 #[target_feature(enable = "fp16")]
165 #[inline]
166 pub(super) unsafe fn divide_f16_fp16(a: u16, b: u16) -> u16 {
167 let result: u16;
168 asm!(
169 "fdiv {0:h}, {1:h}, {2:h}",
170 out(vreg) result,
171 in(vreg) a,
172 in(vreg) b,
173 options(pure, nomem, nostack));
174 result
175 }