Debian Codegraph @

libgomp/config/nvptx/target.c

raw
1 /* Copyright (C) 2013-2025 Free Software Foundation, Inc.
2 Contributed by Jakub Jelinek <[email protected]>.
3
4 This file is part of the GNU Offloading and Multi Processing Library
5 (libgomp).
6
7 Libgomp is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
10 any later version.
11
12 Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14 FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 more details.
16
17 Under Section 7 of GPL version 3, you are granted additional
18 permissions described in the GCC Runtime Library Exception, version
19 3.1, as published by the Free Software Foundation.
20
21 You should have received a copy of the GNU General Public License and
22 a copy of the GCC Runtime Library Exception along with this program;
23 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
24 <http://www.gnu.org/licenses/>. */
25
26 #include "libgomp.h"
27 #include "libgomp-nvptx.h" /* For struct rev_offload + GOMP_REV_OFFLOAD_VAR. */
28 #include <limits.h>
29
30 extern int __gomp_team_num __attribute__((shared));
31 extern volatile struct gomp_offload_icvs GOMP_ADDITIONAL_ICVS;
32 volatile struct rev_offload *GOMP_REV_OFFLOAD_VAR;
33
34 /* Implement OpenMP 'teams' construct.
35
36 Initialize upon FIRST call. Return whether this invocation is active.
37 Depending on whether NUM_TEAMS_LOWER asks for more teams than are provided
38 in hardware, we may need to loop multiple times; in that case make sure to
39 update the team-level variable used by 'omp_get_team_num', as we then can't
40 just use '%ctaid.x'. */
41
42 bool
43 GOMP_teams4 (unsigned int num_teams_lower, unsigned int num_teams_upper,
44 unsigned int thread_limit, bool first)
45 {
46 unsigned int num_blocks, block_id;
47 asm ("mov.u32 %0, %%nctaid.x;" : "=r" (num_blocks));
48 if (!first)
49 {
50 unsigned int team_num;
51 if (num_blocks > gomp_num_teams_var)
52 return false;
53 team_num = __gomp_team_num;
54 if (team_num > gomp_num_teams_var - num_blocks)
55 return false;
56 __gomp_team_num = team_num + num_blocks;
57 return true;
58 }
59 if (thread_limit)
60 {
61 struct gomp_task_icv *icv = gomp_icv (true);
62 icv->thread_limit_var
63 = thread_limit > INT_MAX ? UINT_MAX : thread_limit;
64 }
65 if (!num_teams_upper)
66 num_teams_upper = ((GOMP_ADDITIONAL_ICVS.nteams > 0
67 && num_blocks > GOMP_ADDITIONAL_ICVS.nteams)
68 ? GOMP_ADDITIONAL_ICVS.nteams : num_blocks);
69 else if (num_blocks < num_teams_lower)
70 num_teams_upper = num_teams_lower;
71 else if (num_blocks < num_teams_upper)
72 num_teams_upper = num_blocks;
73 asm ("mov.u32 %0, %%ctaid.x;" : "=r" (block_id));
74 if (block_id >= num_teams_upper)
75 return false;
76 __gomp_team_num = block_id;
77 gomp_num_teams_var = num_teams_upper - 1;
78 return true;
79 }
80
81 int
82 omp_pause_resource (omp_pause_resource_t kind, int device_num)
83 {
84 (void) kind;
85 (void) device_num;
86 return -1;
87 }
88
89 int
90 omp_pause_resource_all (omp_pause_resource_t kind)
91 {
92 (void) kind;
93 return -1;
94 }
95
96 ialias (omp_pause_resource)
97 ialias (omp_pause_resource_all)
98
99 void
100 GOMP_target_ext (int device, void (*fn) (void *), size_t mapnum,
101 void **hostaddrs, size_t *sizes, unsigned short *kinds,
102 unsigned int flags, void **depend, void **args)
103 {
104 static int lock = 0; /* == gomp_mutex_t lock; gomp_mutex_init (&lock); */
105 (void) flags;
106 (void) depend;
107 (void) args;
108
109 if (device != GOMP_DEVICE_HOST_FALLBACK
110 || fn == NULL
111 || GOMP_REV_OFFLOAD_VAR == NULL)
112 return;
113
114 gomp_mutex_lock (&lock);
115
116 GOMP_REV_OFFLOAD_VAR->mapnum = mapnum;
117 GOMP_REV_OFFLOAD_VAR->addrs = (uint64_t) hostaddrs;
118 GOMP_REV_OFFLOAD_VAR->sizes = (uint64_t) sizes;
119 GOMP_REV_OFFLOAD_VAR->kinds = (uint64_t) kinds;
120 GOMP_REV_OFFLOAD_VAR->dev_num = GOMP_ADDITIONAL_ICVS.device_num;
121
122 /* Set 'fn' to trigger processing on the host; wait for completion,
123 which is flagged by setting 'fn' back to 0 on the host. */
124 uint64_t addr_struct_fn = (uint64_t) &GOMP_REV_OFFLOAD_VAR->fn;
125 #if __PTX_SM__ >= 700
126 asm volatile ("st.global.release.sys.u64 [%0], %1;"
127 : : "r"(addr_struct_fn), "r" (fn) : "memory");
128 #else
129 __sync_synchronize (); /* membar.sys */
130 asm volatile ("st.volatile.global.u64 [%0], %1;"
131 : : "r"(addr_struct_fn), "r" (fn) : "memory");
132 #endif
133
134 #if __PTX_SM__ >= 700
135 uint64_t fn2;
136 do
137 {
138 asm volatile ("ld.acquire.sys.global.u64 %0, [%1];"
139 : "=r" (fn2) : "r" (addr_struct_fn) : "memory");
140 }
141 while (fn2 != 0);
142 #else
143 /* ld.global.u64 %r64,[__gomp_rev_offload_var];
144 ld.u64 %r36,[%r64];
145 membar.sys; */
146 while (__atomic_load_n (&GOMP_REV_OFFLOAD_VAR->fn, __ATOMIC_ACQUIRE) != 0)
147 ; /* spin */
148 #endif
149
150 gomp_mutex_unlock (&lock);
151 }
152
153 void
154 GOMP_target_data_ext (int device, size_t mapnum, void **hostaddrs,
155 size_t *sizes, unsigned short *kinds)
156 {
157 (void) device;
158 (void) mapnum;
159 (void) hostaddrs;
160 (void) sizes;
161 (void) kinds;
162 __builtin_unreachable ();
163 }
164
165 void
166 GOMP_target_end_data (void)
167 {
168 __builtin_unreachable ();
169 }
170
171 void
172 GOMP_target_update_ext (int device, size_t mapnum, void **hostaddrs,
173 size_t *sizes, unsigned short *kinds,
174 unsigned int flags, void **depend)
175 {
176 (void) device;
177 (void) mapnum;
178 (void) hostaddrs;
179 (void) sizes;
180 (void) kinds;
181 (void) flags;
182 (void) depend;
183 __builtin_unreachable ();
184 }
185
186 void
187 GOMP_target_enter_exit_data (int device, size_t mapnum, void **hostaddrs,
188 size_t *sizes, unsigned short *kinds,
189 unsigned int flags, void **depend)
190 {
191 (void) device;
192 (void) mapnum;
193 (void) hostaddrs;
194 (void) sizes;
195 (void) kinds;
196 (void) flags;
197 (void) depend;
198 __builtin_unreachable ();
199 }
200
201 int
202 omp_get_num_interop_properties (const omp_interop_t interop
203 __attribute__ ((unused)))
204 {
205 return 0;
206 }
207
208 omp_intptr_t
209 omp_get_interop_int (const omp_interop_t interop,
210 omp_interop_property_t property_id,
211 omp_interop_rc_t *ret_code)
212 {
213 if (ret_code == NULL)
214 return 0;
215 if (property_id < omp_ipr_first || property_id >= 0)
216 *ret_code = omp_irc_out_of_range;
217 else if (interop == omp_interop_none)
218 *ret_code = omp_irc_empty;
219 else
220 *ret_code = omp_irc_other;
221 return 0;
222 }
223
224 void *
225 omp_get_interop_ptr (const omp_interop_t interop,
226 omp_interop_property_t property_id,
227 omp_interop_rc_t *ret_code)
228 {
229 if (ret_code == NULL)
230 return NULL;
231 if (property_id < omp_ipr_first || property_id >= 0)
232 *ret_code = omp_irc_out_of_range;
233 else if (interop == omp_interop_none)
234 *ret_code = omp_irc_empty;
235 else
236 *ret_code = omp_irc_other;
237 return NULL;
238 }
239
240 const char *
241 omp_get_interop_str (const omp_interop_t interop,
242 omp_interop_property_t property_id,
243 omp_interop_rc_t *ret_code)
244 {
245 if (ret_code == NULL)
246 return NULL;
247 if (property_id < omp_ipr_first || property_id >= 0)
248 *ret_code = omp_irc_out_of_range;
249 else if (interop == omp_interop_none)
250 *ret_code = omp_irc_empty;
251 else
252 *ret_code = omp_irc_other;
253 return NULL;
254 }
255
256 const char *
257 omp_get_interop_name (const omp_interop_t interop __attribute__ ((unused)),
258 omp_interop_property_t property_id)
259 {
260 static const char *prop_string[0 - omp_ipr_first]
261 = {"fr_id", "fr_name", "vendor", "vendor_name", "device_num", "platform",
262 "device", "device_context", "targetsync"};
263 if (property_id < omp_ipr_first || property_id >= 0)
264 return NULL;
265 return prop_string[omp_ipr_fr_id - property_id];
266 }
267
268 const char *
269 omp_get_interop_type_desc (const omp_interop_t interop __attribute__ ((unused)),
270 omp_interop_property_t property_id
271 __attribute__ ((unused)))
272 {
273 return NULL;
274 }
275
276 const char *
277 omp_get_interop_rc_desc (const omp_interop_t interop __attribute__ ((unused)),
278 omp_interop_rc_t ret_code)
279 {
280 static const char *rc_strings[omp_irc_no_value - omp_irc_other + 1]
281 = {"no meaningful value available",
282 "successful",
283 "provided interoperability object is equal to omp_interop_none",
284 "property ID is out of range",
285 "property type is integer; use omp_get_interop_int",
286 "property type is pointer; use omp_get_interop_ptr",
287 "property type is string; use omp_get_interop_str",
288 "obtaining properties is only supported on the initial device"};
289 /* omp_irc_other is returned by device-side omp_get_interop_{int,ptr,str};
290 the host returns for omp_irc_other NULL as it is not used. Besides the
291 three omp_interop_rc_t values used on the device side, handle host values
292 leaked to the device side. */
293 if (ret_code > omp_irc_no_value || ret_code < omp_irc_other)
294 return NULL;
295 return rc_strings[omp_irc_no_value - ret_code];
296 }
297
298 const char *
299 omp_get_uid_from_device (int device_num __attribute__ ((unused)))
300 {
301 return NULL;
302 }
303
304 int
305 omp_get_device_from_uid (const char *uid __attribute__ ((unused)))
306 {
307 return omp_invalid_device;
308 }
309
310 ialias (omp_get_num_interop_properties)
311 ialias (omp_get_interop_int)
312 ialias (omp_get_interop_ptr)
313 ialias (omp_get_interop_str)
314 ialias (omp_get_interop_name)
315 ialias (omp_get_interop_type_desc)
316 ialias (omp_get_interop_rc_desc)
317 ialias (omp_get_uid_from_device)
318 ialias (omp_get_device_from_uid)