Debian Codegraph @

llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.h

raw
1 //===----- CGOpenMPRuntime.h - Interface to OpenMP Runtimes -----*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This provides a class for OpenMP runtime code generation.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #ifndef LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H
14 #define LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H
15
16 #include "CGValue.h"
17 #include "clang/AST/DeclOpenMP.h"
18 #include "clang/AST/GlobalDecl.h"
19 #include "clang/AST/Type.h"
20 #include "clang/Basic/OpenMPKinds.h"
21 #include "clang/Basic/SourceLocation.h"
22 #include "llvm/ADT/DenseMap.h"
23 #include "llvm/ADT/PointerIntPair.h"
24 #include "llvm/ADT/SmallPtrSet.h"
25 #include "llvm/ADT/StringMap.h"
26 #include "llvm/ADT/StringSet.h"
27 #include "llvm/Frontend/OpenMP/OMPConstants.h"
28 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
29 #include "llvm/IR/Function.h"
30 #include "llvm/IR/ValueHandle.h"
31 #include "llvm/Support/AtomicOrdering.h"
32
33 namespace llvm {
34 class ArrayType;
35 class Constant;
36 class FunctionType;
37 class GlobalVariable;
38 class Type;
39 class Value;
40 class OpenMPIRBuilder;
41 } // namespace llvm
42
43 namespace clang {
44 class Expr;
45 class OMPDependClause;
46 class OMPExecutableDirective;
47 class OMPLoopDirective;
48 class VarDecl;
49 class OMPDeclareReductionDecl;
50
51 namespace CodeGen {
52 class Address;
53 class CodeGenFunction;
54 class CodeGenModule;
55
56 /// A basic class for pre|post-action for advanced codegen sequence for OpenMP
57 /// region.
58 class PrePostActionTy {
59 public:
60 explicit PrePostActionTy() {}
61 virtual void Enter(CodeGenFunction &CGF) {}
62 virtual void Exit(CodeGenFunction &CGF) {}
63 virtual ~PrePostActionTy() {}
64 };
65
66 /// Class provides a way to call simple version of codegen for OpenMP region, or
67 /// an advanced with possible pre|post-actions in codegen.
68 class RegionCodeGenTy final {
69 intptr_t CodeGen;
70 typedef void (*CodeGenTy)(intptr_t, CodeGenFunction &, PrePostActionTy &);
71 CodeGenTy Callback;
72 mutable PrePostActionTy *PrePostAction;
73 RegionCodeGenTy() = delete;
74 template <typename Callable>
75 static void CallbackFn(intptr_t CodeGen, CodeGenFunction &CGF,
76 PrePostActionTy &Action) {
77 return (*reinterpret_cast<Callable *>(CodeGen))(CGF, Action);
78 }
79
80 public:
81 template <typename Callable>
82 RegionCodeGenTy(
83 Callable &&CodeGen,
84 std::enable_if_t<!std::is_same<std::remove_reference_t<Callable>,
85 RegionCodeGenTy>::value> * = nullptr)
86 : CodeGen(reinterpret_cast<intptr_t>(&CodeGen)),
87 Callback(CallbackFn<std::remove_reference_t<Callable>>),
88 PrePostAction(nullptr) {}
89 void setAction(PrePostActionTy &Action) const { PrePostAction = &Action; }
90 void operator()(CodeGenFunction &CGF) const;
91 };
92
93 struct OMPTaskDataTy final {
94 SmallVector<const Expr *, 4> PrivateVars;
95 SmallVector<const Expr *, 4> PrivateCopies;
96 SmallVector<const Expr *, 4> FirstprivateVars;
97 SmallVector<const Expr *, 4> FirstprivateCopies;
98 SmallVector<const Expr *, 4> FirstprivateInits;
99 SmallVector<const Expr *, 4> LastprivateVars;
100 SmallVector<const Expr *, 4> LastprivateCopies;
101 SmallVector<const Expr *, 4> ReductionVars;
102 SmallVector<const Expr *, 4> ReductionOrigs;
103 SmallVector<const Expr *, 4> ReductionCopies;
104 SmallVector<const Expr *, 4> ReductionOps;
105 SmallVector<CanonicalDeclPtr<const VarDecl>, 4> PrivateLocals;
106 struct DependData {
107 OpenMPDependClauseKind DepKind = OMPC_DEPEND_unknown;
108 const Expr *IteratorExpr = nullptr;
109 SmallVector<const Expr *, 4> DepExprs;
110 explicit DependData() = default;
111 DependData(OpenMPDependClauseKind DepKind, const Expr *IteratorExpr)
112 : DepKind(DepKind), IteratorExpr(IteratorExpr) {}
113 };
114 SmallVector<DependData, 4> Dependences;
115 llvm::PointerIntPair<llvm::Value *, 1, bool> Final;
116 llvm::PointerIntPair<llvm::Value *, 1, bool> Schedule;
117 llvm::PointerIntPair<llvm::Value *, 1, bool> Priority;
118 llvm::Value *Reductions = nullptr;
119 unsigned NumberOfParts = 0;
120 bool Tied = true;
121 bool Nogroup = false;
122 bool IsReductionWithTaskMod = false;
123 bool IsWorksharingReduction = false;
124 bool HasNowaitClause = false;
125 };
126
127 /// Class intended to support codegen of all kind of the reduction clauses.
128 class ReductionCodeGen {
129 private:
130 /// Data required for codegen of reduction clauses.
131 struct ReductionData {
132 /// Reference to the item shared between tasks to reduce into.
133 const Expr *Shared = nullptr;
134 /// Reference to the original item.
135 const Expr *Ref = nullptr;
136 /// Helper expression for generation of private copy.
137 const Expr *Private = nullptr;
138 /// Helper expression for generation reduction operation.
139 const Expr *ReductionOp = nullptr;
140 ReductionData(const Expr *Shared, const Expr *Ref, const Expr *Private,
141 const Expr *ReductionOp)
142 : Shared(Shared), Ref(Ref), Private(Private), ReductionOp(ReductionOp) {
143 }
144 };
145 /// List of reduction-based clauses.
146 SmallVector<ReductionData, 4> ClausesData;
147
148 /// List of addresses of shared variables/expressions.
149 SmallVector<std::pair<LValue, LValue>, 4> SharedAddresses;
150 /// List of addresses of original variables/expressions.
151 SmallVector<std::pair<LValue, LValue>, 4> OrigAddresses;
152 /// Sizes of the reduction items in chars.
153 SmallVector<std::pair<llvm::Value *, llvm::Value *>, 4> Sizes;
154 /// Base declarations for the reduction items.
155 SmallVector<const VarDecl *, 4> BaseDecls;
156
157 /// Emits lvalue for shared expression.
158 LValue emitSharedLValue(CodeGenFunction &CGF, const Expr *E);
159 /// Emits upper bound for shared expression (if array section).
160 LValue emitSharedLValueUB(CodeGenFunction &CGF, const Expr *E);
161 /// Performs aggregate initialization.
162 /// \param N Number of reduction item in the common list.
163 /// \param PrivateAddr Address of the corresponding private item.
164 /// \param SharedAddr Address of the original shared variable.
165 /// \param DRD Declare reduction construct used for reduction item.
166 void emitAggregateInitialization(CodeGenFunction &CGF, unsigned N,
167 Address PrivateAddr, Address SharedAddr,
168 const OMPDeclareReductionDecl *DRD);
169
170 public:
171 ReductionCodeGen(ArrayRef<const Expr *> Shareds, ArrayRef<const Expr *> Origs,
172 ArrayRef<const Expr *> Privates,
173 ArrayRef<const Expr *> ReductionOps);
174 /// Emits lvalue for the shared and original reduction item.
175 /// \param N Number of the reduction item.
176 void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N);
177 /// Emits the code for the variable-modified type, if required.
178 /// \param N Number of the reduction item.
179 void emitAggregateType(CodeGenFunction &CGF, unsigned N);
180 /// Emits the code for the variable-modified type, if required.
181 /// \param N Number of the reduction item.
182 /// \param Size Size of the type in chars.
183 void emitAggregateType(CodeGenFunction &CGF, unsigned N, llvm::Value *Size);
184 /// Performs initialization of the private copy for the reduction item.
185 /// \param N Number of the reduction item.
186 /// \param PrivateAddr Address of the corresponding private item.
187 /// \param DefaultInit Default initialization sequence that should be
188 /// performed if no reduction specific initialization is found.
189 /// \param SharedAddr Address of the original shared variable.
190 void
191 emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr,
192 Address SharedAddr,
193 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit);
194 /// Returns true if the private copy requires cleanups.
195 bool needCleanups(unsigned N);
196 /// Emits cleanup code for the reduction item.
197 /// \param N Number of the reduction item.
198 /// \param PrivateAddr Address of the corresponding private item.
199 void emitCleanups(CodeGenFunction &CGF, unsigned N, Address PrivateAddr);
200 /// Adjusts \p PrivatedAddr for using instead of the original variable
201 /// address in normal operations.
202 /// \param N Number of the reduction item.
203 /// \param PrivateAddr Address of the corresponding private item.
204 Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
205 Address PrivateAddr);
206 /// Returns LValue for the reduction item.
207 LValue getSharedLValue(unsigned N) const { return SharedAddresses[N].first; }
208 /// Returns LValue for the original reduction item.
209 LValue getOrigLValue(unsigned N) const { return OrigAddresses[N].first; }
210 /// Returns the size of the reduction item (in chars and total number of
211 /// elements in the item), or nullptr, if the size is a constant.
212 std::pair<llvm::Value *, llvm::Value *> getSizes(unsigned N) const {
213 return Sizes[N];
214 }
215 /// Returns the base declaration of the reduction item.
216 const VarDecl *getBaseDecl(unsigned N) const { return BaseDecls[N]; }
217 /// Returns the base declaration of the reduction item.
218 const Expr *getRefExpr(unsigned N) const { return ClausesData[N].Ref; }
219 /// Returns true if the initialization of the reduction item uses initializer
220 /// from declare reduction construct.
221 bool usesReductionInitializer(unsigned N) const;
222 /// Return the type of the private item.
223 QualType getPrivateType(unsigned N) const {
224 return cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl())
225 ->getType();
226 }
227 };
228
229 class CGOpenMPRuntime {
230 public:
231 /// Allows to disable automatic handling of functions used in target regions
232 /// as those marked as `omp declare target`.
233 class DisableAutoDeclareTargetRAII {
234 CodeGenModule &CGM;
235 bool SavedShouldMarkAsGlobal;
236
237 public:
238 DisableAutoDeclareTargetRAII(CodeGenModule &CGM);
239 ~DisableAutoDeclareTargetRAII();
240 };
241
242 /// Manages list of nontemporal decls for the specified directive.
243 class NontemporalDeclsRAII {
244 CodeGenModule &CGM;
245 const bool NeedToPush;
246
247 public:
248 NontemporalDeclsRAII(CodeGenModule &CGM, const OMPLoopDirective &S);
249 ~NontemporalDeclsRAII();
250 };
251
252 /// Manages list of nontemporal decls for the specified directive.
253 class UntiedTaskLocalDeclsRAII {
254 CodeGenModule &CGM;
255 const bool NeedToPush;
256
257 public:
258 UntiedTaskLocalDeclsRAII(
259 CodeGenFunction &CGF,
260 const llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
261 std::pair<Address, Address>> &LocalVars);
262 ~UntiedTaskLocalDeclsRAII();
263 };
264
265 /// Maps the expression for the lastprivate variable to the global copy used
266 /// to store new value because original variables are not mapped in inner
267 /// parallel regions. Only private copies are captured but we need also to
268 /// store private copy in shared address.
269 /// Also, stores the expression for the private loop counter and it
270 /// threaprivate name.
271 struct LastprivateConditionalData {
272 llvm::MapVector<CanonicalDeclPtr<const Decl>, SmallString<16>>
273 DeclToUniqueName;
274 LValue IVLVal;
275 llvm::Function *Fn = nullptr;
276 bool Disabled = false;
277 };
278 /// Manages list of lastprivate conditional decls for the specified directive.
279 class LastprivateConditionalRAII {
280 enum class ActionToDo {
281 DoNotPush,
282 PushAsLastprivateConditional,
283 DisableLastprivateConditional,
284 };
285 CodeGenModule &CGM;
286 ActionToDo Action = ActionToDo::DoNotPush;
287
288 /// Check and try to disable analysis of inner regions for changes in
289 /// lastprivate conditional.
290 void tryToDisableInnerAnalysis(const OMPExecutableDirective &S,
291 llvm::DenseSet<CanonicalDeclPtr<const Decl>>
292 &NeedToAddForLPCsAsDisabled) const;
293
294 LastprivateConditionalRAII(CodeGenFunction &CGF,
295 const OMPExecutableDirective &S);
296
297 public:
298 explicit LastprivateConditionalRAII(CodeGenFunction &CGF,
299 const OMPExecutableDirective &S,
300 LValue IVLVal);
301 static LastprivateConditionalRAII disable(CodeGenFunction &CGF,
302 const OMPExecutableDirective &S);
303 ~LastprivateConditionalRAII();
304 };
305
306 llvm::OpenMPIRBuilder &getOMPBuilder() { return OMPBuilder; }
307
308 protected:
309 CodeGenModule &CGM;
310
311 /// An OpenMP-IR-Builder instance.
312 llvm::OpenMPIRBuilder OMPBuilder;
313
314 /// Helper to emit outlined function for 'target' directive.
315 /// \param D Directive to emit.
316 /// \param ParentName Name of the function that encloses the target region.
317 /// \param OutlinedFn Outlined function value to be defined by this call.
318 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
319 /// \param IsOffloadEntry True if the outlined function is an offload entry.
320 /// \param CodeGen Lambda codegen specific to an accelerator device.
321 /// An outlined function may not be an entry if, e.g. the if clause always
322 /// evaluates to false.
323 virtual void emitTargetOutlinedFunctionHelper(const OMPExecutableDirective &D,
324 StringRef ParentName,
325 llvm::Function *&OutlinedFn,
326 llvm::Constant *&OutlinedFnID,
327 bool IsOffloadEntry,
328 const RegionCodeGenTy &CodeGen);
329
330 /// Emits object of ident_t type with info for source location.
331 /// \param Flags Flags for OpenMP location.
332 /// \param EmitLoc emit source location with debug-info is off.
333 ///
334 llvm::Value *emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc,
335 unsigned Flags = 0, bool EmitLoc = false);
336
337 /// Emit the number of teams for a target directive. Inspect the num_teams
338 /// clause associated with a teams construct combined or closely nested
339 /// with the target directive.
340 ///
341 /// Emit a team of size one for directives such as 'target parallel' that
342 /// have no associated teams construct.
343 ///
344 /// Otherwise, return nullptr.
345 const Expr *getNumTeamsExprForTargetDirective(CodeGenFunction &CGF,
346 const OMPExecutableDirective &D,
347 int32_t &DefaultVal);
348 llvm::Value *emitNumTeamsForTargetDirective(CodeGenFunction &CGF,
349 const OMPExecutableDirective &D);
350 /// Emit the number of threads for a target directive. Inspect the
351 /// thread_limit clause associated with a teams construct combined or closely
352 /// nested with the target directive.
353 ///
354 /// Emit the num_threads clause for directives such as 'target parallel' that
355 /// have no associated teams construct.
356 ///
357 /// Otherwise, return nullptr.
358 const Expr *
359 getNumThreadsExprForTargetDirective(CodeGenFunction &CGF,
360 const OMPExecutableDirective &D,
361 int32_t &DefaultVal);
362 llvm::Value *
363 emitNumThreadsForTargetDirective(CodeGenFunction &CGF,
364 const OMPExecutableDirective &D);
365
366 /// Returns pointer to ident_t type.
367 llvm::Type *getIdentTyPointerTy();
368
369 /// Gets thread id value for the current thread.
370 ///
371 llvm::Value *getThreadID(CodeGenFunction &CGF, SourceLocation Loc);
372
373 /// Get the function name of an outlined region.
374 // The name can be customized depending on the target.
375 //
376 virtual StringRef getOutlinedHelperName() const { return ".omp_outlined."; }
377
378 /// Emits \p Callee function call with arguments \p Args with location \p Loc.
379 void emitCall(CodeGenFunction &CGF, SourceLocation Loc,
380 llvm::FunctionCallee Callee,
381 ArrayRef<llvm::Value *> Args = std::nullopt) const;
382
383 /// Emits address of the word in a memory where current thread id is
384 /// stored.
385 virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc);
386
387 void setLocThreadIdInsertPt(CodeGenFunction &CGF,
388 bool AtCurrentPoint = false);
389 void clearLocThreadIdInsertPt(CodeGenFunction &CGF);
390
391 /// Check if the default location must be constant.
392 /// Default is false to support OMPT/OMPD.
393 virtual bool isDefaultLocationConstant() const { return false; }
394
395 /// Returns additional flags that can be stored in reserved_2 field of the
396 /// default location.
397 virtual unsigned getDefaultLocationReserved2Flags() const { return 0; }
398
399 /// Returns default flags for the barriers depending on the directive, for
400 /// which this barier is going to be emitted.
401 static unsigned getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind);
402
403 /// Get the LLVM type for the critical name.
404 llvm::ArrayType *getKmpCriticalNameTy() const {return KmpCriticalNameTy;}
405
406 /// Returns corresponding lock object for the specified critical region
407 /// name. If the lock object does not exist it is created, otherwise the
408 /// reference to the existing copy is returned.
409 /// \param CriticalName Name of the critical region.
410 ///
411 llvm::Value *getCriticalRegionLock(StringRef CriticalName);
412
413 protected:
414 /// Map for SourceLocation and OpenMP runtime library debug locations.
415 typedef llvm::DenseMap<SourceLocation, llvm::Value *> OpenMPDebugLocMapTy;
416 OpenMPDebugLocMapTy OpenMPDebugLocMap;
417 /// The type for a microtask which gets passed to __kmpc_fork_call().
418 /// Original representation is:
419 /// typedef void (kmpc_micro)(kmp_int32 global_tid, kmp_int32 bound_tid,...);
420 llvm::FunctionType *Kmpc_MicroTy = nullptr;
421 /// Stores debug location and ThreadID for the function.
422 struct DebugLocThreadIdTy {
423 llvm::Value *DebugLoc;
424 llvm::Value *ThreadID;
425 /// Insert point for the service instructions.
426 llvm::AssertingVH<llvm::Instruction> ServiceInsertPt = nullptr;
427 };
428 /// Map of local debug location, ThreadId and functions.
429 typedef llvm::DenseMap<llvm::Function *, DebugLocThreadIdTy>
430 OpenMPLocThreadIDMapTy;
431 OpenMPLocThreadIDMapTy OpenMPLocThreadIDMap;
432 /// Map of UDRs and corresponding combiner/initializer.
433 typedef llvm::DenseMap<const OMPDeclareReductionDecl *,
434 std::pair<llvm::Function *, llvm::Function *>>
435 UDRMapTy;
436 UDRMapTy UDRMap;
437 /// Map of functions and locally defined UDRs.
438 typedef llvm::DenseMap<llvm::Function *,
439 SmallVector<const OMPDeclareReductionDecl *, 4>>
440 FunctionUDRMapTy;
441 FunctionUDRMapTy FunctionUDRMap;
442 /// Map from the user-defined mapper declaration to its corresponding
443 /// functions.
444 llvm::DenseMap<const OMPDeclareMapperDecl *, llvm::Function *> UDMMap;
445 /// Map of functions and their local user-defined mappers.
446 using FunctionUDMMapTy =
447 llvm::DenseMap<llvm::Function *,
448 SmallVector<const OMPDeclareMapperDecl *, 4>>;
449 FunctionUDMMapTy FunctionUDMMap;
450 /// Maps local variables marked as lastprivate conditional to their internal
451 /// types.
452 llvm::DenseMap<llvm::Function *,
453 llvm::DenseMap<CanonicalDeclPtr<const Decl>,
454 std::tuple<QualType, const FieldDecl *,
455 const FieldDecl *, LValue>>>
456 LastprivateConditionalToTypes;
457 /// Maps function to the position of the untied task locals stack.
458 llvm::DenseMap<llvm::Function *, unsigned> FunctionToUntiedTaskStackMap;
459 /// Type kmp_critical_name, originally defined as typedef kmp_int32
460 /// kmp_critical_name[8];
461 llvm::ArrayType *KmpCriticalNameTy;
462 /// An ordered map of auto-generated variables to their unique names.
463 /// It stores variables with the following names: 1) ".gomp_critical_user_" +
464 /// <critical_section_name> + ".var" for "omp critical" directives; 2)
465 /// <mangled_name_for_global_var> + ".cache." for cache for threadprivate
466 /// variables.
467 llvm::StringMap<llvm::AssertingVH<llvm::GlobalVariable>,
468 llvm::BumpPtrAllocator> InternalVars;
469 /// Type typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *);
470 llvm::Type *KmpRoutineEntryPtrTy = nullptr;
471 QualType KmpRoutineEntryPtrQTy;
472 /// Type typedef struct kmp_task {
473 /// void * shareds; /**< pointer to block of pointers to
474 /// shared vars */
475 /// kmp_routine_entry_t routine; /**< pointer to routine to call for
476 /// executing task */
477 /// kmp_int32 part_id; /**< part id for the task */
478 /// kmp_routine_entry_t destructors; /* pointer to function to invoke
479 /// deconstructors of firstprivate C++ objects */
480 /// } kmp_task_t;
481 QualType KmpTaskTQTy;
482 /// Saved kmp_task_t for task directive.
483 QualType SavedKmpTaskTQTy;
484 /// Saved kmp_task_t for taskloop-based directive.
485 QualType SavedKmpTaskloopTQTy;
486 /// Type typedef struct kmp_depend_info {
487 /// kmp_intptr_t base_addr;
488 /// size_t len;
489 /// struct {
490 /// bool in:1;
491 /// bool out:1;
492 /// } flags;
493 /// } kmp_depend_info_t;
494 QualType KmpDependInfoTy;
495 /// Type typedef struct kmp_task_affinity_info {
496 /// kmp_intptr_t base_addr;
497 /// size_t len;
498 /// struct {
499 /// bool flag1 : 1;
500 /// bool flag2 : 1;
501 /// kmp_int32 reserved : 30;
502 /// } flags;
503 /// } kmp_task_affinity_info_t;
504 QualType KmpTaskAffinityInfoTy;
505 /// struct kmp_dim { // loop bounds info casted to kmp_int64
506 /// kmp_int64 lo; // lower
507 /// kmp_int64 up; // upper
508 /// kmp_int64 st; // stride
509 /// };
510 QualType KmpDimTy;
511 /// Entity that registers the offloading constants that were emitted so
512 /// far.
513 llvm::OffloadEntriesInfoManager OffloadEntriesInfoManager;
514
515 bool ShouldMarkAsGlobal = true;
516 /// List of the emitted declarations.
517 llvm::DenseSet<CanonicalDeclPtr<const Decl>> AlreadyEmittedTargetDecls;
518 /// List of the global variables with their addresses that should not be
519 /// emitted for the target.
520 llvm::StringMap<llvm::WeakTrackingVH> EmittedNonTargetVariables;
521
522 /// List of variables that can become declare target implicitly and, thus,
523 /// must be emitted.
524 llvm::SmallDenseSet<const VarDecl *> DeferredGlobalVariables;
525
526 using NontemporalDeclsSet = llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>>;
527 /// Stack for list of declarations in current context marked as nontemporal.
528 /// The set is the union of all current stack elements.
529 llvm::SmallVector<NontemporalDeclsSet, 4> NontemporalDeclsStack;
530
531 using UntiedLocalVarsAddressesMap =
532 llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
533 std::pair<Address, Address>>;
534 llvm::SmallVector<UntiedLocalVarsAddressesMap, 4> UntiedLocalVarsStack;
535
536 /// Stack for list of addresses of declarations in current context marked as
537 /// lastprivate conditional. The set is the union of all current stack
538 /// elements.
539 llvm::SmallVector<LastprivateConditionalData, 4> LastprivateConditionalStack;
540
541 /// Flag for keeping track of weather a requires unified_shared_memory
542 /// directive is present.
543 bool HasRequiresUnifiedSharedMemory = false;
544
545 /// Atomic ordering from the omp requires directive.
546 llvm::AtomicOrdering RequiresAtomicOrdering = llvm::AtomicOrdering::Monotonic;
547
548 /// Flag for keeping track of weather a target region has been emitted.
549 bool HasEmittedTargetRegion = false;
550
551 /// Flag for keeping track of weather a device routine has been emitted.
552 /// Device routines are specific to the
553 bool HasEmittedDeclareTargetRegion = false;
554
555 /// Loads all the offload entries information from the host IR
556 /// metadata.
557 void loadOffloadInfoMetadata();
558
559 /// Start scanning from statement \a S and emit all target regions
560 /// found along the way.
561 /// \param S Starting statement.
562 /// \param ParentName Name of the function declaration that is being scanned.
563 void scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName);
564
565 /// Build type kmp_routine_entry_t (if not built yet).
566 void emitKmpRoutineEntryT(QualType KmpInt32Ty);
567
568 /// Returns pointer to kmpc_micro type.
569 llvm::Type *getKmpc_MicroPointerTy();
570
571 /// Returns __kmpc_for_static_init_* runtime function for the specified
572 /// size \a IVSize and sign \a IVSigned. Will create a distribute call
573 /// __kmpc_distribute_static_init* if \a IsGPUDistribute is set.
574 llvm::FunctionCallee createForStaticInitFunction(unsigned IVSize,
575 bool IVSigned,
576 bool IsGPUDistribute);
577
578 /// Returns __kmpc_dispatch_init_* runtime function for the specified
579 /// size \a IVSize and sign \a IVSigned.
580 llvm::FunctionCallee createDispatchInitFunction(unsigned IVSize,
581 bool IVSigned);
582
583 /// Returns __kmpc_dispatch_next_* runtime function for the specified
584 /// size \a IVSize and sign \a IVSigned.
585 llvm::FunctionCallee createDispatchNextFunction(unsigned IVSize,
586 bool IVSigned);
587
588 /// Returns __kmpc_dispatch_fini_* runtime function for the specified
589 /// size \a IVSize and sign \a IVSigned.
590 llvm::FunctionCallee createDispatchFiniFunction(unsigned IVSize,
591 bool IVSigned);
592
593 /// If the specified mangled name is not in the module, create and
594 /// return threadprivate cache object. This object is a pointer's worth of
595 /// storage that's reserved for use by the OpenMP runtime.
596 /// \param VD Threadprivate variable.
597 /// \return Cache variable for the specified threadprivate.
598 llvm::Constant *getOrCreateThreadPrivateCache(const VarDecl *VD);
599
600 /// Set of threadprivate variables with the generated initializer.
601 llvm::StringSet<> ThreadPrivateWithDefinition;
602
603 /// Set of declare target variables with the generated initializer.
604 llvm::StringSet<> DeclareTargetWithDefinition;
605
606 /// Emits initialization code for the threadprivate variables.
607 /// \param VDAddr Address of the global variable \a VD.
608 /// \param Ctor Pointer to a global init function for \a VD.
609 /// \param CopyCtor Pointer to a global copy function for \a VD.
610 /// \param Dtor Pointer to a global destructor function for \a VD.
611 /// \param Loc Location of threadprivate declaration.
612 void emitThreadPrivateVarInit(CodeGenFunction &CGF, Address VDAddr,
613 llvm::Value *Ctor, llvm::Value *CopyCtor,
614 llvm::Value *Dtor, SourceLocation Loc);
615
616 /// Emit the array initialization or deletion portion for user-defined mapper
617 /// code generation.
618 void emitUDMapperArrayInitOrDel(CodeGenFunction &MapperCGF,
619 llvm::Value *Handle, llvm::Value *BasePtr,
620 llvm::Value *Ptr, llvm::Value *Size,
621 llvm::Value *MapType, llvm::Value *MapName,
622 CharUnits ElementSize,
623 llvm::BasicBlock *ExitBB, bool IsInit);
624
625 struct TaskResultTy {
626 llvm::Value *NewTask = nullptr;
627 llvm::Function *TaskEntry = nullptr;
628 llvm::Value *NewTaskNewTaskTTy = nullptr;
629 LValue TDBase;
630 const RecordDecl *KmpTaskTQTyRD = nullptr;
631 llvm::Value *TaskDupFn = nullptr;
632 };
633 /// Emit task region for the task directive. The task region is emitted in
634 /// several steps:
635 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
636 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
637 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
638 /// function:
639 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
640 /// TaskFunction(gtid, tt->part_id, tt->shareds);
641 /// return 0;
642 /// }
643 /// 2. Copy a list of shared variables to field shareds of the resulting
644 /// structure kmp_task_t returned by the previous call (if any).
645 /// 3. Copy a pointer to destructions function to field destructions of the
646 /// resulting structure kmp_task_t.
647 /// \param D Current task directive.
648 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
649 /// /*part_id*/, captured_struct */*__context*/);
650 /// \param SharedsTy A type which contains references the shared variables.
651 /// \param Shareds Context with the list of shared variables from the \p
652 /// TaskFunction.
653 /// \param Data Additional data for task generation like tiednsee, final
654 /// state, list of privates etc.
655 TaskResultTy emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
656 const OMPExecutableDirective &D,
657 llvm::Function *TaskFunction, QualType SharedsTy,
658 Address Shareds, const OMPTaskDataTy &Data);
659
660 /// Return the trip count of loops associated with constructs / 'target teams
661 /// distribute' and 'teams distribute parallel for'. \param SizeEmitter Emits
662 /// the int64 value for the number of iterations of the associated loop.
663 llvm::Value *emitTargetNumIterationsCall(
664 CodeGenFunction &CGF, const OMPExecutableDirective &D,
665 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
666 const OMPLoopDirective &D)>
667 SizeEmitter);
668
669 /// Emit update for lastprivate conditional data.
670 void emitLastprivateConditionalUpdate(CodeGenFunction &CGF, LValue IVLVal,
671 StringRef UniqueDeclName, LValue LVal,
672 SourceLocation Loc);
673
674 /// Returns the number of the elements and the address of the depobj
675 /// dependency array.
676 /// \return Number of elements in depobj array and the pointer to the array of
677 /// dependencies.
678 std::pair<llvm::Value *, LValue> getDepobjElements(CodeGenFunction &CGF,
679 LValue DepobjLVal,
680 SourceLocation Loc);
681
682 SmallVector<llvm::Value *, 4>
683 emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
684 const OMPTaskDataTy::DependData &Data);
685
686 void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
687 LValue PosLVal, const OMPTaskDataTy::DependData &Data,
688 Address DependenciesArray);
689
690 public:
691 explicit CGOpenMPRuntime(CodeGenModule &CGM);
692 virtual ~CGOpenMPRuntime() {}
693 virtual void clear();
694
695 /// Returns true if the current target is a GPU.
696 virtual bool isTargetCodegen() const { return false; }
697
698 /// Emits code for OpenMP 'if' clause using specified \a CodeGen
699 /// function. Here is the logic:
700 /// if (Cond) {
701 /// ThenGen();
702 /// } else {
703 /// ElseGen();
704 /// }
705 void emitIfClause(CodeGenFunction &CGF, const Expr *Cond,
706 const RegionCodeGenTy &ThenGen,
707 const RegionCodeGenTy &ElseGen);
708
709 /// Checks if the \p Body is the \a CompoundStmt and returns its child
710 /// statement iff there is only one that is not evaluatable at the compile
711 /// time.
712 static const Stmt *getSingleCompoundChild(ASTContext &Ctx, const Stmt *Body);
713
714 /// Get the platform-specific name separator.
715 std::string getName(ArrayRef<StringRef> Parts) const;
716
717 /// Emit code for the specified user defined reduction construct.
718 virtual void emitUserDefinedReduction(CodeGenFunction *CGF,
719 const OMPDeclareReductionDecl *D);
720 /// Get combiner/initializer for the specified user-defined reduction, if any.
721 virtual std::pair<llvm::Function *, llvm::Function *>
722 getUserDefinedReduction(const OMPDeclareReductionDecl *D);
723
724 /// Emit the function for the user defined mapper construct.
725 void emitUserDefinedMapper(const OMPDeclareMapperDecl *D,
726 CodeGenFunction *CGF = nullptr);
727 /// Get the function for the specified user-defined mapper. If it does not
728 /// exist, create one.
729 llvm::Function *
730 getOrCreateUserDefinedMapperFunc(const OMPDeclareMapperDecl *D);
731
732 /// Emits outlined function for the specified OpenMP parallel directive
733 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
734 /// kmp_int32 BoundID, struct context_vars*).
735 /// \param D OpenMP directive.
736 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
737 /// \param InnermostKind Kind of innermost directive (for simple directives it
738 /// is a directive itself, for combined - its innermost directive).
739 /// \param CodeGen Code generation sequence for the \a D directive.
740 virtual llvm::Function *emitParallelOutlinedFunction(
741 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
742 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen);
743
744 /// Emits outlined function for the specified OpenMP teams directive
745 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
746 /// kmp_int32 BoundID, struct context_vars*).
747 /// \param D OpenMP directive.
748 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
749 /// \param InnermostKind Kind of innermost directive (for simple directives it
750 /// is a directive itself, for combined - its innermost directive).
751 /// \param CodeGen Code generation sequence for the \a D directive.
752 virtual llvm::Function *emitTeamsOutlinedFunction(
753 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
754 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen);
755
756 /// Emits outlined function for the OpenMP task directive \a D. This
757 /// outlined function has type void(*)(kmp_int32 ThreadID, struct task_t*
758 /// TaskT).
759 /// \param D OpenMP directive.
760 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
761 /// \param PartIDVar Variable for partition id in the current OpenMP untied
762 /// task region.
763 /// \param TaskTVar Variable for task_t argument.
764 /// \param InnermostKind Kind of innermost directive (for simple directives it
765 /// is a directive itself, for combined - its innermost directive).
766 /// \param CodeGen Code generation sequence for the \a D directive.
767 /// \param Tied true if task is generated for tied task, false otherwise.
768 /// \param NumberOfParts Number of parts in untied task. Ignored for tied
769 /// tasks.
770 ///
771 virtual llvm::Function *emitTaskOutlinedFunction(
772 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
773 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
774 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
775 bool Tied, unsigned &NumberOfParts);
776
777 /// Cleans up references to the objects in finished function.
778 ///
779 virtual void functionFinished(CodeGenFunction &CGF);
780
781 /// Emits code for parallel or serial call of the \a OutlinedFn with
782 /// variables captured in a record which address is stored in \a
783 /// CapturedStruct.
784 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
785 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
786 /// \param CapturedVars A pointer to the record with the references to
787 /// variables used in \a OutlinedFn function.
788 /// \param IfCond Condition in the associated 'if' clause, if it was
789 /// specified, nullptr otherwise.
790 /// \param NumThreads The value corresponding to the num_threads clause, if
791 /// any, or nullptr.
792 ///
793 virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
794 llvm::Function *OutlinedFn,
795 ArrayRef<llvm::Value *> CapturedVars,
796 const Expr *IfCond, llvm::Value *NumThreads);
797
798 /// Emits a critical region.
799 /// \param CriticalName Name of the critical region.
800 /// \param CriticalOpGen Generator for the statement associated with the given
801 /// critical region.
802 /// \param Hint Value of the 'hint' clause (optional).
803 virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,
804 const RegionCodeGenTy &CriticalOpGen,
805 SourceLocation Loc,
806 const Expr *Hint = nullptr);
807
808 /// Emits a master region.
809 /// \param MasterOpGen Generator for the statement associated with the given
810 /// master region.
811 virtual void emitMasterRegion(CodeGenFunction &CGF,
812 const RegionCodeGenTy &MasterOpGen,
813 SourceLocation Loc);
814
815 /// Emits a masked region.
816 /// \param MaskedOpGen Generator for the statement associated with the given
817 /// masked region.
818 virtual void emitMaskedRegion(CodeGenFunction &CGF,
819 const RegionCodeGenTy &MaskedOpGen,
820 SourceLocation Loc,
821 const Expr *Filter = nullptr);
822
823 /// Emits code for a taskyield directive.
824 virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc);
825
826 /// Emit __kmpc_error call for error directive
827 /// extern void __kmpc_error(ident_t *loc, int severity, const char *message);
828 virtual void emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc, Expr *ME,
829 bool IsFatal);
830
831 /// Emit a taskgroup region.
832 /// \param TaskgroupOpGen Generator for the statement associated with the
833 /// given taskgroup region.
834 virtual void emitTaskgroupRegion(CodeGenFunction &CGF,
835 const RegionCodeGenTy &TaskgroupOpGen,
836 SourceLocation Loc);
837
838 /// Emits a single region.
839 /// \param SingleOpGen Generator for the statement associated with the given
840 /// single region.
841 virtual void emitSingleRegion(CodeGenFunction &CGF,
842 const RegionCodeGenTy &SingleOpGen,
843 SourceLocation Loc,
844 ArrayRef<const Expr *> CopyprivateVars,
845 ArrayRef<const Expr *> DestExprs,
846 ArrayRef<const Expr *> SrcExprs,
847 ArrayRef<const Expr *> AssignmentOps);
848
849 /// Emit an ordered region.
850 /// \param OrderedOpGen Generator for the statement associated with the given
851 /// ordered region.
852 virtual void emitOrderedRegion(CodeGenFunction &CGF,
853 const RegionCodeGenTy &OrderedOpGen,
854 SourceLocation Loc, bool IsThreads);
855
856 /// Emit an implicit/explicit barrier for OpenMP threads.
857 /// \param Kind Directive for which this implicit barrier call must be
858 /// generated. Must be OMPD_barrier for explicit barrier generation.
859 /// \param EmitChecks true if need to emit checks for cancellation barriers.
860 /// \param ForceSimpleCall true simple barrier call must be emitted, false if
861 /// runtime class decides which one to emit (simple or with cancellation
862 /// checks).
863 ///
864 virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
865 OpenMPDirectiveKind Kind,
866 bool EmitChecks = true,
867 bool ForceSimpleCall = false);
868
869 /// Check if the specified \a ScheduleKind is static non-chunked.
870 /// This kind of worksharing directive is emitted without outer loop.
871 /// \param ScheduleKind Schedule kind specified in the 'schedule' clause.
872 /// \param Chunked True if chunk is specified in the clause.
873 ///
874 virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
875 bool Chunked) const;
876
877 /// Check if the specified \a ScheduleKind is static non-chunked.
878 /// This kind of distribute directive is emitted without outer loop.
879 /// \param ScheduleKind Schedule kind specified in the 'dist_schedule' clause.
880 /// \param Chunked True if chunk is specified in the clause.
881 ///
882 virtual bool isStaticNonchunked(OpenMPDistScheduleClauseKind ScheduleKind,
883 bool Chunked) const;
884
885 /// Check if the specified \a ScheduleKind is static chunked.
886 /// \param ScheduleKind Schedule kind specified in the 'schedule' clause.
887 /// \param Chunked True if chunk is specified in the clause.
888 ///
889 virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
890 bool Chunked) const;
891
892 /// Check if the specified \a ScheduleKind is static non-chunked.
893 /// \param ScheduleKind Schedule kind specified in the 'dist_schedule' clause.
894 /// \param Chunked True if chunk is specified in the clause.
895 ///
896 virtual bool isStaticChunked(OpenMPDistScheduleClauseKind ScheduleKind,
897 bool Chunked) const;
898
899 /// Check if the specified \a ScheduleKind is dynamic.
900 /// This kind of worksharing directive is emitted without outer loop.
901 /// \param ScheduleKind Schedule Kind specified in the 'schedule' clause.
902 ///
903 virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const;
904
905 /// struct with the values to be passed to the dispatch runtime function
906 struct DispatchRTInput {
907 /// Loop lower bound
908 llvm::Value *LB = nullptr;
909 /// Loop upper bound
910 llvm::Value *UB = nullptr;
911 /// Chunk size specified using 'schedule' clause (nullptr if chunk
912 /// was not specified)
913 llvm::Value *Chunk = nullptr;
914 DispatchRTInput() = default;
915 DispatchRTInput(llvm::Value *LB, llvm::Value *UB, llvm::Value *Chunk)
916 : LB(LB), UB(UB), Chunk(Chunk) {}
917 };
918
919 /// Call the appropriate runtime routine to initialize it before start
920 /// of loop.
921
922 /// This is used for non static scheduled types and when the ordered
923 /// clause is present on the loop construct.
924 /// Depending on the loop schedule, it is necessary to call some runtime
925 /// routine before start of the OpenMP loop to get the loop upper / lower
926 /// bounds \a LB and \a UB and stride \a ST.
927 ///
928 /// \param CGF Reference to current CodeGenFunction.
929 /// \param Loc Clang source location.
930 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
931 /// \param IVSize Size of the iteration variable in bits.
932 /// \param IVSigned Sign of the iteration variable.
933 /// \param Ordered true if loop is ordered, false otherwise.
934 /// \param DispatchValues struct containing llvm values for lower bound, upper
935 /// bound, and chunk expression.
936 /// For the default (nullptr) value, the chunk 1 will be used.
937 ///
938 virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc,
939 const OpenMPScheduleTy &ScheduleKind,
940 unsigned IVSize, bool IVSigned, bool Ordered,
941 const DispatchRTInput &DispatchValues);
942
943 /// Struct with the values to be passed to the static runtime function
944 struct StaticRTInput {
945 /// Size of the iteration variable in bits.
946 unsigned IVSize = 0;
947 /// Sign of the iteration variable.
948 bool IVSigned = false;
949 /// true if loop is ordered, false otherwise.
950 bool Ordered = false;
951 /// Address of the output variable in which the flag of the last iteration
952 /// is returned.
953 Address IL = Address::invalid();
954 /// Address of the output variable in which the lower iteration number is
955 /// returned.
956 Address LB = Address::invalid();
957 /// Address of the output variable in which the upper iteration number is
958 /// returned.
959 Address UB = Address::invalid();
960 /// Address of the output variable in which the stride value is returned
961 /// necessary to generated the static_chunked scheduled loop.
962 Address ST = Address::invalid();
963 /// Value of the chunk for the static_chunked scheduled loop. For the
964 /// default (nullptr) value, the chunk 1 will be used.
965 llvm::Value *Chunk = nullptr;
966 StaticRTInput(unsigned IVSize, bool IVSigned, bool Ordered, Address IL,
967 Address LB, Address UB, Address ST,
968 llvm::Value *Chunk = nullptr)
969 : IVSize(IVSize), IVSigned(IVSigned), Ordered(Ordered), IL(IL), LB(LB),
970 UB(UB), ST(ST), Chunk(Chunk) {}
971 };
972 /// Call the appropriate runtime routine to initialize it before start
973 /// of loop.
974 ///
975 /// This is used only in case of static schedule, when the user did not
976 /// specify a ordered clause on the loop construct.
977 /// Depending on the loop schedule, it is necessary to call some runtime
978 /// routine before start of the OpenMP loop to get the loop upper / lower
979 /// bounds LB and UB and stride ST.
980 ///
981 /// \param CGF Reference to current CodeGenFunction.
982 /// \param Loc Clang source location.
983 /// \param DKind Kind of the directive.
984 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
985 /// \param Values Input arguments for the construct.
986 ///
987 virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
988 OpenMPDirectiveKind DKind,
989 const OpenMPScheduleTy &ScheduleKind,
990 const StaticRTInput &Values);
991
992 ///
993 /// \param CGF Reference to current CodeGenFunction.
994 /// \param Loc Clang source location.
995 /// \param SchedKind Schedule kind, specified by the 'dist_schedule' clause.
996 /// \param Values Input arguments for the construct.
997 ///
998 virtual void emitDistributeStaticInit(CodeGenFunction &CGF,
999 SourceLocation Loc,
1000 OpenMPDistScheduleClauseKind SchedKind,
1001 const StaticRTInput &Values);
1002
1003 /// Call the appropriate runtime routine to notify that we finished
1004 /// iteration of the ordered loop with the dynamic scheduling.
1005 ///
1006 /// \param CGF Reference to current CodeGenFunction.
1007 /// \param Loc Clang source location.
1008 /// \param IVSize Size of the iteration variable in bits.
1009 /// \param IVSigned Sign of the iteration variable.
1010 ///
1011 virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF,
1012 SourceLocation Loc, unsigned IVSize,
1013 bool IVSigned);
1014
1015 /// Call the appropriate runtime routine to notify that we finished
1016 /// all the work with current loop.
1017 ///
1018 /// \param CGF Reference to current CodeGenFunction.
1019 /// \param Loc Clang source location.
1020 /// \param DKind Kind of the directive for which the static finish is emitted.
1021 ///
1022 virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc,
1023 OpenMPDirectiveKind DKind);
1024
1025 /// Call __kmpc_dispatch_next(
1026 /// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
1027 /// kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
1028 /// kmp_int[32|64] *p_stride);
1029 /// \param IVSize Size of the iteration variable in bits.
1030 /// \param IVSigned Sign of the iteration variable.
1031 /// \param IL Address of the output variable in which the flag of the
1032 /// last iteration is returned.
1033 /// \param LB Address of the output variable in which the lower iteration
1034 /// number is returned.
1035 /// \param UB Address of the output variable in which the upper iteration
1036 /// number is returned.
1037 /// \param ST Address of the output variable in which the stride value is
1038 /// returned.
1039 virtual llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc,
1040 unsigned IVSize, bool IVSigned,
1041 Address IL, Address LB,
1042 Address UB, Address ST);
1043
1044 /// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32
1045 /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'
1046 /// clause.
1047 /// \param NumThreads An integer value of threads.
1048 virtual void emitNumThreadsClause(CodeGenFunction &CGF,
1049 llvm::Value *NumThreads,
1050 SourceLocation Loc);
1051
1052 /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32
1053 /// global_tid, int proc_bind) to generate code for 'proc_bind' clause.
1054 virtual void emitProcBindClause(CodeGenFunction &CGF,
1055 llvm::omp::ProcBindKind ProcBind,
1056 SourceLocation Loc);
1057
1058 /// Returns address of the threadprivate variable for the current
1059 /// thread.
1060 /// \param VD Threadprivate variable.
1061 /// \param VDAddr Address of the global variable \a VD.
1062 /// \param Loc Location of the reference to threadprivate var.
1063 /// \return Address of the threadprivate variable for the current thread.
1064 virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF,
1065 const VarDecl *VD,
1066 Address VDAddr,
1067 SourceLocation Loc);
1068
1069 /// Returns the address of the variable marked as declare target with link
1070 /// clause OR as declare target with to clause and unified memory.
1071 virtual Address getAddrOfDeclareTargetVar(const VarDecl *VD);
1072
1073 /// Emit a code for initialization of threadprivate variable. It emits
1074 /// a call to runtime library which adds initial value to the newly created
1075 /// threadprivate variable (if it is not constant) and registers destructor
1076 /// for the variable (if any).
1077 /// \param VD Threadprivate variable.
1078 /// \param VDAddr Address of the global variable \a VD.
1079 /// \param Loc Location of threadprivate declaration.
1080 /// \param PerformInit true if initialization expression is not constant.
1081 virtual llvm::Function *
1082 emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr,
1083 SourceLocation Loc, bool PerformInit,
1084 CodeGenFunction *CGF = nullptr);
1085
1086 /// Emit a code for initialization of declare target variable.
1087 /// \param VD Declare target variable.
1088 /// \param Addr Address of the global variable \a VD.
1089 /// \param PerformInit true if initialization expression is not constant.
1090 virtual bool emitDeclareTargetVarDefinition(const VarDecl *VD,
1091 llvm::GlobalVariable *Addr,
1092 bool PerformInit);
1093
1094 /// Creates artificial threadprivate variable with name \p Name and type \p
1095 /// VarType.
1096 /// \param VarType Type of the artificial threadprivate variable.
1097 /// \param Name Name of the artificial threadprivate variable.
1098 virtual Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
1099 QualType VarType,
1100 StringRef Name);
1101
1102 /// Emit flush of the variables specified in 'omp flush' directive.
1103 /// \param Vars List of variables to flush.
1104 virtual void emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars,
1105 SourceLocation Loc, llvm::AtomicOrdering AO);
1106
1107 /// Emit task region for the task directive. The task region is
1108 /// emitted in several steps:
1109 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1110 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1111 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1112 /// function:
1113 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1114 /// TaskFunction(gtid, tt->part_id, tt->shareds);
1115 /// return 0;
1116 /// }
1117 /// 2. Copy a list of shared variables to field shareds of the resulting
1118 /// structure kmp_task_t returned by the previous call (if any).
1119 /// 3. Copy a pointer to destructions function to field destructions of the
1120 /// resulting structure kmp_task_t.
1121 /// 4. Emit a call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid,
1122 /// kmp_task_t *new_task), where new_task is a resulting structure from
1123 /// previous items.
1124 /// \param D Current task directive.
1125 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
1126 /// /*part_id*/, captured_struct */*__context*/);
1127 /// \param SharedsTy A type which contains references the shared variables.
1128 /// \param Shareds Context with the list of shared variables from the \p
1129 /// TaskFunction.
1130 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
1131 /// otherwise.
1132 /// \param Data Additional data for task generation like tiednsee, final
1133 /// state, list of privates etc.
1134 virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
1135 const OMPExecutableDirective &D,
1136 llvm::Function *TaskFunction, QualType SharedsTy,
1137 Address Shareds, const Expr *IfCond,
1138 const OMPTaskDataTy &Data);
1139
1140 /// Emit task region for the taskloop directive. The taskloop region is
1141 /// emitted in several steps:
1142 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1143 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1144 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1145 /// function:
1146 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1147 /// TaskFunction(gtid, tt->part_id, tt->shareds);
1148 /// return 0;
1149 /// }
1150 /// 2. Copy a list of shared variables to field shareds of the resulting
1151 /// structure kmp_task_t returned by the previous call (if any).
1152 /// 3. Copy a pointer to destructions function to field destructions of the
1153 /// resulting structure kmp_task_t.
1154 /// 4. Emit a call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t
1155 /// *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int
1156 /// nogroup, int sched, kmp_uint64 grainsize, void *task_dup ), where new_task
1157 /// is a resulting structure from
1158 /// previous items.
1159 /// \param D Current task directive.
1160 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
1161 /// /*part_id*/, captured_struct */*__context*/);
1162 /// \param SharedsTy A type which contains references the shared variables.
1163 /// \param Shareds Context with the list of shared variables from the \p
1164 /// TaskFunction.
1165 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
1166 /// otherwise.
1167 /// \param Data Additional data for task generation like tiednsee, final
1168 /// state, list of privates etc.
1169 virtual void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
1170 const OMPLoopDirective &D,
1171 llvm::Function *TaskFunction,
1172 QualType SharedsTy, Address Shareds,
1173 const Expr *IfCond, const OMPTaskDataTy &Data);
1174
1175 /// Emit code for the directive that does not require outlining.
1176 ///
1177 /// \param InnermostKind Kind of innermost directive (for simple directives it
1178 /// is a directive itself, for combined - its innermost directive).
1179 /// \param CodeGen Code generation sequence for the \a D directive.
1180 /// \param HasCancel true if region has inner cancel directive, false
1181 /// otherwise.
1182 virtual void emitInlinedDirective(CodeGenFunction &CGF,
1183 OpenMPDirectiveKind InnermostKind,
1184 const RegionCodeGenTy &CodeGen,
1185 bool HasCancel = false);
1186
1187 /// Emits reduction function.
1188 /// \param ArgsElemType Array type containing pointers to reduction variables.
1189 /// \param Privates List of private copies for original reduction arguments.
1190 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
1191 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
1192 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
1193 /// or 'operator binop(LHS, RHS)'.
1194 llvm::Function *emitReductionFunction(SourceLocation Loc,
1195 llvm::Type *ArgsElemType,
1196 ArrayRef<const Expr *> Privates,
1197 ArrayRef<const Expr *> LHSExprs,
1198 ArrayRef<const Expr *> RHSExprs,
1199 ArrayRef<const Expr *> ReductionOps);
1200
1201 /// Emits single reduction combiner
1202 void emitSingleReductionCombiner(CodeGenFunction &CGF,
1203 const Expr *ReductionOp,
1204 const Expr *PrivateRef,
1205 const DeclRefExpr *LHS,
1206 const DeclRefExpr *RHS);
1207
1208 struct ReductionOptionsTy {
1209 bool WithNowait;
1210 bool SimpleReduction;
1211 OpenMPDirectiveKind ReductionKind;
1212 };
1213 /// Emit a code for reduction clause. Next code should be emitted for
1214 /// reduction:
1215 /// \code
1216 ///
1217 /// static kmp_critical_name lock = { 0 };
1218 ///
1219 /// void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
1220 /// ...
1221 /// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
1222 /// ...
1223 /// }
1224 ///
1225 /// ...
1226 /// void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
1227 /// switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
1228 /// RedList, reduce_func, &<lock>)) {
1229 /// case 1:
1230 /// ...
1231 /// <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
1232 /// ...
1233 /// __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
1234 /// break;
1235 /// case 2:
1236 /// ...
1237 /// Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
1238 /// ...
1239 /// break;
1240 /// default:;
1241 /// }
1242 /// \endcode
1243 ///
1244 /// \param Privates List of private copies for original reduction arguments.
1245 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
1246 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
1247 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
1248 /// or 'operator binop(LHS, RHS)'.
1249 /// \param Options List of options for reduction codegen:
1250 /// WithNowait true if parent directive has also nowait clause, false
1251 /// otherwise.
1252 /// SimpleReduction Emit reduction operation only. Used for omp simd
1253 /// directive on the host.
1254 /// ReductionKind The kind of reduction to perform.
1255 virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
1256 ArrayRef<const Expr *> Privates,
1257 ArrayRef<const Expr *> LHSExprs,
1258 ArrayRef<const Expr *> RHSExprs,
1259 ArrayRef<const Expr *> ReductionOps,
1260 ReductionOptionsTy Options);
1261
1262 /// Emit a code for initialization of task reduction clause. Next code
1263 /// should be emitted for reduction:
1264 /// \code
1265 ///
1266 /// _taskred_item_t red_data[n];
1267 /// ...
1268 /// red_data[i].shar = &shareds[i];
1269 /// red_data[i].orig = &origs[i];
1270 /// red_data[i].size = sizeof(origs[i]);
1271 /// red_data[i].f_init = (void*)RedInit<i>;
1272 /// red_data[i].f_fini = (void*)RedDest<i>;
1273 /// red_data[i].f_comb = (void*)RedOp<i>;
1274 /// red_data[i].flags = <Flag_i>;
1275 /// ...
1276 /// void* tg1 = __kmpc_taskred_init(gtid, n, red_data);
1277 /// \endcode
1278 /// For reduction clause with task modifier it emits the next call:
1279 /// \code
1280 ///
1281 /// _taskred_item_t red_data[n];
1282 /// ...
1283 /// red_data[i].shar = &shareds[i];
1284 /// red_data[i].orig = &origs[i];
1285 /// red_data[i].size = sizeof(origs[i]);
1286 /// red_data[i].f_init = (void*)RedInit<i>;
1287 /// red_data[i].f_fini = (void*)RedDest<i>;
1288 /// red_data[i].f_comb = (void*)RedOp<i>;
1289 /// red_data[i].flags = <Flag_i>;
1290 /// ...
1291 /// void* tg1 = __kmpc_taskred_modifier_init(loc, gtid, is_worksharing, n,
1292 /// red_data);
1293 /// \endcode
1294 /// \param LHSExprs List of LHS in \a Data.ReductionOps reduction operations.
1295 /// \param RHSExprs List of RHS in \a Data.ReductionOps reduction operations.
1296 /// \param Data Additional data for task generation like tiedness, final
1297 /// state, list of privates, reductions etc.
1298 virtual llvm::Value *emitTaskReductionInit(CodeGenFunction &CGF,
1299 SourceLocation Loc,
1300 ArrayRef<const Expr *> LHSExprs,
1301 ArrayRef<const Expr *> RHSExprs,
1302 const OMPTaskDataTy &Data);
1303
1304 /// Emits the following code for reduction clause with task modifier:
1305 /// \code
1306 /// __kmpc_task_reduction_modifier_fini(loc, gtid, is_worksharing);
1307 /// \endcode
1308 virtual void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc,
1309 bool IsWorksharingReduction);
1310
1311 /// Required to resolve existing problems in the runtime. Emits threadprivate
1312 /// variables to store the size of the VLAs/array sections for
1313 /// initializer/combiner/finalizer functions.
1314 /// \param RCG Allows to reuse an existing data for the reductions.
1315 /// \param N Reduction item for which fixups must be emitted.
1316 virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc,
1317 ReductionCodeGen &RCG, unsigned N);
1318
1319 /// Get the address of `void *` type of the privatue copy of the reduction
1320 /// item specified by the \p SharedLVal.
1321 /// \param ReductionsPtr Pointer to the reduction data returned by the
1322 /// emitTaskReductionInit function.
1323 /// \param SharedLVal Address of the original reduction item.
1324 virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc,
1325 llvm::Value *ReductionsPtr,
1326 LValue SharedLVal);
1327
1328 /// Emit code for 'taskwait' directive.
1329 virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
1330 const OMPTaskDataTy &Data);
1331
1332 /// Emit code for 'cancellation point' construct.
1333 /// \param CancelRegion Region kind for which the cancellation point must be
1334 /// emitted.
1335 ///
1336 virtual void emitCancellationPointCall(CodeGenFunction &CGF,
1337 SourceLocation Loc,
1338 OpenMPDirectiveKind CancelRegion);
1339
1340 /// Emit code for 'cancel' construct.
1341 /// \param IfCond Condition in the associated 'if' clause, if it was
1342 /// specified, nullptr otherwise.
1343 /// \param CancelRegion Region kind for which the cancel must be emitted.
1344 ///
1345 virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
1346 const Expr *IfCond,
1347 OpenMPDirectiveKind CancelRegion);
1348
1349 /// Emit outilined function for 'target' directive.
1350 /// \param D Directive to emit.
1351 /// \param ParentName Name of the function that encloses the target region.
1352 /// \param OutlinedFn Outlined function value to be defined by this call.
1353 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
1354 /// \param IsOffloadEntry True if the outlined function is an offload entry.
1355 /// \param CodeGen Code generation sequence for the \a D directive.
1356 /// An outlined function may not be an entry if, e.g. the if clause always
1357 /// evaluates to false.
1358 virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D,
1359 StringRef ParentName,
1360 llvm::Function *&OutlinedFn,
1361 llvm::Constant *&OutlinedFnID,
1362 bool IsOffloadEntry,
1363 const RegionCodeGenTy &CodeGen);
1364
1365 /// Emit the target offloading code associated with \a D. The emitted
1366 /// code attempts offloading the execution to the device, an the event of
1367 /// a failure it executes the host version outlined in \a OutlinedFn.
1368 /// \param D Directive to emit.
1369 /// \param OutlinedFn Host version of the code to be offloaded.
1370 /// \param OutlinedFnID ID of host version of the code to be offloaded.
1371 /// \param IfCond Expression evaluated in if clause associated with the target
1372 /// directive, or null if no if clause is used.
1373 /// \param Device Expression evaluated in device clause associated with the
1374 /// target directive, or null if no device clause is used and device modifier.
1375 /// \param SizeEmitter Callback to emit number of iterations for loop-based
1376 /// directives.
1377 virtual void emitTargetCall(
1378 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1379 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
1380 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
1381 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
1382 const OMPLoopDirective &D)>
1383 SizeEmitter);
1384
1385 /// Emit the target regions enclosed in \a GD function definition or
1386 /// the function itself in case it is a valid device function. Returns true if
1387 /// \a GD was dealt with successfully.
1388 /// \param GD Function to scan.
1389 virtual bool emitTargetFunctions(GlobalDecl GD);
1390
1391 /// Emit the global variable if it is a valid device global variable.
1392 /// Returns true if \a GD was dealt with successfully.
1393 /// \param GD Variable declaration to emit.
1394 virtual bool emitTargetGlobalVariable(GlobalDecl GD);
1395
1396 /// Checks if the provided global decl \a GD is a declare target variable and
1397 /// registers it when emitting code for the host.
1398 virtual void registerTargetGlobalVariable(const VarDecl *VD,
1399 llvm::Constant *Addr);
1400
1401 /// Emit the global \a GD if it is meaningful for the target. Returns
1402 /// if it was emitted successfully.
1403 /// \param GD Global to scan.
1404 virtual bool emitTargetGlobal(GlobalDecl GD);
1405
1406 /// Creates and returns a registration function for when at least one
1407 /// requires directives was used in the current module.
1408 llvm::Function *emitRequiresDirectiveRegFun();
1409
1410 /// Creates all the offload entries in the current compilation unit
1411 /// along with the associated metadata.
1412 void createOffloadEntriesAndInfoMetadata();
1413
1414 /// Emits code for teams call of the \a OutlinedFn with
1415 /// variables captured in a record which address is stored in \a
1416 /// CapturedStruct.
1417 /// \param OutlinedFn Outlined function to be run by team masters. Type of
1418 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
1419 /// \param CapturedVars A pointer to the record with the references to
1420 /// variables used in \a OutlinedFn function.
1421 ///
1422 virtual void emitTeamsCall(CodeGenFunction &CGF,
1423 const OMPExecutableDirective &D,
1424 SourceLocation Loc, llvm::Function *OutlinedFn,
1425 ArrayRef<llvm::Value *> CapturedVars);
1426
1427 /// Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32
1428 /// global_tid, kmp_int32 num_teams, kmp_int32 thread_limit) to generate code
1429 /// for num_teams clause.
1430 /// \param NumTeams An integer expression of teams.
1431 /// \param ThreadLimit An integer expression of threads.
1432 virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams,
1433 const Expr *ThreadLimit, SourceLocation Loc);
1434
1435 /// Struct that keeps all the relevant information that should be kept
1436 /// throughout a 'target data' region.
1437 class TargetDataInfo : public llvm::OpenMPIRBuilder::TargetDataInfo {
1438 public:
1439 explicit TargetDataInfo() : llvm::OpenMPIRBuilder::TargetDataInfo() {}
1440 explicit TargetDataInfo(bool RequiresDevicePointerInfo,
1441 bool SeparateBeginEndCalls)
1442 : llvm::OpenMPIRBuilder::TargetDataInfo(RequiresDevicePointerInfo,
1443 SeparateBeginEndCalls) {}
1444 /// Map between the a declaration of a capture and the corresponding base
1445 /// pointer address where the runtime returns the device pointers.
1446 llvm::DenseMap<const ValueDecl *, Address> CaptureDeviceAddrMap;
1447 };
1448
1449 /// Emit the target data mapping code associated with \a D.
1450 /// \param D Directive to emit.
1451 /// \param IfCond Expression evaluated in if clause associated with the
1452 /// target directive, or null if no device clause is used.
1453 /// \param Device Expression evaluated in device clause associated with the
1454 /// target directive, or null if no device clause is used.
1455 /// \param Info A record used to store information that needs to be preserved
1456 /// until the region is closed.
1457 virtual void emitTargetDataCalls(CodeGenFunction &CGF,
1458 const OMPExecutableDirective &D,
1459 const Expr *IfCond, const Expr *Device,
1460 const RegionCodeGenTy &CodeGen,
1461 CGOpenMPRuntime::TargetDataInfo &Info);
1462
1463 /// Emit the data mapping/movement code associated with the directive
1464 /// \a D that should be of the form 'target [{enter|exit} data | update]'.
1465 /// \param D Directive to emit.
1466 /// \param IfCond Expression evaluated in if clause associated with the target
1467 /// directive, or null if no if clause is used.
1468 /// \param Device Expression evaluated in device clause associated with the
1469 /// target directive, or null if no device clause is used.
1470 virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF,
1471 const OMPExecutableDirective &D,
1472 const Expr *IfCond,
1473 const Expr *Device);
1474
1475 /// Marks function \a Fn with properly mangled versions of vector functions.
1476 /// \param FD Function marked as 'declare simd'.
1477 /// \param Fn LLVM function that must be marked with 'declare simd'
1478 /// attributes.
1479 virtual void emitDeclareSimdFunction(const FunctionDecl *FD,
1480 llvm::Function *Fn);
1481
1482 /// Emit initialization for doacross loop nesting support.
1483 /// \param D Loop-based construct used in doacross nesting construct.
1484 virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D,
1485 ArrayRef<Expr *> NumIterations);
1486
1487 /// Emit code for doacross ordered directive with 'depend' clause.
1488 /// \param C 'depend' clause with 'sink|source' dependency kind.
1489 virtual void emitDoacrossOrdered(CodeGenFunction &CGF,
1490 const OMPDependClause *C);
1491
1492 /// Translates the native parameter of outlined function if this is required
1493 /// for target.
1494 /// \param FD Field decl from captured record for the parameter.
1495 /// \param NativeParam Parameter itself.
1496 virtual const VarDecl *translateParameter(const FieldDecl *FD,
1497 const VarDecl *NativeParam) const {
1498 return NativeParam;
1499 }
1500
1501 /// Gets the address of the native argument basing on the address of the
1502 /// target-specific parameter.
1503 /// \param NativeParam Parameter itself.
1504 /// \param TargetParam Corresponding target-specific parameter.
1505 virtual Address getParameterAddress(CodeGenFunction &CGF,
1506 const VarDecl *NativeParam,
1507 const VarDecl *TargetParam) const;
1508
1509 /// Choose default schedule type and chunk value for the
1510 /// dist_schedule clause.
1511 virtual void getDefaultDistScheduleAndChunk(CodeGenFunction &CGF,
1512 const OMPLoopDirective &S, OpenMPDistScheduleClauseKind &ScheduleKind,
1513 llvm::Value *&Chunk) const {}
1514
1515 /// Choose default schedule type and chunk value for the
1516 /// schedule clause.
1517 virtual void getDefaultScheduleAndChunk(CodeGenFunction &CGF,
1518 const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind,
1519 const Expr *&ChunkExpr) const;
1520
1521 /// Emits call of the outlined function with the provided arguments,
1522 /// translating these arguments to correct target-specific arguments.
1523 virtual void
1524 emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc,
1525 llvm::FunctionCallee OutlinedFn,
1526 ArrayRef<llvm::Value *> Args = std::nullopt) const;
1527
1528 /// Emits OpenMP-specific function prolog.
1529 /// Required for device constructs.
1530 virtual void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D);
1531
1532 /// Gets the OpenMP-specific address of the local variable.
1533 virtual Address getAddressOfLocalVariable(CodeGenFunction &CGF,
1534 const VarDecl *VD);
1535
1536 /// Marks the declaration as already emitted for the device code and returns
1537 /// true, if it was marked already, and false, otherwise.
1538 bool markAsGlobalTarget(GlobalDecl GD);
1539
1540 /// Emit deferred declare target variables marked for deferred emission.
1541 void emitDeferredTargetDecls() const;
1542
1543 /// Adjust some parameters for the target-based directives, like addresses of
1544 /// the variables captured by reference in lambdas.
1545 virtual void
1546 adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF,
1547 const OMPExecutableDirective &D) const;
1548
1549 /// Perform check on requires decl to ensure that target architecture
1550 /// supports unified addressing
1551 virtual void processRequiresDirective(const OMPRequiresDecl *D);
1552
1553 /// Gets default memory ordering as specified in requires directive.
1554 llvm::AtomicOrdering getDefaultMemoryOrdering() const;
1555
1556 /// Checks if the variable has associated OMPAllocateDeclAttr attribute with
1557 /// the predefined allocator and translates it into the corresponding address
1558 /// space.
1559 virtual bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS);
1560
1561 /// Return whether the unified_shared_memory has been specified.
1562 bool hasRequiresUnifiedSharedMemory() const;
1563
1564 /// Checks if the \p VD variable is marked as nontemporal declaration in
1565 /// current context.
1566 bool isNontemporalDecl(const ValueDecl *VD) const;
1567
1568 /// Create specialized alloca to handle lastprivate conditionals.
1569 Address emitLastprivateConditionalInit(CodeGenFunction &CGF,
1570 const VarDecl *VD);
1571
1572 /// Checks if the provided \p LVal is lastprivate conditional and emits the
1573 /// code to update the value of the original variable.
1574 /// \code
1575 /// lastprivate(conditional: a)
1576 /// ...
1577 /// <type> a;
1578 /// lp_a = ...;
1579 /// #pragma omp critical(a)
1580 /// if (last_iv_a <= iv) {
1581 /// last_iv_a = iv;
1582 /// global_a = lp_a;
1583 /// }
1584 /// \endcode
1585 virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF,
1586 const Expr *LHS);
1587
1588 /// Checks if the lastprivate conditional was updated in inner region and
1589 /// writes the value.
1590 /// \code
1591 /// lastprivate(conditional: a)
1592 /// ...
1593 /// <type> a;bool Fired = false;
1594 /// #pragma omp ... shared(a)
1595 /// {
1596 /// lp_a = ...;
1597 /// Fired = true;
1598 /// }
1599 /// if (Fired) {
1600 /// #pragma omp critical(a)
1601 /// if (last_iv_a <= iv) {
1602 /// last_iv_a = iv;
1603 /// global_a = lp_a;
1604 /// }
1605 /// Fired = false;
1606 /// }
1607 /// \endcode
1608 virtual void checkAndEmitSharedLastprivateConditional(
1609 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1610 const llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> &IgnoredDecls);
1611
1612 /// Gets the address of the global copy used for lastprivate conditional
1613 /// update, if any.
1614 /// \param PrivLVal LValue for the private copy.
1615 /// \param VD Original lastprivate declaration.
1616 virtual void emitLastprivateConditionalFinalUpdate(CodeGenFunction &CGF,
1617 LValue PrivLVal,
1618 const VarDecl *VD,
1619 SourceLocation Loc);
1620
1621 /// Emits list of dependecies based on the provided data (array of
1622 /// dependence/expression pairs).
1623 /// \returns Pointer to the first element of the array casted to VoidPtr type.
1624 std::pair<llvm::Value *, Address>
1625 emitDependClause(CodeGenFunction &CGF,
1626 ArrayRef<OMPTaskDataTy::DependData> Dependencies,
1627 SourceLocation Loc);
1628
1629 /// Emits list of dependecies based on the provided data (array of
1630 /// dependence/expression pairs) for depobj construct. In this case, the
1631 /// variable is allocated in dynamically. \returns Pointer to the first
1632 /// element of the array casted to VoidPtr type.
1633 Address emitDepobjDependClause(CodeGenFunction &CGF,
1634 const OMPTaskDataTy::DependData &Dependencies,
1635 SourceLocation Loc);
1636
1637 /// Emits the code to destroy the dependency object provided in depobj
1638 /// directive.
1639 void emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal,
1640 SourceLocation Loc);
1641
1642 /// Updates the dependency kind in the specified depobj object.
1643 /// \param DepobjLVal LValue for the main depobj object.
1644 /// \param NewDepKind New dependency kind.
1645 void emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal,
1646 OpenMPDependClauseKind NewDepKind, SourceLocation Loc);
1647
1648 /// Initializes user defined allocators specified in the uses_allocators
1649 /// clauses.
1650 void emitUsesAllocatorsInit(CodeGenFunction &CGF, const Expr *Allocator,
1651 const Expr *AllocatorTraits);
1652
1653 /// Destroys user defined allocators specified in the uses_allocators clause.
1654 void emitUsesAllocatorsFini(CodeGenFunction &CGF, const Expr *Allocator);
1655
1656 /// Returns true if the variable is a local variable in untied task.
1657 bool isLocalVarInUntiedTask(CodeGenFunction &CGF, const VarDecl *VD) const;
1658 };
1659
1660 /// Class supports emissionof SIMD-only code.
1661 class CGOpenMPSIMDRuntime final : public CGOpenMPRuntime {
1662 public:
1663 explicit CGOpenMPSIMDRuntime(CodeGenModule &CGM) : CGOpenMPRuntime(CGM) {}
1664 ~CGOpenMPSIMDRuntime() override {}
1665
1666 /// Emits outlined function for the specified OpenMP parallel directive
1667 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
1668 /// kmp_int32 BoundID, struct context_vars*).
1669 /// \param D OpenMP directive.
1670 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
1671 /// \param InnermostKind Kind of innermost directive (for simple directives it
1672 /// is a directive itself, for combined - its innermost directive).
1673 /// \param CodeGen Code generation sequence for the \a D directive.
1674 llvm::Function *
1675 emitParallelOutlinedFunction(const OMPExecutableDirective &D,
1676 const VarDecl *ThreadIDVar,
1677 OpenMPDirectiveKind InnermostKind,
1678 const RegionCodeGenTy &CodeGen) override;
1679
1680 /// Emits outlined function for the specified OpenMP teams directive
1681 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
1682 /// kmp_int32 BoundID, struct context_vars*).
1683 /// \param D OpenMP directive.
1684 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
1685 /// \param InnermostKind Kind of innermost directive (for simple directives it
1686 /// is a directive itself, for combined - its innermost directive).
1687 /// \param CodeGen Code generation sequence for the \a D directive.
1688 llvm::Function *
1689 emitTeamsOutlinedFunction(const OMPExecutableDirective &D,
1690 const VarDecl *ThreadIDVar,
1691 OpenMPDirectiveKind InnermostKind,
1692 const RegionCodeGenTy &CodeGen) override;
1693
1694 /// Emits outlined function for the OpenMP task directive \a D. This
1695 /// outlined function has type void(*)(kmp_int32 ThreadID, struct task_t*
1696 /// TaskT).
1697 /// \param D OpenMP directive.
1698 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
1699 /// \param PartIDVar Variable for partition id in the current OpenMP untied
1700 /// task region.
1701 /// \param TaskTVar Variable for task_t argument.
1702 /// \param InnermostKind Kind of innermost directive (for simple directives it
1703 /// is a directive itself, for combined - its innermost directive).
1704 /// \param CodeGen Code generation sequence for the \a D directive.
1705 /// \param Tied true if task is generated for tied task, false otherwise.
1706 /// \param NumberOfParts Number of parts in untied task. Ignored for tied
1707 /// tasks.
1708 ///
1709 llvm::Function *emitTaskOutlinedFunction(
1710 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1711 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1712 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1713 bool Tied, unsigned &NumberOfParts) override;
1714
1715 /// Emits code for parallel or serial call of the \a OutlinedFn with
1716 /// variables captured in a record which address is stored in \a
1717 /// CapturedStruct.
1718 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
1719 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
1720 /// \param CapturedVars A pointer to the record with the references to
1721 /// variables used in \a OutlinedFn function.
1722 /// \param IfCond Condition in the associated 'if' clause, if it was
1723 /// specified, nullptr otherwise.
1724 /// \param NumThreads The value corresponding to the num_threads clause, if
1725 /// any, or nullptr.
1726 ///
1727 void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
1728 llvm::Function *OutlinedFn,
1729 ArrayRef<llvm::Value *> CapturedVars,
1730 const Expr *IfCond, llvm::Value *NumThreads) override;
1731
1732 /// Emits a critical region.
1733 /// \param CriticalName Name of the critical region.
1734 /// \param CriticalOpGen Generator for the statement associated with the given
1735 /// critical region.
1736 /// \param Hint Value of the 'hint' clause (optional).
1737 void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,
1738 const RegionCodeGenTy &CriticalOpGen,
1739 SourceLocation Loc,
1740 const Expr *Hint = nullptr) override;
1741
1742 /// Emits a master region.
1743 /// \param MasterOpGen Generator for the statement associated with the given
1744 /// master region.
1745 void emitMasterRegion(CodeGenFunction &CGF,
1746 const RegionCodeGenTy &MasterOpGen,
1747 SourceLocation Loc) override;
1748
1749 /// Emits a masked region.
1750 /// \param MaskedOpGen Generator for the statement associated with the given
1751 /// masked region.
1752 void emitMaskedRegion(CodeGenFunction &CGF,
1753 const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc,
1754 const Expr *Filter = nullptr) override;
1755
1756 /// Emits a masked region.
1757 /// \param MaskedOpGen Generator for the statement associated with the given
1758 /// masked region.
1759
1760 /// Emits code for a taskyield directive.
1761 void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc) override;
1762
1763 /// Emit a taskgroup region.
1764 /// \param TaskgroupOpGen Generator for the statement associated with the
1765 /// given taskgroup region.
1766 void emitTaskgroupRegion(CodeGenFunction &CGF,
1767 const RegionCodeGenTy &TaskgroupOpGen,
1768 SourceLocation Loc) override;
1769
1770 /// Emits a single region.
1771 /// \param SingleOpGen Generator for the statement associated with the given
1772 /// single region.
1773 void emitSingleRegion(CodeGenFunction &CGF,
1774 const RegionCodeGenTy &SingleOpGen, SourceLocation Loc,
1775 ArrayRef<const Expr *> CopyprivateVars,
1776 ArrayRef<const Expr *> DestExprs,
1777 ArrayRef<const Expr *> SrcExprs,
1778 ArrayRef<const Expr *> AssignmentOps) override;
1779
1780 /// Emit an ordered region.
1781 /// \param OrderedOpGen Generator for the statement associated with the given
1782 /// ordered region.
1783 void emitOrderedRegion(CodeGenFunction &CGF,
1784 const RegionCodeGenTy &OrderedOpGen,
1785 SourceLocation Loc, bool IsThreads) override;
1786
1787 /// Emit an implicit/explicit barrier for OpenMP threads.
1788 /// \param Kind Directive for which this implicit barrier call must be
1789 /// generated. Must be OMPD_barrier for explicit barrier generation.
1790 /// \param EmitChecks true if need to emit checks for cancellation barriers.
1791 /// \param ForceSimpleCall true simple barrier call must be emitted, false if
1792 /// runtime class decides which one to emit (simple or with cancellation
1793 /// checks).
1794 ///
1795 void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
1796 OpenMPDirectiveKind Kind, bool EmitChecks = true,
1797 bool ForceSimpleCall = false) override;
1798
1799 /// This is used for non static scheduled types and when the ordered
1800 /// clause is present on the loop construct.
1801 /// Depending on the loop schedule, it is necessary to call some runtime
1802 /// routine before start of the OpenMP loop to get the loop upper / lower
1803 /// bounds \a LB and \a UB and stride \a ST.
1804 ///
1805 /// \param CGF Reference to current CodeGenFunction.
1806 /// \param Loc Clang source location.
1807 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
1808 /// \param IVSize Size of the iteration variable in bits.
1809 /// \param IVSigned Sign of the iteration variable.
1810 /// \param Ordered true if loop is ordered, false otherwise.
1811 /// \param DispatchValues struct containing llvm values for lower bound, upper
1812 /// bound, and chunk expression.
1813 /// For the default (nullptr) value, the chunk 1 will be used.
1814 ///
1815 void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc,
1816 const OpenMPScheduleTy &ScheduleKind,
1817 unsigned IVSize, bool IVSigned, bool Ordered,
1818 const DispatchRTInput &DispatchValues) override;
1819
1820 /// Call the appropriate runtime routine to initialize it before start
1821 /// of loop.
1822 ///
1823 /// This is used only in case of static schedule, when the user did not
1824 /// specify a ordered clause on the loop construct.
1825 /// Depending on the loop schedule, it is necessary to call some runtime
1826 /// routine before start of the OpenMP loop to get the loop upper / lower
1827 /// bounds LB and UB and stride ST.
1828 ///
1829 /// \param CGF Reference to current CodeGenFunction.
1830 /// \param Loc Clang source location.
1831 /// \param DKind Kind of the directive.
1832 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
1833 /// \param Values Input arguments for the construct.
1834 ///
1835 void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
1836 OpenMPDirectiveKind DKind,
1837 const OpenMPScheduleTy &ScheduleKind,
1838 const StaticRTInput &Values) override;
1839
1840 ///
1841 /// \param CGF Reference to current CodeGenFunction.
1842 /// \param Loc Clang source location.
1843 /// \param SchedKind Schedule kind, specified by the 'dist_schedule' clause.
1844 /// \param Values Input arguments for the construct.
1845 ///
1846 void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
1847 OpenMPDistScheduleClauseKind SchedKind,
1848 const StaticRTInput &Values) override;
1849
1850 /// Call the appropriate runtime routine to notify that we finished
1851 /// iteration of the ordered loop with the dynamic scheduling.
1852 ///
1853 /// \param CGF Reference to current CodeGenFunction.
1854 /// \param Loc Clang source location.
1855 /// \param IVSize Size of the iteration variable in bits.
1856 /// \param IVSigned Sign of the iteration variable.
1857 ///
1858 void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc,
1859 unsigned IVSize, bool IVSigned) override;
1860
1861 /// Call the appropriate runtime routine to notify that we finished
1862 /// all the work with current loop.
1863 ///
1864 /// \param CGF Reference to current CodeGenFunction.
1865 /// \param Loc Clang source location.
1866 /// \param DKind Kind of the directive for which the static finish is emitted.
1867 ///
1868 void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc,
1869 OpenMPDirectiveKind DKind) override;
1870
1871 /// Call __kmpc_dispatch_next(
1872 /// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
1873 /// kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
1874 /// kmp_int[32|64] *p_stride);
1875 /// \param IVSize Size of the iteration variable in bits.
1876 /// \param IVSigned Sign of the iteration variable.
1877 /// \param IL Address of the output variable in which the flag of the
1878 /// last iteration is returned.
1879 /// \param LB Address of the output variable in which the lower iteration
1880 /// number is returned.
1881 /// \param UB Address of the output variable in which the upper iteration
1882 /// number is returned.
1883 /// \param ST Address of the output variable in which the stride value is
1884 /// returned.
1885 llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc,
1886 unsigned IVSize, bool IVSigned, Address IL,
1887 Address LB, Address UB, Address ST) override;
1888
1889 /// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32
1890 /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'
1891 /// clause.
1892 /// \param NumThreads An integer value of threads.
1893 void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads,
1894 SourceLocation Loc) override;
1895
1896 /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32
1897 /// global_tid, int proc_bind) to generate code for 'proc_bind' clause.
1898 void emitProcBindClause(CodeGenFunction &CGF,
1899 llvm::omp::ProcBindKind ProcBind,
1900 SourceLocation Loc) override;
1901
1902 /// Returns address of the threadprivate variable for the current
1903 /// thread.
1904 /// \param VD Threadprivate variable.
1905 /// \param VDAddr Address of the global variable \a VD.
1906 /// \param Loc Location of the reference to threadprivate var.
1907 /// \return Address of the threadprivate variable for the current thread.
1908 Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD,
1909 Address VDAddr, SourceLocation Loc) override;
1910
1911 /// Emit a code for initialization of threadprivate variable. It emits
1912 /// a call to runtime library which adds initial value to the newly created
1913 /// threadprivate variable (if it is not constant) and registers destructor
1914 /// for the variable (if any).
1915 /// \param VD Threadprivate variable.
1916 /// \param VDAddr Address of the global variable \a VD.
1917 /// \param Loc Location of threadprivate declaration.
1918 /// \param PerformInit true if initialization expression is not constant.
1919 llvm::Function *
1920 emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr,
1921 SourceLocation Loc, bool PerformInit,
1922 CodeGenFunction *CGF = nullptr) override;
1923
1924 /// Creates artificial threadprivate variable with name \p Name and type \p
1925 /// VarType.
1926 /// \param VarType Type of the artificial threadprivate variable.
1927 /// \param Name Name of the artificial threadprivate variable.
1928 Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
1929 QualType VarType,
1930 StringRef Name) override;
1931
1932 /// Emit flush of the variables specified in 'omp flush' directive.
1933 /// \param Vars List of variables to flush.
1934 void emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars,
1935 SourceLocation Loc, llvm::AtomicOrdering AO) override;
1936
1937 /// Emit task region for the task directive. The task region is
1938 /// emitted in several steps:
1939 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1940 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1941 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1942 /// function:
1943 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1944 /// TaskFunction(gtid, tt->part_id, tt->shareds);
1945 /// return 0;
1946 /// }
1947 /// 2. Copy a list of shared variables to field shareds of the resulting
1948 /// structure kmp_task_t returned by the previous call (if any).
1949 /// 3. Copy a pointer to destructions function to field destructions of the
1950 /// resulting structure kmp_task_t.
1951 /// 4. Emit a call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid,
1952 /// kmp_task_t *new_task), where new_task is a resulting structure from
1953 /// previous items.
1954 /// \param D Current task directive.
1955 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
1956 /// /*part_id*/, captured_struct */*__context*/);
1957 /// \param SharedsTy A type which contains references the shared variables.
1958 /// \param Shareds Context with the list of shared variables from the \p
1959 /// TaskFunction.
1960 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
1961 /// otherwise.
1962 /// \param Data Additional data for task generation like tiednsee, final
1963 /// state, list of privates etc.
1964 void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
1965 const OMPExecutableDirective &D,
1966 llvm::Function *TaskFunction, QualType SharedsTy,
1967 Address Shareds, const Expr *IfCond,
1968 const OMPTaskDataTy &Data) override;
1969
1970 /// Emit task region for the taskloop directive. The taskloop region is
1971 /// emitted in several steps:
1972 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1973 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1974 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1975 /// function:
1976 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1977 /// TaskFunction(gtid, tt->part_id, tt->shareds);
1978 /// return 0;
1979 /// }
1980 /// 2. Copy a list of shared variables to field shareds of the resulting
1981 /// structure kmp_task_t returned by the previous call (if any).
1982 /// 3. Copy a pointer to destructions function to field destructions of the
1983 /// resulting structure kmp_task_t.
1984 /// 4. Emit a call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t
1985 /// *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int
1986 /// nogroup, int sched, kmp_uint64 grainsize, void *task_dup ), where new_task
1987 /// is a resulting structure from
1988 /// previous items.
1989 /// \param D Current task directive.
1990 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
1991 /// /*part_id*/, captured_struct */*__context*/);
1992 /// \param SharedsTy A type which contains references the shared variables.
1993 /// \param Shareds Context with the list of shared variables from the \p
1994 /// TaskFunction.
1995 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
1996 /// otherwise.
1997 /// \param Data Additional data for task generation like tiednsee, final
1998 /// state, list of privates etc.
1999 void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
2000 const OMPLoopDirective &D, llvm::Function *TaskFunction,
2001 QualType SharedsTy, Address Shareds, const Expr *IfCond,
2002 const OMPTaskDataTy &Data) override;
2003
2004 /// Emit a code for reduction clause. Next code should be emitted for
2005 /// reduction:
2006 /// \code
2007 ///
2008 /// static kmp_critical_name lock = { 0 };
2009 ///
2010 /// void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
2011 /// ...
2012 /// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
2013 /// ...
2014 /// }
2015 ///
2016 /// ...
2017 /// void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
2018 /// switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
2019 /// RedList, reduce_func, &<lock>)) {
2020 /// case 1:
2021 /// ...
2022 /// <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
2023 /// ...
2024 /// __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
2025 /// break;
2026 /// case 2:
2027 /// ...
2028 /// Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
2029 /// ...
2030 /// break;
2031 /// default:;
2032 /// }
2033 /// \endcode
2034 ///
2035 /// \param Privates List of private copies for original reduction arguments.
2036 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
2037 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
2038 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
2039 /// or 'operator binop(LHS, RHS)'.
2040 /// \param Options List of options for reduction codegen:
2041 /// WithNowait true if parent directive has also nowait clause, false
2042 /// otherwise.
2043 /// SimpleReduction Emit reduction operation only. Used for omp simd
2044 /// directive on the host.
2045 /// ReductionKind The kind of reduction to perform.
2046 void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
2047 ArrayRef<const Expr *> Privates,
2048 ArrayRef<const Expr *> LHSExprs,
2049 ArrayRef<const Expr *> RHSExprs,
2050 ArrayRef<const Expr *> ReductionOps,
2051 ReductionOptionsTy Options) override;
2052
2053 /// Emit a code for initialization of task reduction clause. Next code
2054 /// should be emitted for reduction:
2055 /// \code
2056 ///
2057 /// _taskred_item_t red_data[n];
2058 /// ...
2059 /// red_data[i].shar = &shareds[i];
2060 /// red_data[i].orig = &origs[i];
2061 /// red_data[i].size = sizeof(origs[i]);
2062 /// red_data[i].f_init = (void*)RedInit<i>;
2063 /// red_data[i].f_fini = (void*)RedDest<i>;
2064 /// red_data[i].f_comb = (void*)RedOp<i>;
2065 /// red_data[i].flags = <Flag_i>;
2066 /// ...
2067 /// void* tg1 = __kmpc_taskred_init(gtid, n, red_data);
2068 /// \endcode
2069 /// For reduction clause with task modifier it emits the next call:
2070 /// \code
2071 ///
2072 /// _taskred_item_t red_data[n];
2073 /// ...
2074 /// red_data[i].shar = &shareds[i];
2075 /// red_data[i].orig = &origs[i];
2076 /// red_data[i].size = sizeof(origs[i]);
2077 /// red_data[i].f_init = (void*)RedInit<i>;
2078 /// red_data[i].f_fini = (void*)RedDest<i>;
2079 /// red_data[i].f_comb = (void*)RedOp<i>;
2080 /// red_data[i].flags = <Flag_i>;
2081 /// ...
2082 /// void* tg1 = __kmpc_taskred_modifier_init(loc, gtid, is_worksharing, n,
2083 /// red_data);
2084 /// \endcode
2085 /// \param LHSExprs List of LHS in \a Data.ReductionOps reduction operations.
2086 /// \param RHSExprs List of RHS in \a Data.ReductionOps reduction operations.
2087 /// \param Data Additional data for task generation like tiedness, final
2088 /// state, list of privates, reductions etc.
2089 llvm::Value *emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc,
2090 ArrayRef<const Expr *> LHSExprs,
2091 ArrayRef<const Expr *> RHSExprs,
2092 const OMPTaskDataTy &Data) override;
2093
2094 /// Emits the following code for reduction clause with task modifier:
2095 /// \code
2096 /// __kmpc_task_reduction_modifier_fini(loc, gtid, is_worksharing);
2097 /// \endcode
2098 void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc,
2099 bool IsWorksharingReduction) override;
2100
2101 /// Required to resolve existing problems in the runtime. Emits threadprivate
2102 /// variables to store the size of the VLAs/array sections for
2103 /// initializer/combiner/finalizer functions + emits threadprivate variable to
2104 /// store the pointer to the original reduction item for the custom
2105 /// initializer defined by declare reduction construct.
2106 /// \param RCG Allows to reuse an existing data for the reductions.
2107 /// \param N Reduction item for which fixups must be emitted.
2108 void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc,
2109 ReductionCodeGen &RCG, unsigned N) override;
2110
2111 /// Get the address of `void *` type of the privatue copy of the reduction
2112 /// item specified by the \p SharedLVal.
2113 /// \param ReductionsPtr Pointer to the reduction data returned by the
2114 /// emitTaskReductionInit function.
2115 /// \param SharedLVal Address of the original reduction item.
2116 Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc,
2117 llvm::Value *ReductionsPtr,
2118 LValue SharedLVal) override;
2119
2120 /// Emit code for 'taskwait' directive.
2121 void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
2122 const OMPTaskDataTy &Data) override;
2123
2124 /// Emit code for 'cancellation point' construct.
2125 /// \param CancelRegion Region kind for which the cancellation point must be
2126 /// emitted.
2127 ///
2128 void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc,
2129 OpenMPDirectiveKind CancelRegion) override;
2130
2131 /// Emit code for 'cancel' construct.
2132 /// \param IfCond Condition in the associated 'if' clause, if it was
2133 /// specified, nullptr otherwise.
2134 /// \param CancelRegion Region kind for which the cancel must be emitted.
2135 ///
2136 void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
2137 const Expr *IfCond,
2138 OpenMPDirectiveKind CancelRegion) override;
2139
2140 /// Emit outilined function for 'target' directive.
2141 /// \param D Directive to emit.
2142 /// \param ParentName Name of the function that encloses the target region.
2143 /// \param OutlinedFn Outlined function value to be defined by this call.
2144 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
2145 /// \param IsOffloadEntry True if the outlined function is an offload entry.
2146 /// \param CodeGen Code generation sequence for the \a D directive.
2147 /// An outlined function may not be an entry if, e.g. the if clause always
2148 /// evaluates to false.
2149 void emitTargetOutlinedFunction(const OMPExecutableDirective &D,
2150 StringRef ParentName,
2151 llvm::Function *&OutlinedFn,
2152 llvm::Constant *&OutlinedFnID,
2153 bool IsOffloadEntry,
2154 const RegionCodeGenTy &CodeGen) override;
2155
2156 /// Emit the target offloading code associated with \a D. The emitted
2157 /// code attempts offloading the execution to the device, an the event of
2158 /// a failure it executes the host version outlined in \a OutlinedFn.
2159 /// \param D Directive to emit.
2160 /// \param OutlinedFn Host version of the code to be offloaded.
2161 /// \param OutlinedFnID ID of host version of the code to be offloaded.
2162 /// \param IfCond Expression evaluated in if clause associated with the target
2163 /// directive, or null if no if clause is used.
2164 /// \param Device Expression evaluated in device clause associated with the
2165 /// target directive, or null if no device clause is used and device modifier.
2166 void emitTargetCall(
2167 CodeGenFunction &CGF, const OMPExecutableDirective &D,
2168 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
2169 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
2170 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
2171 const OMPLoopDirective &D)>
2172 SizeEmitter) override;
2173
2174 /// Emit the target regions enclosed in \a GD function definition or
2175 /// the function itself in case it is a valid device function. Returns true if
2176 /// \a GD was dealt with successfully.
2177 /// \param GD Function to scan.
2178 bool emitTargetFunctions(GlobalDecl GD) override;
2179
2180 /// Emit the global variable if it is a valid device global variable.
2181 /// Returns true if \a GD was dealt with successfully.
2182 /// \param GD Variable declaration to emit.
2183 bool emitTargetGlobalVariable(GlobalDecl GD) override;
2184
2185 /// Emit the global \a GD if it is meaningful for the target. Returns
2186 /// if it was emitted successfully.
2187 /// \param GD Global to scan.
2188 bool emitTargetGlobal(GlobalDecl GD) override;
2189
2190 /// Emits code for teams call of the \a OutlinedFn with
2191 /// variables captured in a record which address is stored in \a
2192 /// CapturedStruct.
2193 /// \param OutlinedFn Outlined function to be run by team masters. Type of
2194 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
2195 /// \param CapturedVars A pointer to the record with the references to
2196 /// variables used in \a OutlinedFn function.
2197 ///
2198 void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D,
2199 SourceLocation Loc, llvm::Function *OutlinedFn,
2200 ArrayRef<llvm::Value *> CapturedVars) override;
2201
2202 /// Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32
2203 /// global_tid, kmp_int32 num_teams, kmp_int32 thread_limit) to generate code
2204 /// for num_teams clause.
2205 /// \param NumTeams An integer expression of teams.
2206 /// \param ThreadLimit An integer expression of threads.
2207 void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams,
2208 const Expr *ThreadLimit, SourceLocation Loc) override;
2209
2210 /// Emit the target data mapping code associated with \a D.
2211 /// \param D Directive to emit.
2212 /// \param IfCond Expression evaluated in if clause associated with the
2213 /// target directive, or null if no device clause is used.
2214 /// \param Device Expression evaluated in device clause associated with the
2215 /// target directive, or null if no device clause is used.
2216 /// \param Info A record used to store information that needs to be preserved
2217 /// until the region is closed.
2218 void emitTargetDataCalls(CodeGenFunction &CGF,
2219 const OMPExecutableDirective &D, const Expr *IfCond,
2220 const Expr *Device, const RegionCodeGenTy &CodeGen,
2221 CGOpenMPRuntime::TargetDataInfo &Info) override;
2222
2223 /// Emit the data mapping/movement code associated with the directive
2224 /// \a D that should be of the form 'target [{enter|exit} data | update]'.
2225 /// \param D Directive to emit.
2226 /// \param IfCond Expression evaluated in if clause associated with the target
2227 /// directive, or null if no if clause is used.
2228 /// \param Device Expression evaluated in device clause associated with the
2229 /// target directive, or null if no device clause is used.
2230 void emitTargetDataStandAloneCall(CodeGenFunction &CGF,
2231 const OMPExecutableDirective &D,
2232 const Expr *IfCond,
2233 const Expr *Device) override;
2234
2235 /// Emit initialization for doacross loop nesting support.
2236 /// \param D Loop-based construct used in doacross nesting construct.
2237 void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D,
2238 ArrayRef<Expr *> NumIterations) override;
2239
2240 /// Emit code for doacross ordered directive with 'depend' clause.
2241 /// \param C 'depend' clause with 'sink|source' dependency kind.
2242 void emitDoacrossOrdered(CodeGenFunction &CGF,
2243 const OMPDependClause *C) override;
2244
2245 /// Translates the native parameter of outlined function if this is required
2246 /// for target.
2247 /// \param FD Field decl from captured record for the parameter.
2248 /// \param NativeParam Parameter itself.
2249 const VarDecl *translateParameter(const FieldDecl *FD,
2250 const VarDecl *NativeParam) const override;
2251
2252 /// Gets the address of the native argument basing on the address of the
2253 /// target-specific parameter.
2254 /// \param NativeParam Parameter itself.
2255 /// \param TargetParam Corresponding target-specific parameter.
2256 Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam,
2257 const VarDecl *TargetParam) const override;
2258
2259 /// Gets the OpenMP-specific address of the local variable.
2260 Address getAddressOfLocalVariable(CodeGenFunction &CGF,
2261 const VarDecl *VD) override {
2262 return Address::invalid();
2263 }
2264 };
2265
2266 } // namespace CodeGen
2267 } // namespace clang
2268
2269 #endif