Debian Codegraph @

llvm-project/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp

raw
1 //===- llvm/unittest/IR/OpenMPIRBuilderTest.cpp - OpenMPIRBuilder tests ---===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/Frontend/OpenMP/OMPConstants.h"
10 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
11 #include "llvm/IR/BasicBlock.h"
12 #include "llvm/IR/DIBuilder.h"
13 #include "llvm/IR/Function.h"
14 #include "llvm/IR/InstIterator.h"
15 #include "llvm/IR/Instructions.h"
16 #include "llvm/IR/LLVMContext.h"
17 #include "llvm/IR/Module.h"
18 #include "llvm/IR/Verifier.h"
19 #include "llvm/Passes/PassBuilder.h"
20 #include "llvm/Support/Casting.h"
21 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
22 #include "gtest/gtest.h"
23 #include <optional>
24
25 using namespace llvm;
26 using namespace omp;
27
28 namespace {
29
30 /// Create an instruction that uses the values in \p Values. We use "printf"
31 /// just because it is often used for this purpose in test code, but it is never
32 /// executed here.
33 static CallInst *createPrintfCall(IRBuilder<> &Builder, StringRef FormatStr,
34 ArrayRef<Value *> Values) {
35 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
36
37 GlobalVariable *GV = Builder.CreateGlobalString(FormatStr, "", 0, M);
38 Constant *Zero = ConstantInt::get(Type::getInt32Ty(M->getContext()), 0);
39 Constant *Indices[] = {Zero, Zero};
40 Constant *FormatStrConst =
41 ConstantExpr::getInBoundsGetElementPtr(GV->getValueType(), GV, Indices);
42
43 Function *PrintfDecl = M->getFunction("printf");
44 if (!PrintfDecl) {
45 GlobalValue::LinkageTypes Linkage = Function::ExternalLinkage;
46 FunctionType *Ty = FunctionType::get(Builder.getInt32Ty(), true);
47 PrintfDecl = Function::Create(Ty, Linkage, "printf", M);
48 }
49
50 SmallVector<Value *, 4> Args;
51 Args.push_back(FormatStrConst);
52 Args.append(Values.begin(), Values.end());
53 return Builder.CreateCall(PrintfDecl, Args);
54 }
55
56 /// Verify that blocks in \p RefOrder are corresponds to the depth-first visit
57 /// order the control flow of \p F.
58 ///
59 /// This is an easy way to verify the branching structure of the CFG without
60 /// checking every branch instruction individually. For the CFG of a
61 /// CanonicalLoopInfo, the Cond BB's terminating branch's first edge is entering
62 /// the body, i.e. the DFS order corresponds to the execution order with one
63 /// loop iteration.
64 static testing::AssertionResult
65 verifyDFSOrder(Function *F, ArrayRef<BasicBlock *> RefOrder) {
66 ArrayRef<BasicBlock *>::iterator It = RefOrder.begin();
67 ArrayRef<BasicBlock *>::iterator E = RefOrder.end();
68
69 df_iterator_default_set<BasicBlock *, 16> Visited;
70 auto DFS = llvm::depth_first_ext(&F->getEntryBlock(), Visited);
71
72 BasicBlock *Prev = nullptr;
73 for (BasicBlock *BB : DFS) {
74 if (It != E && BB == *It) {
75 Prev = *It;
76 ++It;
77 }
78 }
79
80 if (It == E)
81 return testing::AssertionSuccess();
82 if (!Prev)
83 return testing::AssertionFailure()
84 << "Did not find " << (*It)->getName() << " in control flow";
85 return testing::AssertionFailure()
86 << "Expected " << Prev->getName() << " before " << (*It)->getName()
87 << " in control flow";
88 }
89
90 /// Verify that blocks in \p RefOrder are in the same relative order in the
91 /// linked lists of blocks in \p F. The linked list may contain additional
92 /// blocks in-between.
93 ///
94 /// While the order in the linked list is not relevant for semantics, keeping
95 /// the order roughly in execution order makes its printout easier to read.
96 static testing::AssertionResult
97 verifyListOrder(Function *F, ArrayRef<BasicBlock *> RefOrder) {
98 ArrayRef<BasicBlock *>::iterator It = RefOrder.begin();
99 ArrayRef<BasicBlock *>::iterator E = RefOrder.end();
100
101 BasicBlock *Prev = nullptr;
102 for (BasicBlock &BB : *F) {
103 if (It != E && &BB == *It) {
104 Prev = *It;
105 ++It;
106 }
107 }
108
109 if (It == E)
110 return testing::AssertionSuccess();
111 if (!Prev)
112 return testing::AssertionFailure() << "Did not find " << (*It)->getName()
113 << " in function " << F->getName();
114 return testing::AssertionFailure()
115 << "Expected " << Prev->getName() << " before " << (*It)->getName()
116 << " in function " << F->getName();
117 }
118
119 /// Populate Calls with call instructions calling the function with the given
120 /// FnID from the given function F.
121 static void findCalls(Function *F, omp::RuntimeFunction FnID,
122 OpenMPIRBuilder &OMPBuilder,
123 SmallVectorImpl<CallInst *> &Calls) {
124 Function *Fn = OMPBuilder.getOrCreateRuntimeFunctionPtr(FnID);
125 for (BasicBlock &BB : *F) {
126 for (Instruction &I : BB) {
127 auto *Call = dyn_cast<CallInst>(&I);
128 if (Call && Call->getCalledFunction() == Fn)
129 Calls.push_back(Call);
130 }
131 }
132 }
133
134 /// Assuming \p F contains only one call to the function with the given \p FnID,
135 /// return that call.
136 static CallInst *findSingleCall(Function *F, omp::RuntimeFunction FnID,
137 OpenMPIRBuilder &OMPBuilder) {
138 SmallVector<CallInst *, 1> Calls;
139 findCalls(F, FnID, OMPBuilder, Calls);
140 EXPECT_EQ(1u, Calls.size());
141 if (Calls.size() != 1)
142 return nullptr;
143 return Calls.front();
144 }
145
146 static omp::ScheduleKind getSchedKind(omp::OMPScheduleType SchedType) {
147 switch (SchedType & ~omp::OMPScheduleType::ModifierMask) {
148 case omp::OMPScheduleType::BaseDynamicChunked:
149 return omp::OMP_SCHEDULE_Dynamic;
150 case omp::OMPScheduleType::BaseGuidedChunked:
151 return omp::OMP_SCHEDULE_Guided;
152 case omp::OMPScheduleType::BaseAuto:
153 return omp::OMP_SCHEDULE_Auto;
154 case omp::OMPScheduleType::BaseRuntime:
155 return omp::OMP_SCHEDULE_Runtime;
156 default:
157 llvm_unreachable("unknown type for this test");
158 }
159 }
160
161 class OpenMPIRBuilderTest : public testing::Test {
162 protected:
163 void SetUp() override {
164 Ctx.setOpaquePointers(false); // TODO: Update tests for opaque pointers.
165 M.reset(new Module("MyModule", Ctx));
166 FunctionType *FTy =
167 FunctionType::get(Type::getVoidTy(Ctx), {Type::getInt32Ty(Ctx)},
168 /*isVarArg=*/false);
169 F = Function::Create(FTy, Function::ExternalLinkage, "", M.get());
170 BB = BasicBlock::Create(Ctx, "", F);
171
172 DIBuilder DIB(*M);
173 auto File = DIB.createFile("test.dbg", "/src", std::nullopt,
174 std::optional<StringRef>("/src/test.dbg"));
175 auto CU =
176 DIB.createCompileUnit(dwarf::DW_LANG_C, File, "llvm-C", true, "", 0);
177 auto Type =
178 DIB.createSubroutineType(DIB.getOrCreateTypeArray(std::nullopt));
179 auto SP = DIB.createFunction(
180 CU, "foo", "", File, 1, Type, 1, DINode::FlagZero,
181 DISubprogram::SPFlagDefinition | DISubprogram::SPFlagOptimized);
182 F->setSubprogram(SP);
183 auto Scope = DIB.createLexicalBlockFile(SP, File, 0);
184 DIB.finalize();
185 DL = DILocation::get(Ctx, 3, 7, Scope);
186 }
187
188 void TearDown() override {
189 BB = nullptr;
190 M.reset();
191 }
192
193 /// Create a function with a simple loop that calls printf using the logical
194 /// loop counter for use with tests that need a CanonicalLoopInfo object.
195 CanonicalLoopInfo *buildSingleLoopFunction(DebugLoc DL,
196 OpenMPIRBuilder &OMPBuilder,
197 int UseIVBits,
198 CallInst **Call = nullptr,
199 BasicBlock **BodyCode = nullptr) {
200 OMPBuilder.initialize();
201 F->setName("func");
202
203 IRBuilder<> Builder(BB);
204 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
205 Value *TripCount = F->getArg(0);
206
207 Type *IVType = Type::getIntNTy(Builder.getContext(), UseIVBits);
208 Value *CastedTripCount =
209 Builder.CreateZExtOrTrunc(TripCount, IVType, "tripcount");
210
211 auto LoopBodyGenCB = [&](OpenMPIRBuilder::InsertPointTy CodeGenIP,
212 llvm::Value *LC) {
213 Builder.restoreIP(CodeGenIP);
214 if (BodyCode)
215 *BodyCode = Builder.GetInsertBlock();
216
217 // Add something that consumes the induction variable to the body.
218 CallInst *CallInst = createPrintfCall(Builder, "%d\\n", {LC});
219 if (Call)
220 *Call = CallInst;
221 };
222 CanonicalLoopInfo *Loop =
223 OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, CastedTripCount);
224
225 // Finalize the function.
226 Builder.restoreIP(Loop->getAfterIP());
227 Builder.CreateRetVoid();
228
229 return Loop;
230 }
231
232 LLVMContext Ctx;
233 std::unique_ptr<Module> M;
234 Function *F;
235 BasicBlock *BB;
236 DebugLoc DL;
237 };
238
239 class OpenMPIRBuilderTestWithParams
240 : public OpenMPIRBuilderTest,
241 public ::testing::WithParamInterface<omp::OMPScheduleType> {};
242
243 class OpenMPIRBuilderTestWithIVBits
244 : public OpenMPIRBuilderTest,
245 public ::testing::WithParamInterface<int> {};
246
247 // Returns the value stored in the given allocation. Returns null if the given
248 // value is not a result of an InstTy instruction, if no value is stored or if
249 // there is more than one store.
250 template <typename InstTy> static Value *findStoredValue(Value *AllocaValue) {
251 Instruction *Inst = dyn_cast<InstTy>(AllocaValue);
252 if (!Inst)
253 return nullptr;
254 StoreInst *Store = nullptr;
255 for (Use &U : Inst->uses()) {
256 if (auto *CandidateStore = dyn_cast<StoreInst>(U.getUser())) {
257 EXPECT_EQ(Store, nullptr);
258 Store = CandidateStore;
259 }
260 }
261 if (!Store)
262 return nullptr;
263 return Store->getValueOperand();
264 }
265
266 // Returns the value stored in the aggregate argument of an outlined function,
267 // or nullptr if it is not found.
268 static Value *findStoredValueInAggregateAt(LLVMContext &Ctx, Value *Aggregate,
269 unsigned Idx) {
270 GetElementPtrInst *GEPAtIdx = nullptr;
271 // Find GEP instruction at that index.
272 for (User *Usr : Aggregate->users()) {
273 GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Usr);
274 if (!GEP)
275 continue;
276
277 if (GEP->getOperand(2) != ConstantInt::get(Type::getInt32Ty(Ctx), Idx))
278 continue;
279
280 EXPECT_EQ(GEPAtIdx, nullptr);
281 GEPAtIdx = GEP;
282 }
283
284 EXPECT_NE(GEPAtIdx, nullptr);
285 EXPECT_EQ(GEPAtIdx->getNumUses(), 1U);
286
287 // Find the value stored to the aggregate.
288 StoreInst *StoreToAgg = dyn_cast<StoreInst>(*GEPAtIdx->user_begin());
289 Value *StoredAggValue = StoreToAgg->getValueOperand();
290
291 Value *StoredValue = nullptr;
292
293 // Find the value stored to the value stored in the aggregate.
294 for (User *Usr : StoredAggValue->users()) {
295 StoreInst *Store = dyn_cast<StoreInst>(Usr);
296 if (!Store)
297 continue;
298
299 if (Store->getPointerOperand() != StoredAggValue)
300 continue;
301
302 EXPECT_EQ(StoredValue, nullptr);
303 StoredValue = Store->getValueOperand();
304 }
305
306 return StoredValue;
307 }
308
309 // Returns the aggregate that the value is originating from.
310 static Value *findAggregateFromValue(Value *V) {
311 // Expects a load instruction that loads from the aggregate.
312 LoadInst *Load = dyn_cast<LoadInst>(V);
313 EXPECT_NE(Load, nullptr);
314 // Find the GEP instruction used in the load instruction.
315 GetElementPtrInst *GEP =
316 dyn_cast<GetElementPtrInst>(Load->getPointerOperand());
317 EXPECT_NE(GEP, nullptr);
318 // Find the aggregate used in the GEP instruction.
319 Value *Aggregate = GEP->getPointerOperand();
320
321 return Aggregate;
322 }
323
324 TEST_F(OpenMPIRBuilderTest, CreateBarrier) {
325 OpenMPIRBuilder OMPBuilder(*M);
326 OMPBuilder.initialize();
327
328 IRBuilder<> Builder(BB);
329
330 OMPBuilder.createBarrier({IRBuilder<>::InsertPoint()}, OMPD_for);
331 EXPECT_TRUE(M->global_empty());
332 EXPECT_EQ(M->size(), 1U);
333 EXPECT_EQ(F->size(), 1U);
334 EXPECT_EQ(BB->size(), 0U);
335
336 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});
337 OMPBuilder.createBarrier(Loc, OMPD_for);
338 EXPECT_FALSE(M->global_empty());
339 EXPECT_EQ(M->size(), 3U);
340 EXPECT_EQ(F->size(), 1U);
341 EXPECT_EQ(BB->size(), 2U);
342
343 CallInst *GTID = dyn_cast<CallInst>(&BB->front());
344 EXPECT_NE(GTID, nullptr);
345 EXPECT_EQ(GTID->arg_size(), 1U);
346 EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
347 EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
348 EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
349
350 CallInst *Barrier = dyn_cast<CallInst>(GTID->getNextNode());
351 EXPECT_NE(Barrier, nullptr);
352 EXPECT_EQ(Barrier->arg_size(), 2U);
353 EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_barrier");
354 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory());
355 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory());
356
357 EXPECT_EQ(cast<CallInst>(Barrier)->getArgOperand(1), GTID);
358
359 Builder.CreateUnreachable();
360 EXPECT_FALSE(verifyModule(*M, &errs()));
361 }
362
363 TEST_F(OpenMPIRBuilderTest, CreateCancel) {
364 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
365 OpenMPIRBuilder OMPBuilder(*M);
366 OMPBuilder.initialize();
367
368 BasicBlock *CBB = BasicBlock::Create(Ctx, "", F);
369 new UnreachableInst(Ctx, CBB);
370 auto FiniCB = [&](InsertPointTy IP) {
371 ASSERT_NE(IP.getBlock(), nullptr);
372 ASSERT_EQ(IP.getBlock()->end(), IP.getPoint());
373 BranchInst::Create(CBB, IP.getBlock());
374 };
375 OMPBuilder.pushFinalizationCB({FiniCB, OMPD_parallel, true});
376
377 IRBuilder<> Builder(BB);
378
379 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});
380 auto NewIP = OMPBuilder.createCancel(Loc, nullptr, OMPD_parallel);
381 Builder.restoreIP(NewIP);
382 EXPECT_FALSE(M->global_empty());
383 EXPECT_EQ(M->size(), 4U);
384 EXPECT_EQ(F->size(), 4U);
385 EXPECT_EQ(BB->size(), 4U);
386
387 CallInst *GTID = dyn_cast<CallInst>(&BB->front());
388 EXPECT_NE(GTID, nullptr);
389 EXPECT_EQ(GTID->arg_size(), 1U);
390 EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
391 EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
392 EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
393
394 CallInst *Cancel = dyn_cast<CallInst>(GTID->getNextNode());
395 EXPECT_NE(Cancel, nullptr);
396 EXPECT_EQ(Cancel->arg_size(), 3U);
397 EXPECT_EQ(Cancel->getCalledFunction()->getName(), "__kmpc_cancel");
398 EXPECT_FALSE(Cancel->getCalledFunction()->doesNotAccessMemory());
399 EXPECT_FALSE(Cancel->getCalledFunction()->doesNotFreeMemory());
400 EXPECT_EQ(Cancel->getNumUses(), 1U);
401 Instruction *CancelBBTI = Cancel->getParent()->getTerminator();
402 EXPECT_EQ(CancelBBTI->getNumSuccessors(), 2U);
403 EXPECT_EQ(CancelBBTI->getSuccessor(0), NewIP.getBlock());
404 EXPECT_EQ(CancelBBTI->getSuccessor(1)->size(), 3U);
405 CallInst *GTID1 = dyn_cast<CallInst>(&CancelBBTI->getSuccessor(1)->front());
406 EXPECT_NE(GTID1, nullptr);
407 EXPECT_EQ(GTID1->arg_size(), 1U);
408 EXPECT_EQ(GTID1->getCalledFunction()->getName(), "__kmpc_global_thread_num");
409 EXPECT_FALSE(GTID1->getCalledFunction()->doesNotAccessMemory());
410 EXPECT_FALSE(GTID1->getCalledFunction()->doesNotFreeMemory());
411 CallInst *Barrier = dyn_cast<CallInst>(GTID1->getNextNode());
412 EXPECT_NE(Barrier, nullptr);
413 EXPECT_EQ(Barrier->arg_size(), 2U);
414 EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_cancel_barrier");
415 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory());
416 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory());
417 EXPECT_EQ(Barrier->getNumUses(), 0U);
418 EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(),
419 1U);
420 EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getSuccessor(0), CBB);
421
422 EXPECT_EQ(cast<CallInst>(Cancel)->getArgOperand(1), GTID);
423
424 OMPBuilder.popFinalizationCB();
425
426 Builder.CreateUnreachable();
427 EXPECT_FALSE(verifyModule(*M, &errs()));
428 }
429
430 TEST_F(OpenMPIRBuilderTest, CreateCancelIfCond) {
431 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
432 OpenMPIRBuilder OMPBuilder(*M);
433 OMPBuilder.initialize();
434
435 BasicBlock *CBB = BasicBlock::Create(Ctx, "", F);
436 new UnreachableInst(Ctx, CBB);
437 auto FiniCB = [&](InsertPointTy IP) {
438 ASSERT_NE(IP.getBlock(), nullptr);
439 ASSERT_EQ(IP.getBlock()->end(), IP.getPoint());
440 BranchInst::Create(CBB, IP.getBlock());
441 };
442 OMPBuilder.pushFinalizationCB({FiniCB, OMPD_parallel, true});
443
444 IRBuilder<> Builder(BB);
445
446 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});
447 auto NewIP = OMPBuilder.createCancel(Loc, Builder.getTrue(), OMPD_parallel);
448 Builder.restoreIP(NewIP);
449 EXPECT_FALSE(M->global_empty());
450 EXPECT_EQ(M->size(), 4U);
451 EXPECT_EQ(F->size(), 7U);
452 EXPECT_EQ(BB->size(), 1U);
453 ASSERT_TRUE(isa<BranchInst>(BB->getTerminator()));
454 ASSERT_EQ(BB->getTerminator()->getNumSuccessors(), 2U);
455 BB = BB->getTerminator()->getSuccessor(0);
456 EXPECT_EQ(BB->size(), 4U);
457
458 CallInst *GTID = dyn_cast<CallInst>(&BB->front());
459 EXPECT_NE(GTID, nullptr);
460 EXPECT_EQ(GTID->arg_size(), 1U);
461 EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
462 EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
463 EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
464
465 CallInst *Cancel = dyn_cast<CallInst>(GTID->getNextNode());
466 EXPECT_NE(Cancel, nullptr);
467 EXPECT_EQ(Cancel->arg_size(), 3U);
468 EXPECT_EQ(Cancel->getCalledFunction()->getName(), "__kmpc_cancel");
469 EXPECT_FALSE(Cancel->getCalledFunction()->doesNotAccessMemory());
470 EXPECT_FALSE(Cancel->getCalledFunction()->doesNotFreeMemory());
471 EXPECT_EQ(Cancel->getNumUses(), 1U);
472 Instruction *CancelBBTI = Cancel->getParent()->getTerminator();
473 EXPECT_EQ(CancelBBTI->getNumSuccessors(), 2U);
474 EXPECT_EQ(CancelBBTI->getSuccessor(0)->size(), 1U);
475 EXPECT_EQ(CancelBBTI->getSuccessor(0)->getUniqueSuccessor(),
476 NewIP.getBlock());
477 EXPECT_EQ(CancelBBTI->getSuccessor(1)->size(), 3U);
478 CallInst *GTID1 = dyn_cast<CallInst>(&CancelBBTI->getSuccessor(1)->front());
479 EXPECT_NE(GTID1, nullptr);
480 EXPECT_EQ(GTID1->arg_size(), 1U);
481 EXPECT_EQ(GTID1->getCalledFunction()->getName(), "__kmpc_global_thread_num");
482 EXPECT_FALSE(GTID1->getCalledFunction()->doesNotAccessMemory());
483 EXPECT_FALSE(GTID1->getCalledFunction()->doesNotFreeMemory());
484 CallInst *Barrier = dyn_cast<CallInst>(GTID1->getNextNode());
485 EXPECT_NE(Barrier, nullptr);
486 EXPECT_EQ(Barrier->arg_size(), 2U);
487 EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_cancel_barrier");
488 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory());
489 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory());
490 EXPECT_EQ(Barrier->getNumUses(), 0U);
491 EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(),
492 1U);
493 EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getSuccessor(0), CBB);
494
495 EXPECT_EQ(cast<CallInst>(Cancel)->getArgOperand(1), GTID);
496
497 OMPBuilder.popFinalizationCB();
498
499 Builder.CreateUnreachable();
500 EXPECT_FALSE(verifyModule(*M, &errs()));
501 }
502
503 TEST_F(OpenMPIRBuilderTest, CreateCancelBarrier) {
504 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
505 OpenMPIRBuilder OMPBuilder(*M);
506 OMPBuilder.initialize();
507
508 BasicBlock *CBB = BasicBlock::Create(Ctx, "", F);
509 new UnreachableInst(Ctx, CBB);
510 auto FiniCB = [&](InsertPointTy IP) {
511 ASSERT_NE(IP.getBlock(), nullptr);
512 ASSERT_EQ(IP.getBlock()->end(), IP.getPoint());
513 BranchInst::Create(CBB, IP.getBlock());
514 };
515 OMPBuilder.pushFinalizationCB({FiniCB, OMPD_parallel, true});
516
517 IRBuilder<> Builder(BB);
518
519 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});
520 auto NewIP = OMPBuilder.createBarrier(Loc, OMPD_for);
521 Builder.restoreIP(NewIP);
522 EXPECT_FALSE(M->global_empty());
523 EXPECT_EQ(M->size(), 3U);
524 EXPECT_EQ(F->size(), 4U);
525 EXPECT_EQ(BB->size(), 4U);
526
527 CallInst *GTID = dyn_cast<CallInst>(&BB->front());
528 EXPECT_NE(GTID, nullptr);
529 EXPECT_EQ(GTID->arg_size(), 1U);
530 EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
531 EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
532 EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
533
534 CallInst *Barrier = dyn_cast<CallInst>(GTID->getNextNode());
535 EXPECT_NE(Barrier, nullptr);
536 EXPECT_EQ(Barrier->arg_size(), 2U);
537 EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_cancel_barrier");
538 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory());
539 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory());
540 EXPECT_EQ(Barrier->getNumUses(), 1U);
541 Instruction *BarrierBBTI = Barrier->getParent()->getTerminator();
542 EXPECT_EQ(BarrierBBTI->getNumSuccessors(), 2U);
543 EXPECT_EQ(BarrierBBTI->getSuccessor(0), NewIP.getBlock());
544 EXPECT_EQ(BarrierBBTI->getSuccessor(1)->size(), 1U);
545 EXPECT_EQ(BarrierBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(),
546 1U);
547 EXPECT_EQ(BarrierBBTI->getSuccessor(1)->getTerminator()->getSuccessor(0),
548 CBB);
549
550 EXPECT_EQ(cast<CallInst>(Barrier)->getArgOperand(1), GTID);
551
552 OMPBuilder.popFinalizationCB();
553
554 Builder.CreateUnreachable();
555 EXPECT_FALSE(verifyModule(*M, &errs()));
556 }
557
558 TEST_F(OpenMPIRBuilderTest, DbgLoc) {
559 OpenMPIRBuilder OMPBuilder(*M);
560 OMPBuilder.initialize();
561 F->setName("func");
562
563 IRBuilder<> Builder(BB);
564
565 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
566 OMPBuilder.createBarrier(Loc, OMPD_for);
567 CallInst *GTID = dyn_cast<CallInst>(&BB->front());
568 CallInst *Barrier = dyn_cast<CallInst>(GTID->getNextNode());
569 EXPECT_EQ(GTID->getDebugLoc(), DL);
570 EXPECT_EQ(Barrier->getDebugLoc(), DL);
571 EXPECT_TRUE(isa<GlobalVariable>(Barrier->getOperand(0)));
572 if (!isa<GlobalVariable>(Barrier->getOperand(0)))
573 return;
574 GlobalVariable *Ident = cast<GlobalVariable>(Barrier->getOperand(0));
575 EXPECT_TRUE(Ident->hasInitializer());
576 if (!Ident->hasInitializer())
577 return;
578 Constant *Initializer = Ident->getInitializer();
579 EXPECT_TRUE(
580 isa<GlobalVariable>(Initializer->getOperand(4)->stripPointerCasts()));
581 GlobalVariable *SrcStrGlob =
582 cast<GlobalVariable>(Initializer->getOperand(4)->stripPointerCasts());
583 if (!SrcStrGlob)
584 return;
585 EXPECT_TRUE(isa<ConstantDataArray>(SrcStrGlob->getInitializer()));
586 ConstantDataArray *SrcSrc =
587 dyn_cast<ConstantDataArray>(SrcStrGlob->getInitializer());
588 if (!SrcSrc)
589 return;
590 EXPECT_EQ(SrcSrc->getAsCString(), ";/src/test.dbg;foo;3;7;;");
591 }
592
593 TEST_F(OpenMPIRBuilderTest, ParallelSimple) {
594 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
595 OpenMPIRBuilder OMPBuilder(*M);
596 OMPBuilder.initialize();
597 F->setName("func");
598 IRBuilder<> Builder(BB);
599
600 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
601 Builder.CreateBr(EnterBB);
602 Builder.SetInsertPoint(EnterBB);
603 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
604
605 AllocaInst *PrivAI = nullptr;
606
607 unsigned NumBodiesGenerated = 0;
608 unsigned NumPrivatizedVars = 0;
609 unsigned NumFinalizationPoints = 0;
610
611 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
612 ++NumBodiesGenerated;
613
614 Builder.restoreIP(AllocaIP);
615 PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
616 Builder.CreateStore(F->arg_begin(), PrivAI);
617
618 Builder.restoreIP(CodeGenIP);
619 Value *PrivLoad =
620 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
621 Value *Cmp = Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
622 Instruction *ThenTerm, *ElseTerm;
623 SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(),
624 &ThenTerm, &ElseTerm);
625 };
626
627 auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
628 Value &Orig, Value &Inner,
629 Value *&ReplacementValue) -> InsertPointTy {
630 ++NumPrivatizedVars;
631
632 if (!isa<AllocaInst>(Orig)) {
633 EXPECT_EQ(&Orig, F->arg_begin());
634 ReplacementValue = &Inner;
635 return CodeGenIP;
636 }
637
638 // Since the original value is an allocation, it has a pointer type and
639 // therefore no additional wrapping should happen.
640 EXPECT_EQ(&Orig, &Inner);
641
642 // Trivial copy (=firstprivate).
643 Builder.restoreIP(AllocaIP);
644 Type *VTy = ReplacementValue->getType();
645 Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload");
646 ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy");
647 Builder.restoreIP(CodeGenIP);
648 Builder.CreateStore(V, ReplacementValue);
649 return CodeGenIP;
650 };
651
652 auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; };
653
654 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
655 F->getEntryBlock().getFirstInsertionPt());
656 IRBuilder<>::InsertPoint AfterIP =
657 OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
658 nullptr, nullptr, OMP_PROC_BIND_default, false);
659 EXPECT_EQ(NumBodiesGenerated, 1U);
660 EXPECT_EQ(NumPrivatizedVars, 1U);
661 EXPECT_EQ(NumFinalizationPoints, 1U);
662
663 Builder.restoreIP(AfterIP);
664 Builder.CreateRetVoid();
665
666 OMPBuilder.finalize();
667
668 EXPECT_NE(PrivAI, nullptr);
669 Function *OutlinedFn = PrivAI->getFunction();
670 EXPECT_NE(F, OutlinedFn);
671 EXPECT_FALSE(verifyModule(*M, &errs()));
672 EXPECT_TRUE(OutlinedFn->hasFnAttribute(Attribute::NoUnwind));
673 EXPECT_TRUE(OutlinedFn->hasFnAttribute(Attribute::NoRecurse));
674 EXPECT_TRUE(OutlinedFn->hasParamAttribute(0, Attribute::NoAlias));
675 EXPECT_TRUE(OutlinedFn->hasParamAttribute(1, Attribute::NoAlias));
676
677 EXPECT_TRUE(OutlinedFn->hasInternalLinkage());
678 EXPECT_EQ(OutlinedFn->arg_size(), 3U);
679
680 EXPECT_EQ(&OutlinedFn->getEntryBlock(), PrivAI->getParent());
681 EXPECT_EQ(OutlinedFn->getNumUses(), 1U);
682 User *Usr = OutlinedFn->user_back();
683 ASSERT_TRUE(isa<ConstantExpr>(Usr));
684 CallInst *ForkCI = dyn_cast<CallInst>(Usr->user_back());
685 ASSERT_NE(ForkCI, nullptr);
686
687 EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call");
688 EXPECT_EQ(ForkCI->arg_size(), 4U);
689 EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0)));
690 EXPECT_EQ(ForkCI->getArgOperand(1),
691 ConstantInt::get(Type::getInt32Ty(Ctx), 1U));
692 EXPECT_EQ(ForkCI->getArgOperand(2), Usr);
693 Value *StoredValue =
694 findStoredValueInAggregateAt(Ctx, ForkCI->getArgOperand(3), 0);
695 EXPECT_EQ(StoredValue, F->arg_begin());
696 }
697
698 TEST_F(OpenMPIRBuilderTest, ParallelNested) {
699 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
700 OpenMPIRBuilder OMPBuilder(*M);
701 OMPBuilder.initialize();
702 F->setName("func");
703 IRBuilder<> Builder(BB);
704
705 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
706 Builder.CreateBr(EnterBB);
707 Builder.SetInsertPoint(EnterBB);
708 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
709
710 unsigned NumInnerBodiesGenerated = 0;
711 unsigned NumOuterBodiesGenerated = 0;
712 unsigned NumFinalizationPoints = 0;
713
714 auto InnerBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
715 ++NumInnerBodiesGenerated;
716 };
717
718 auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
719 Value &Orig, Value &Inner,
720 Value *&ReplacementValue) -> InsertPointTy {
721 // Trivial copy (=firstprivate).
722 Builder.restoreIP(AllocaIP);
723 Type *VTy = ReplacementValue->getType();
724 Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload");
725 ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy");
726 Builder.restoreIP(CodeGenIP);
727 Builder.CreateStore(V, ReplacementValue);
728 return CodeGenIP;
729 };
730
731 auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; };
732
733 auto OuterBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
734 ++NumOuterBodiesGenerated;
735 Builder.restoreIP(CodeGenIP);
736 BasicBlock *CGBB = CodeGenIP.getBlock();
737 BasicBlock *NewBB = SplitBlock(CGBB, &*CodeGenIP.getPoint());
738 CGBB->getTerminator()->eraseFromParent();
739 ;
740
741 IRBuilder<>::InsertPoint AfterIP = OMPBuilder.createParallel(
742 InsertPointTy(CGBB, CGBB->end()), AllocaIP, InnerBodyGenCB, PrivCB,
743 FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false);
744
745 Builder.restoreIP(AfterIP);
746 Builder.CreateBr(NewBB);
747 };
748
749 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
750 F->getEntryBlock().getFirstInsertionPt());
751 IRBuilder<>::InsertPoint AfterIP =
752 OMPBuilder.createParallel(Loc, AllocaIP, OuterBodyGenCB, PrivCB, FiniCB,
753 nullptr, nullptr, OMP_PROC_BIND_default, false);
754
755 EXPECT_EQ(NumInnerBodiesGenerated, 1U);
756 EXPECT_EQ(NumOuterBodiesGenerated, 1U);
757 EXPECT_EQ(NumFinalizationPoints, 2U);
758
759 Builder.restoreIP(AfterIP);
760 Builder.CreateRetVoid();
761
762 OMPBuilder.finalize();
763
764 EXPECT_EQ(M->size(), 5U);
765 for (Function &OutlinedFn : *M) {
766 if (F == &OutlinedFn || OutlinedFn.isDeclaration())
767 continue;
768 EXPECT_FALSE(verifyModule(*M, &errs()));
769 EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoUnwind));
770 EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoRecurse));
771 EXPECT_TRUE(OutlinedFn.hasParamAttribute(0, Attribute::NoAlias));
772 EXPECT_TRUE(OutlinedFn.hasParamAttribute(1, Attribute::NoAlias));
773
774 EXPECT_TRUE(OutlinedFn.hasInternalLinkage());
775 EXPECT_EQ(OutlinedFn.arg_size(), 2U);
776
777 EXPECT_EQ(OutlinedFn.getNumUses(), 1U);
778 User *Usr = OutlinedFn.user_back();
779 ASSERT_TRUE(isa<ConstantExpr>(Usr));
780 CallInst *ForkCI = dyn_cast<CallInst>(Usr->user_back());
781 ASSERT_NE(ForkCI, nullptr);
782
783 EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call");
784 EXPECT_EQ(ForkCI->arg_size(), 3U);
785 EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0)));
786 EXPECT_EQ(ForkCI->getArgOperand(1),
787 ConstantInt::get(Type::getInt32Ty(Ctx), 0U));
788 EXPECT_EQ(ForkCI->getArgOperand(2), Usr);
789 }
790 }
791
792 TEST_F(OpenMPIRBuilderTest, ParallelNested2Inner) {
793 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
794 OpenMPIRBuilder OMPBuilder(*M);
795 OMPBuilder.initialize();
796 F->setName("func");
797 IRBuilder<> Builder(BB);
798
799 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
800 Builder.CreateBr(EnterBB);
801 Builder.SetInsertPoint(EnterBB);
802 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
803
804 unsigned NumInnerBodiesGenerated = 0;
805 unsigned NumOuterBodiesGenerated = 0;
806 unsigned NumFinalizationPoints = 0;
807
808 auto InnerBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
809 ++NumInnerBodiesGenerated;
810 };
811
812 auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
813 Value &Orig, Value &Inner,
814 Value *&ReplacementValue) -> InsertPointTy {
815 // Trivial copy (=firstprivate).
816 Builder.restoreIP(AllocaIP);
817 Type *VTy = ReplacementValue->getType();
818 Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload");
819 ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy");
820 Builder.restoreIP(CodeGenIP);
821 Builder.CreateStore(V, ReplacementValue);
822 return CodeGenIP;
823 };
824
825 auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; };
826
827 auto OuterBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
828 ++NumOuterBodiesGenerated;
829 Builder.restoreIP(CodeGenIP);
830 BasicBlock *CGBB = CodeGenIP.getBlock();
831 BasicBlock *NewBB1 = SplitBlock(CGBB, &*CodeGenIP.getPoint());
832 BasicBlock *NewBB2 = SplitBlock(NewBB1, &*NewBB1->getFirstInsertionPt());
833 CGBB->getTerminator()->eraseFromParent();
834 ;
835 NewBB1->getTerminator()->eraseFromParent();
836 ;
837
838 IRBuilder<>::InsertPoint AfterIP1 = OMPBuilder.createParallel(
839 InsertPointTy(CGBB, CGBB->end()), AllocaIP, InnerBodyGenCB, PrivCB,
840 FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false);
841
842 Builder.restoreIP(AfterIP1);
843 Builder.CreateBr(NewBB1);
844
845 IRBuilder<>::InsertPoint AfterIP2 = OMPBuilder.createParallel(
846 InsertPointTy(NewBB1, NewBB1->end()), AllocaIP, InnerBodyGenCB, PrivCB,
847 FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false);
848
849 Builder.restoreIP(AfterIP2);
850 Builder.CreateBr(NewBB2);
851 };
852
853 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
854 F->getEntryBlock().getFirstInsertionPt());
855 IRBuilder<>::InsertPoint AfterIP =
856 OMPBuilder.createParallel(Loc, AllocaIP, OuterBodyGenCB, PrivCB, FiniCB,
857 nullptr, nullptr, OMP_PROC_BIND_default, false);
858
859 EXPECT_EQ(NumInnerBodiesGenerated, 2U);
860 EXPECT_EQ(NumOuterBodiesGenerated, 1U);
861 EXPECT_EQ(NumFinalizationPoints, 3U);
862
863 Builder.restoreIP(AfterIP);
864 Builder.CreateRetVoid();
865
866 OMPBuilder.finalize();
867
868 EXPECT_EQ(M->size(), 6U);
869 for (Function &OutlinedFn : *M) {
870 if (F == &OutlinedFn || OutlinedFn.isDeclaration())
871 continue;
872 EXPECT_FALSE(verifyModule(*M, &errs()));
873 EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoUnwind));
874 EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoRecurse));
875 EXPECT_TRUE(OutlinedFn.hasParamAttribute(0, Attribute::NoAlias));
876 EXPECT_TRUE(OutlinedFn.hasParamAttribute(1, Attribute::NoAlias));
877
878 EXPECT_TRUE(OutlinedFn.hasInternalLinkage());
879 EXPECT_EQ(OutlinedFn.arg_size(), 2U);
880
881 unsigned NumAllocas = 0;
882 for (Instruction &I : instructions(OutlinedFn))
883 NumAllocas += isa<AllocaInst>(I);
884 EXPECT_EQ(NumAllocas, 1U);
885
886 EXPECT_EQ(OutlinedFn.getNumUses(), 1U);
887 User *Usr = OutlinedFn.user_back();
888 ASSERT_TRUE(isa<ConstantExpr>(Usr));
889 CallInst *ForkCI = dyn_cast<CallInst>(Usr->user_back());
890 ASSERT_NE(ForkCI, nullptr);
891
892 EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call");
893 EXPECT_EQ(ForkCI->arg_size(), 3U);
894 EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0)));
895 EXPECT_EQ(ForkCI->getArgOperand(1),
896 ConstantInt::get(Type::getInt32Ty(Ctx), 0U));
897 EXPECT_EQ(ForkCI->getArgOperand(2), Usr);
898 }
899 }
900
901 TEST_F(OpenMPIRBuilderTest, ParallelIfCond) {
902 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
903 OpenMPIRBuilder OMPBuilder(*M);
904 OMPBuilder.initialize();
905 F->setName("func");
906 IRBuilder<> Builder(BB);
907
908 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
909 Builder.CreateBr(EnterBB);
910 Builder.SetInsertPoint(EnterBB);
911 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
912
913 AllocaInst *PrivAI = nullptr;
914
915 unsigned NumBodiesGenerated = 0;
916 unsigned NumPrivatizedVars = 0;
917 unsigned NumFinalizationPoints = 0;
918
919 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
920 ++NumBodiesGenerated;
921
922 Builder.restoreIP(AllocaIP);
923 PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
924 Builder.CreateStore(F->arg_begin(), PrivAI);
925
926 Builder.restoreIP(CodeGenIP);
927 Value *PrivLoad =
928 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
929 Value *Cmp = Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
930 Instruction *ThenTerm, *ElseTerm;
931 SplitBlockAndInsertIfThenElse(Cmp, &*Builder.GetInsertPoint(), &ThenTerm,
932 &ElseTerm);
933 };
934
935 auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
936 Value &Orig, Value &Inner,
937 Value *&ReplacementValue) -> InsertPointTy {
938 ++NumPrivatizedVars;
939
940 if (!isa<AllocaInst>(Orig)) {
941 EXPECT_EQ(&Orig, F->arg_begin());
942 ReplacementValue = &Inner;
943 return CodeGenIP;
944 }
945
946 // Since the original value is an allocation, it has a pointer type and
947 // therefore no additional wrapping should happen.
948 EXPECT_EQ(&Orig, &Inner);
949
950 // Trivial copy (=firstprivate).
951 Builder.restoreIP(AllocaIP);
952 Type *VTy = ReplacementValue->getType();
953 Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload");
954 ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy");
955 Builder.restoreIP(CodeGenIP);
956 Builder.CreateStore(V, ReplacementValue);
957 return CodeGenIP;
958 };
959
960 auto FiniCB = [&](InsertPointTy CodeGenIP) {
961 ++NumFinalizationPoints;
962 // No destructors.
963 };
964
965 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
966 F->getEntryBlock().getFirstInsertionPt());
967 IRBuilder<>::InsertPoint AfterIP =
968 OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
969 Builder.CreateIsNotNull(F->arg_begin()),
970 nullptr, OMP_PROC_BIND_default, false);
971
972 EXPECT_EQ(NumBodiesGenerated, 1U);
973 EXPECT_EQ(NumPrivatizedVars, 1U);
974 EXPECT_EQ(NumFinalizationPoints, 1U);
975
976 Builder.restoreIP(AfterIP);
977 Builder.CreateRetVoid();
978 OMPBuilder.finalize();
979
980 EXPECT_NE(PrivAI, nullptr);
981 Function *OutlinedFn = PrivAI->getFunction();
982 EXPECT_NE(F, OutlinedFn);
983 EXPECT_FALSE(verifyModule(*M, &errs()));
984
985 EXPECT_TRUE(OutlinedFn->hasInternalLinkage());
986 EXPECT_EQ(OutlinedFn->arg_size(), 3U);
987
988 EXPECT_EQ(&OutlinedFn->getEntryBlock(), PrivAI->getParent());
989 ASSERT_EQ(OutlinedFn->getNumUses(), 1U);
990
991 CallInst *ForkCI = nullptr;
992 for (User *Usr : OutlinedFn->users()) {
993 ASSERT_TRUE(isa<ConstantExpr>(Usr));
994 ASSERT_EQ(Usr->getNumUses(), 1U);
995 ASSERT_TRUE(isa<CallInst>(Usr->user_back()));
996 ForkCI = cast<CallInst>(Usr->user_back());
997 }
998
999 EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call_if");
1000 EXPECT_EQ(ForkCI->arg_size(), 5U);
1001 EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0)));
1002 EXPECT_EQ(ForkCI->getArgOperand(1),
1003 ConstantInt::get(Type::getInt32Ty(Ctx), 1));
1004 EXPECT_EQ(ForkCI->getArgOperand(3)->getType(), Type::getInt32Ty(Ctx));
1005 }
1006
1007 TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) {
1008 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1009 OpenMPIRBuilder OMPBuilder(*M);
1010 OMPBuilder.initialize();
1011 F->setName("func");
1012 IRBuilder<> Builder(BB);
1013
1014 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
1015 Builder.CreateBr(EnterBB);
1016 Builder.SetInsertPoint(EnterBB);
1017 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1018
1019 unsigned NumBodiesGenerated = 0;
1020 unsigned NumPrivatizedVars = 0;
1021 unsigned NumFinalizationPoints = 0;
1022
1023 CallInst *CheckedBarrier = nullptr;
1024 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
1025 ++NumBodiesGenerated;
1026
1027 Builder.restoreIP(CodeGenIP);
1028
1029 // Create three barriers, two cancel barriers but only one checked.
1030 Function *CBFn, *BFn;
1031
1032 Builder.restoreIP(
1033 OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel));
1034
1035 CBFn = M->getFunction("__kmpc_cancel_barrier");
1036 BFn = M->getFunction("__kmpc_barrier");
1037 ASSERT_NE(CBFn, nullptr);
1038 ASSERT_EQ(BFn, nullptr);
1039 ASSERT_EQ(CBFn->getNumUses(), 1U);
1040 ASSERT_TRUE(isa<CallInst>(CBFn->user_back()));
1041 ASSERT_EQ(CBFn->user_back()->getNumUses(), 1U);
1042 CheckedBarrier = cast<CallInst>(CBFn->user_back());
1043
1044 Builder.restoreIP(
1045 OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel, true));
1046 CBFn = M->getFunction("__kmpc_cancel_barrier");
1047 BFn = M->getFunction("__kmpc_barrier");
1048 ASSERT_NE(CBFn, nullptr);
1049 ASSERT_NE(BFn, nullptr);
1050 ASSERT_EQ(CBFn->getNumUses(), 1U);
1051 ASSERT_EQ(BFn->getNumUses(), 1U);
1052 ASSERT_TRUE(isa<CallInst>(BFn->user_back()));
1053 ASSERT_EQ(BFn->user_back()->getNumUses(), 0U);
1054
1055 Builder.restoreIP(OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel,
1056 false, false));
1057 ASSERT_EQ(CBFn->getNumUses(), 2U);
1058 ASSERT_EQ(BFn->getNumUses(), 1U);
1059 ASSERT_TRUE(CBFn->user_back() != CheckedBarrier);
1060 ASSERT_TRUE(isa<CallInst>(CBFn->user_back()));
1061 ASSERT_EQ(CBFn->user_back()->getNumUses(), 0U);
1062 };
1063
1064 auto PrivCB = [&](InsertPointTy, InsertPointTy, Value &V, Value &,
1065 Value *&) -> InsertPointTy {
1066 ++NumPrivatizedVars;
1067 llvm_unreachable("No privatization callback call expected!");
1068 };
1069
1070 FunctionType *FakeDestructorTy =
1071 FunctionType::get(Type::getVoidTy(Ctx), {Type::getInt32Ty(Ctx)},
1072 /*isVarArg=*/false);
1073 auto *FakeDestructor = Function::Create(
1074 FakeDestructorTy, Function::ExternalLinkage, "fakeDestructor", M.get());
1075
1076 auto FiniCB = [&](InsertPointTy IP) {
1077 ++NumFinalizationPoints;
1078 Builder.restoreIP(IP);
1079 Builder.CreateCall(FakeDestructor,
1080 {Builder.getInt32(NumFinalizationPoints)});
1081 };
1082
1083 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
1084 F->getEntryBlock().getFirstInsertionPt());
1085 IRBuilder<>::InsertPoint AfterIP =
1086 OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
1087 Builder.CreateIsNotNull(F->arg_begin()),
1088 nullptr, OMP_PROC_BIND_default, true);
1089
1090 EXPECT_EQ(NumBodiesGenerated, 1U);
1091 EXPECT_EQ(NumPrivatizedVars, 0U);
1092 EXPECT_EQ(NumFinalizationPoints, 2U);
1093 EXPECT_EQ(FakeDestructor->getNumUses(), 2U);
1094
1095 Builder.restoreIP(AfterIP);
1096 Builder.CreateRetVoid();
1097 OMPBuilder.finalize();
1098
1099 EXPECT_FALSE(verifyModule(*M, &errs()));
1100
1101 BasicBlock *ExitBB = nullptr;
1102 for (const User *Usr : FakeDestructor->users()) {
1103 const CallInst *CI = dyn_cast<CallInst>(Usr);
1104 ASSERT_EQ(CI->getCalledFunction(), FakeDestructor);
1105 ASSERT_TRUE(isa<BranchInst>(CI->getNextNode()));
1106 ASSERT_EQ(CI->getNextNode()->getNumSuccessors(), 1U);
1107 if (ExitBB)
1108 ASSERT_EQ(CI->getNextNode()->getSuccessor(0), ExitBB);
1109 else
1110 ExitBB = CI->getNextNode()->getSuccessor(0);
1111 ASSERT_EQ(ExitBB->size(), 1U);
1112 if (!isa<ReturnInst>(ExitBB->front())) {
1113 ASSERT_TRUE(isa<BranchInst>(ExitBB->front()));
1114 ASSERT_EQ(cast<BranchInst>(ExitBB->front()).getNumSuccessors(), 1U);
1115 ASSERT_TRUE(isa<ReturnInst>(
1116 cast<BranchInst>(ExitBB->front()).getSuccessor(0)->front()));
1117 }
1118 }
1119 }
1120
1121 TEST_F(OpenMPIRBuilderTest, ParallelForwardAsPointers) {
1122 OpenMPIRBuilder OMPBuilder(*M);
1123 OMPBuilder.initialize();
1124 F->setName("func");
1125 IRBuilder<> Builder(BB);
1126 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1127 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1128
1129 Type *I32Ty = Type::getInt32Ty(M->getContext());
1130 Type *I32PtrTy = Type::getInt32PtrTy(M->getContext());
1131 Type *StructTy = StructType::get(I32Ty, I32PtrTy);
1132 Type *StructPtrTy = StructTy->getPointerTo();
1133 StructType *ArgStructTy =
1134 StructType::get(I32PtrTy, StructPtrTy, I32PtrTy, StructPtrTy);
1135 Type *VoidTy = Type::getVoidTy(M->getContext());
1136 FunctionCallee RetI32Func = M->getOrInsertFunction("ret_i32", I32Ty);
1137 FunctionCallee TakeI32Func =
1138 M->getOrInsertFunction("take_i32", VoidTy, I32Ty);
1139 FunctionCallee RetI32PtrFunc = M->getOrInsertFunction("ret_i32ptr", I32PtrTy);
1140 FunctionCallee TakeI32PtrFunc =
1141 M->getOrInsertFunction("take_i32ptr", VoidTy, I32PtrTy);
1142 FunctionCallee RetStructFunc = M->getOrInsertFunction("ret_struct", StructTy);
1143 FunctionCallee TakeStructFunc =
1144 M->getOrInsertFunction("take_struct", VoidTy, StructTy);
1145 FunctionCallee RetStructPtrFunc =
1146 M->getOrInsertFunction("ret_structptr", StructPtrTy);
1147 FunctionCallee TakeStructPtrFunc =
1148 M->getOrInsertFunction("take_structPtr", VoidTy, StructPtrTy);
1149 Value *I32Val = Builder.CreateCall(RetI32Func);
1150 Value *I32PtrVal = Builder.CreateCall(RetI32PtrFunc);
1151 Value *StructVal = Builder.CreateCall(RetStructFunc);
1152 Value *StructPtrVal = Builder.CreateCall(RetStructPtrFunc);
1153
1154 Instruction *Internal;
1155 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
1156 IRBuilder<>::InsertPointGuard Guard(Builder);
1157 Builder.restoreIP(CodeGenIP);
1158 Internal = Builder.CreateCall(TakeI32Func, I32Val);
1159 Builder.CreateCall(TakeI32PtrFunc, I32PtrVal);
1160 Builder.CreateCall(TakeStructFunc, StructVal);
1161 Builder.CreateCall(TakeStructPtrFunc, StructPtrVal);
1162 };
1163 auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value &,
1164 Value &Inner, Value *&ReplacementValue) {
1165 ReplacementValue = &Inner;
1166 return CodeGenIP;
1167 };
1168 auto FiniCB = [](InsertPointTy) {};
1169
1170 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
1171 F->getEntryBlock().getFirstInsertionPt());
1172 IRBuilder<>::InsertPoint AfterIP =
1173 OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
1174 nullptr, nullptr, OMP_PROC_BIND_default, false);
1175 Builder.restoreIP(AfterIP);
1176 Builder.CreateRetVoid();
1177
1178 OMPBuilder.finalize();
1179
1180 EXPECT_FALSE(verifyModule(*M, &errs()));
1181 Function *OutlinedFn = Internal->getFunction();
1182
1183 Type *Arg2Type = OutlinedFn->getArg(2)->getType();
1184 EXPECT_TRUE(Arg2Type->isPointerTy());
1185 EXPECT_TRUE(
1186 cast<PointerType>(Arg2Type)->isOpaqueOrPointeeTypeMatches(ArgStructTy));
1187 }
1188
1189 TEST_F(OpenMPIRBuilderTest, CanonicalLoopSimple) {
1190 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1191 OpenMPIRBuilder OMPBuilder(*M);
1192 OMPBuilder.initialize();
1193 IRBuilder<> Builder(BB);
1194 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1195 Value *TripCount = F->getArg(0);
1196
1197 unsigned NumBodiesGenerated = 0;
1198 auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) {
1199 NumBodiesGenerated += 1;
1200
1201 Builder.restoreIP(CodeGenIP);
1202
1203 Value *Cmp = Builder.CreateICmpEQ(LC, TripCount);
1204 Instruction *ThenTerm, *ElseTerm;
1205 SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(),
1206 &ThenTerm, &ElseTerm);
1207 };
1208
1209 CanonicalLoopInfo *Loop =
1210 OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, TripCount);
1211
1212 Builder.restoreIP(Loop->getAfterIP());
1213 ReturnInst *RetInst = Builder.CreateRetVoid();
1214 OMPBuilder.finalize();
1215
1216 Loop->assertOK();
1217 EXPECT_FALSE(verifyModule(*M, &errs()));
1218
1219 EXPECT_EQ(NumBodiesGenerated, 1U);
1220
1221 // Verify control flow structure (in addition to Loop->assertOK()).
1222 EXPECT_EQ(Loop->getPreheader()->getSinglePredecessor(), &F->getEntryBlock());
1223 EXPECT_EQ(Loop->getAfter(), Builder.GetInsertBlock());
1224
1225 Instruction *IndVar = Loop->getIndVar();
1226 EXPECT_TRUE(isa<PHINode>(IndVar));
1227 EXPECT_EQ(IndVar->getType(), TripCount->getType());
1228 EXPECT_EQ(IndVar->getParent(), Loop->getHeader());
1229
1230 EXPECT_EQ(Loop->getTripCount(), TripCount);
1231
1232 BasicBlock *Body = Loop->getBody();
1233 Instruction *CmpInst = &Body->front();
1234 EXPECT_TRUE(isa<ICmpInst>(CmpInst));
1235 EXPECT_EQ(CmpInst->getOperand(0), IndVar);
1236
1237 BasicBlock *LatchPred = Loop->getLatch()->getSinglePredecessor();
1238 EXPECT_TRUE(llvm::all_of(successors(Body), [=](BasicBlock *SuccBB) {
1239 return SuccBB->getSingleSuccessor() == LatchPred;
1240 }));
1241
1242 EXPECT_EQ(&Loop->getAfter()->front(), RetInst);
1243 }
1244
1245 TEST_F(OpenMPIRBuilderTest, CanonicalLoopBounds) {
1246 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1247 OpenMPIRBuilder OMPBuilder(*M);
1248 OMPBuilder.initialize();
1249 IRBuilder<> Builder(BB);
1250
1251 // Check the trip count is computed correctly. We generate the canonical loop
1252 // but rely on the IRBuilder's constant folder to compute the final result
1253 // since all inputs are constant. To verify overflow situations, limit the
1254 // trip count / loop counter widths to 16 bits.
1255 auto EvalTripCount = [&](int64_t Start, int64_t Stop, int64_t Step,
1256 bool IsSigned, bool InclusiveStop) -> int64_t {
1257 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1258 Type *LCTy = Type::getInt16Ty(Ctx);
1259 Value *StartVal = ConstantInt::get(LCTy, Start);
1260 Value *StopVal = ConstantInt::get(LCTy, Stop);
1261 Value *StepVal = ConstantInt::get(LCTy, Step);
1262 auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) {};
1263 CanonicalLoopInfo *Loop =
1264 OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, StartVal, StopVal,
1265 StepVal, IsSigned, InclusiveStop);
1266 Loop->assertOK();
1267 Builder.restoreIP(Loop->getAfterIP());
1268 Value *TripCount = Loop->getTripCount();
1269 return cast<ConstantInt>(TripCount)->getValue().getZExtValue();
1270 };
1271
1272 EXPECT_EQ(EvalTripCount(0, 0, 1, false, false), 0);
1273 EXPECT_EQ(EvalTripCount(0, 1, 2, false, false), 1);
1274 EXPECT_EQ(EvalTripCount(0, 42, 1, false, false), 42);
1275 EXPECT_EQ(EvalTripCount(0, 42, 2, false, false), 21);
1276 EXPECT_EQ(EvalTripCount(21, 42, 1, false, false), 21);
1277 EXPECT_EQ(EvalTripCount(0, 5, 5, false, false), 1);
1278 EXPECT_EQ(EvalTripCount(0, 9, 5, false, false), 2);
1279 EXPECT_EQ(EvalTripCount(0, 11, 5, false, false), 3);
1280 EXPECT_EQ(EvalTripCount(0, 0xFFFF, 1, false, false), 0xFFFF);
1281 EXPECT_EQ(EvalTripCount(0xFFFF, 0, 1, false, false), 0);
1282 EXPECT_EQ(EvalTripCount(0xFFFE, 0xFFFF, 1, false, false), 1);
1283 EXPECT_EQ(EvalTripCount(0, 0xFFFF, 0x100, false, false), 0x100);
1284 EXPECT_EQ(EvalTripCount(0, 0xFFFF, 0xFFFF, false, false), 1);
1285
1286 EXPECT_EQ(EvalTripCount(0, 6, 5, false, false), 2);
1287 EXPECT_EQ(EvalTripCount(0, 0xFFFF, 0xFFFE, false, false), 2);
1288 EXPECT_EQ(EvalTripCount(0, 0, 1, false, true), 1);
1289 EXPECT_EQ(EvalTripCount(0, 0, 0xFFFF, false, true), 1);
1290 EXPECT_EQ(EvalTripCount(0, 0xFFFE, 1, false, true), 0xFFFF);
1291 EXPECT_EQ(EvalTripCount(0, 0xFFFE, 2, false, true), 0x8000);
1292
1293 EXPECT_EQ(EvalTripCount(0, 0, -1, true, false), 0);
1294 EXPECT_EQ(EvalTripCount(0, 1, -1, true, true), 0);
1295 EXPECT_EQ(EvalTripCount(20, 5, -5, true, false), 3);
1296 EXPECT_EQ(EvalTripCount(20, 5, -5, true, true), 4);
1297 EXPECT_EQ(EvalTripCount(-4, -2, 2, true, false), 1);
1298 EXPECT_EQ(EvalTripCount(-4, -3, 2, true, false), 1);
1299 EXPECT_EQ(EvalTripCount(-4, -2, 2, true, true), 2);
1300
1301 EXPECT_EQ(EvalTripCount(INT16_MIN, 0, 1, true, false), 0x8000);
1302 EXPECT_EQ(EvalTripCount(INT16_MIN, 0, 1, true, true), 0x8001);
1303 EXPECT_EQ(EvalTripCount(INT16_MIN, 0x7FFF, 1, true, false), 0xFFFF);
1304 EXPECT_EQ(EvalTripCount(INT16_MIN + 1, 0x7FFF, 1, true, true), 0xFFFF);
1305 EXPECT_EQ(EvalTripCount(INT16_MIN, 0, 0x7FFF, true, false), 2);
1306 EXPECT_EQ(EvalTripCount(0x7FFF, 0, -1, true, false), 0x7FFF);
1307 EXPECT_EQ(EvalTripCount(0, INT16_MIN, -1, true, false), 0x8000);
1308 EXPECT_EQ(EvalTripCount(0, INT16_MIN, -16, true, false), 0x800);
1309 EXPECT_EQ(EvalTripCount(0x7FFF, INT16_MIN, -1, true, false), 0xFFFF);
1310 EXPECT_EQ(EvalTripCount(0x7FFF, 1, INT16_MIN, true, false), 1);
1311 EXPECT_EQ(EvalTripCount(0x7FFF, -1, INT16_MIN, true, true), 2);
1312
1313 // Finalize the function and verify it.
1314 Builder.CreateRetVoid();
1315 OMPBuilder.finalize();
1316 EXPECT_FALSE(verifyModule(*M, &errs()));
1317 }
1318
1319 TEST_F(OpenMPIRBuilderTest, CollapseNestedLoops) {
1320 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1321 OpenMPIRBuilder OMPBuilder(*M);
1322 OMPBuilder.initialize();
1323 F->setName("func");
1324
1325 IRBuilder<> Builder(BB);
1326
1327 Type *LCTy = F->getArg(0)->getType();
1328 Constant *One = ConstantInt::get(LCTy, 1);
1329 Constant *Two = ConstantInt::get(LCTy, 2);
1330 Value *OuterTripCount =
1331 Builder.CreateAdd(F->getArg(0), Two, "tripcount.outer");
1332 Value *InnerTripCount =
1333 Builder.CreateAdd(F->getArg(0), One, "tripcount.inner");
1334
1335 // Fix an insertion point for ComputeIP.
1336 BasicBlock *LoopNextEnter =
1337 BasicBlock::Create(M->getContext(), "loopnest.enter", F,
1338 Builder.GetInsertBlock()->getNextNode());
1339 BranchInst *EnterBr = Builder.CreateBr(LoopNextEnter);
1340 InsertPointTy ComputeIP{EnterBr->getParent(), EnterBr->getIterator()};
1341
1342 Builder.SetInsertPoint(LoopNextEnter);
1343 OpenMPIRBuilder::LocationDescription OuterLoc(Builder.saveIP(), DL);
1344
1345 CanonicalLoopInfo *InnerLoop = nullptr;
1346 CallInst *InbetweenLead = nullptr;
1347 CallInst *InbetweenTrail = nullptr;
1348 CallInst *Call = nullptr;
1349 auto OuterLoopBodyGenCB = [&](InsertPointTy OuterCodeGenIP, Value *OuterLC) {
1350 Builder.restoreIP(OuterCodeGenIP);
1351 InbetweenLead =
1352 createPrintfCall(Builder, "In-between lead i=%d\\n", {OuterLC});
1353
1354 auto InnerLoopBodyGenCB = [&](InsertPointTy InnerCodeGenIP,
1355 Value *InnerLC) {
1356 Builder.restoreIP(InnerCodeGenIP);
1357 Call = createPrintfCall(Builder, "body i=%d j=%d\\n", {OuterLC, InnerLC});
1358 };
1359 InnerLoop = OMPBuilder.createCanonicalLoop(
1360 Builder.saveIP(), InnerLoopBodyGenCB, InnerTripCount, "inner");
1361
1362 Builder.restoreIP(InnerLoop->getAfterIP());
1363 InbetweenTrail =
1364 createPrintfCall(Builder, "In-between trail i=%d\\n", {OuterLC});
1365 };
1366 CanonicalLoopInfo *OuterLoop = OMPBuilder.createCanonicalLoop(
1367 OuterLoc, OuterLoopBodyGenCB, OuterTripCount, "outer");
1368
1369 // Finish the function.
1370 Builder.restoreIP(OuterLoop->getAfterIP());
1371 Builder.CreateRetVoid();
1372
1373 CanonicalLoopInfo *Collapsed =
1374 OMPBuilder.collapseLoops(DL, {OuterLoop, InnerLoop}, ComputeIP);
1375
1376 OMPBuilder.finalize();
1377 EXPECT_FALSE(verifyModule(*M, &errs()));
1378
1379 // Verify control flow and BB order.
1380 BasicBlock *RefOrder[] = {
1381 Collapsed->getPreheader(), Collapsed->getHeader(),
1382 Collapsed->getCond(), Collapsed->getBody(),
1383 InbetweenLead->getParent(), Call->getParent(),
1384 InbetweenTrail->getParent(), Collapsed->getLatch(),
1385 Collapsed->getExit(), Collapsed->getAfter(),
1386 };
1387 EXPECT_TRUE(verifyDFSOrder(F, RefOrder));
1388 EXPECT_TRUE(verifyListOrder(F, RefOrder));
1389
1390 // Verify the total trip count.
1391 auto *TripCount = cast<MulOperator>(Collapsed->getTripCount());
1392 EXPECT_EQ(TripCount->getOperand(0), OuterTripCount);
1393 EXPECT_EQ(TripCount->getOperand(1), InnerTripCount);
1394
1395 // Verify the changed indvar.
1396 auto *OuterIV = cast<BinaryOperator>(Call->getOperand(1));
1397 EXPECT_EQ(OuterIV->getOpcode(), Instruction::UDiv);
1398 EXPECT_EQ(OuterIV->getParent(), Collapsed->getBody());
1399 EXPECT_EQ(OuterIV->getOperand(1), InnerTripCount);
1400 EXPECT_EQ(OuterIV->getOperand(0), Collapsed->getIndVar());
1401
1402 auto *InnerIV = cast<BinaryOperator>(Call->getOperand(2));
1403 EXPECT_EQ(InnerIV->getOpcode(), Instruction::URem);
1404 EXPECT_EQ(InnerIV->getParent(), Collapsed->getBody());
1405 EXPECT_EQ(InnerIV->getOperand(0), Collapsed->getIndVar());
1406 EXPECT_EQ(InnerIV->getOperand(1), InnerTripCount);
1407
1408 EXPECT_EQ(InbetweenLead->getOperand(1), OuterIV);
1409 EXPECT_EQ(InbetweenTrail->getOperand(1), OuterIV);
1410 }
1411
1412 TEST_F(OpenMPIRBuilderTest, TileSingleLoop) {
1413 OpenMPIRBuilder OMPBuilder(*M);
1414 CallInst *Call;
1415 BasicBlock *BodyCode;
1416 CanonicalLoopInfo *Loop =
1417 buildSingleLoopFunction(DL, OMPBuilder, 32, &Call, &BodyCode);
1418
1419 Instruction *OrigIndVar = Loop->getIndVar();
1420 EXPECT_EQ(Call->getOperand(1), OrigIndVar);
1421
1422 // Tile the loop.
1423 Constant *TileSize = ConstantInt::get(Loop->getIndVarType(), APInt(32, 7));
1424 std::vector<CanonicalLoopInfo *> GenLoops =
1425 OMPBuilder.tileLoops(DL, {Loop}, {TileSize});
1426
1427 OMPBuilder.finalize();
1428 EXPECT_FALSE(verifyModule(*M, &errs()));
1429
1430 EXPECT_EQ(GenLoops.size(), 2u);
1431 CanonicalLoopInfo *Floor = GenLoops[0];
1432 CanonicalLoopInfo *Tile = GenLoops[1];
1433
1434 BasicBlock *RefOrder[] = {
1435 Floor->getPreheader(), Floor->getHeader(), Floor->getCond(),
1436 Floor->getBody(), Tile->getPreheader(), Tile->getHeader(),
1437 Tile->getCond(), Tile->getBody(), BodyCode,
1438 Tile->getLatch(), Tile->getExit(), Tile->getAfter(),
1439 Floor->getLatch(), Floor->getExit(), Floor->getAfter(),
1440 };
1441 EXPECT_TRUE(verifyDFSOrder(F, RefOrder));
1442 EXPECT_TRUE(verifyListOrder(F, RefOrder));
1443
1444 // Check the induction variable.
1445 EXPECT_EQ(Call->getParent(), BodyCode);
1446 auto *Shift = cast<AddOperator>(Call->getOperand(1));
1447 EXPECT_EQ(cast<Instruction>(Shift)->getParent(), Tile->getBody());
1448 EXPECT_EQ(Shift->getOperand(1), Tile->getIndVar());
1449 auto *Scale = cast<MulOperator>(Shift->getOperand(0));
1450 EXPECT_EQ(cast<Instruction>(Scale)->getParent(), Tile->getBody());
1451 EXPECT_EQ(Scale->getOperand(0), TileSize);
1452 EXPECT_EQ(Scale->getOperand(1), Floor->getIndVar());
1453 }
1454
1455 TEST_F(OpenMPIRBuilderTest, TileNestedLoops) {
1456 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1457 OpenMPIRBuilder OMPBuilder(*M);
1458 OMPBuilder.initialize();
1459 F->setName("func");
1460
1461 IRBuilder<> Builder(BB);
1462 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1463 Value *TripCount = F->getArg(0);
1464 Type *LCTy = TripCount->getType();
1465
1466 BasicBlock *BodyCode = nullptr;
1467 CanonicalLoopInfo *InnerLoop = nullptr;
1468 auto OuterLoopBodyGenCB = [&](InsertPointTy OuterCodeGenIP,
1469 llvm::Value *OuterLC) {
1470 auto InnerLoopBodyGenCB = [&](InsertPointTy InnerCodeGenIP,
1471 llvm::Value *InnerLC) {
1472 Builder.restoreIP(InnerCodeGenIP);
1473 BodyCode = Builder.GetInsertBlock();
1474
1475 // Add something that consumes the induction variables to the body.
1476 createPrintfCall(Builder, "i=%d j=%d\\n", {OuterLC, InnerLC});
1477 };
1478 InnerLoop = OMPBuilder.createCanonicalLoop(
1479 OuterCodeGenIP, InnerLoopBodyGenCB, TripCount, "inner");
1480 };
1481 CanonicalLoopInfo *OuterLoop = OMPBuilder.createCanonicalLoop(
1482 Loc, OuterLoopBodyGenCB, TripCount, "outer");
1483
1484 // Finalize the function.
1485 Builder.restoreIP(OuterLoop->getAfterIP());
1486 Builder.CreateRetVoid();
1487
1488 // Tile to loop nest.
1489 Constant *OuterTileSize = ConstantInt::get(LCTy, APInt(32, 11));
1490 Constant *InnerTileSize = ConstantInt::get(LCTy, APInt(32, 7));
1491 std::vector<CanonicalLoopInfo *> GenLoops = OMPBuilder.tileLoops(
1492 DL, {OuterLoop, InnerLoop}, {OuterTileSize, InnerTileSize});
1493
1494 OMPBuilder.finalize();
1495 EXPECT_FALSE(verifyModule(*M, &errs()));
1496
1497 EXPECT_EQ(GenLoops.size(), 4u);
1498 CanonicalLoopInfo *Floor1 = GenLoops[0];
1499 CanonicalLoopInfo *Floor2 = GenLoops[1];
1500 CanonicalLoopInfo *Tile1 = GenLoops[2];
1501 CanonicalLoopInfo *Tile2 = GenLoops[3];
1502
1503 BasicBlock *RefOrder[] = {
1504 Floor1->getPreheader(),
1505 Floor1->getHeader(),
1506 Floor1->getCond(),
1507 Floor1->getBody(),
1508 Floor2->getPreheader(),
1509 Floor2->getHeader(),
1510 Floor2->getCond(),
1511 Floor2->getBody(),
1512 Tile1->getPreheader(),
1513 Tile1->getHeader(),
1514 Tile1->getCond(),
1515 Tile1->getBody(),
1516 Tile2->getPreheader(),
1517 Tile2->getHeader(),
1518 Tile2->getCond(),
1519 Tile2->getBody(),
1520 BodyCode,
1521 Tile2->getLatch(),
1522 Tile2->getExit(),
1523 Tile2->getAfter(),
1524 Tile1->getLatch(),
1525 Tile1->getExit(),
1526 Tile1->getAfter(),
1527 Floor2->getLatch(),
1528 Floor2->getExit(),
1529 Floor2->getAfter(),
1530 Floor1->getLatch(),
1531 Floor1->getExit(),
1532 Floor1->getAfter(),
1533 };
1534 EXPECT_TRUE(verifyDFSOrder(F, RefOrder));
1535 EXPECT_TRUE(verifyListOrder(F, RefOrder));
1536 }
1537
1538 TEST_F(OpenMPIRBuilderTest, TileNestedLoopsWithBounds) {
1539 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1540 OpenMPIRBuilder OMPBuilder(*M);
1541 OMPBuilder.initialize();
1542 F->setName("func");
1543
1544 IRBuilder<> Builder(BB);
1545 Value *TripCount = F->getArg(0);
1546 Type *LCTy = TripCount->getType();
1547
1548 Value *OuterStartVal = ConstantInt::get(LCTy, 2);
1549 Value *OuterStopVal = TripCount;
1550 Value *OuterStep = ConstantInt::get(LCTy, 5);
1551 Value *InnerStartVal = ConstantInt::get(LCTy, 13);
1552 Value *InnerStopVal = TripCount;
1553 Value *InnerStep = ConstantInt::get(LCTy, 3);
1554
1555 // Fix an insertion point for ComputeIP.
1556 BasicBlock *LoopNextEnter =
1557 BasicBlock::Create(M->getContext(), "loopnest.enter", F,
1558 Builder.GetInsertBlock()->getNextNode());
1559 BranchInst *EnterBr = Builder.CreateBr(LoopNextEnter);
1560 InsertPointTy ComputeIP{EnterBr->getParent(), EnterBr->getIterator()};
1561
1562 InsertPointTy LoopIP{LoopNextEnter, LoopNextEnter->begin()};
1563 OpenMPIRBuilder::LocationDescription Loc({LoopIP, DL});
1564
1565 BasicBlock *BodyCode = nullptr;
1566 CanonicalLoopInfo *InnerLoop = nullptr;
1567 CallInst *Call = nullptr;
1568 auto OuterLoopBodyGenCB = [&](InsertPointTy OuterCodeGenIP,
1569 llvm::Value *OuterLC) {
1570 auto InnerLoopBodyGenCB = [&](InsertPointTy InnerCodeGenIP,
1571 llvm::Value *InnerLC) {
1572 Builder.restoreIP(InnerCodeGenIP);
1573 BodyCode = Builder.GetInsertBlock();
1574
1575 // Add something that consumes the induction variable to the body.
1576 Call = createPrintfCall(Builder, "i=%d j=%d\\n", {OuterLC, InnerLC});
1577 };
1578 InnerLoop = OMPBuilder.createCanonicalLoop(
1579 OuterCodeGenIP, InnerLoopBodyGenCB, InnerStartVal, InnerStopVal,
1580 InnerStep, false, false, ComputeIP, "inner");
1581 };
1582 CanonicalLoopInfo *OuterLoop = OMPBuilder.createCanonicalLoop(
1583 Loc, OuterLoopBodyGenCB, OuterStartVal, OuterStopVal, OuterStep, false,
1584 false, ComputeIP, "outer");
1585
1586 // Finalize the function
1587 Builder.restoreIP(OuterLoop->getAfterIP());
1588 Builder.CreateRetVoid();
1589
1590 // Tile the loop nest.
1591 Constant *TileSize0 = ConstantInt::get(LCTy, APInt(32, 11));
1592 Constant *TileSize1 = ConstantInt::get(LCTy, APInt(32, 7));
1593 std::vector<CanonicalLoopInfo *> GenLoops =
1594 OMPBuilder.tileLoops(DL, {OuterLoop, InnerLoop}, {TileSize0, TileSize1});
1595
1596 OMPBuilder.finalize();
1597 EXPECT_FALSE(verifyModule(*M, &errs()));
1598
1599 EXPECT_EQ(GenLoops.size(), 4u);
1600 CanonicalLoopInfo *Floor0 = GenLoops[0];
1601 CanonicalLoopInfo *Floor1 = GenLoops[1];
1602 CanonicalLoopInfo *Tile0 = GenLoops[2];
1603 CanonicalLoopInfo *Tile1 = GenLoops[3];
1604
1605 BasicBlock *RefOrder[] = {
1606 Floor0->getPreheader(),
1607 Floor0->getHeader(),
1608 Floor0->getCond(),
1609 Floor0->getBody(),
1610 Floor1->getPreheader(),
1611 Floor1->getHeader(),
1612 Floor1->getCond(),
1613 Floor1->getBody(),
1614 Tile0->getPreheader(),
1615 Tile0->getHeader(),
1616 Tile0->getCond(),
1617 Tile0->getBody(),
1618 Tile1->getPreheader(),
1619 Tile1->getHeader(),
1620 Tile1->getCond(),
1621 Tile1->getBody(),
1622 BodyCode,
1623 Tile1->getLatch(),
1624 Tile1->getExit(),
1625 Tile1->getAfter(),
1626 Tile0->getLatch(),
1627 Tile0->getExit(),
1628 Tile0->getAfter(),
1629 Floor1->getLatch(),
1630 Floor1->getExit(),
1631 Floor1->getAfter(),
1632 Floor0->getLatch(),
1633 Floor0->getExit(),
1634 Floor0->getAfter(),
1635 };
1636 EXPECT_TRUE(verifyDFSOrder(F, RefOrder));
1637 EXPECT_TRUE(verifyListOrder(F, RefOrder));
1638
1639 EXPECT_EQ(Call->getParent(), BodyCode);
1640
1641 auto *RangeShift0 = cast<AddOperator>(Call->getOperand(1));
1642 EXPECT_EQ(RangeShift0->getOperand(1), OuterStartVal);
1643 auto *RangeScale0 = cast<MulOperator>(RangeShift0->getOperand(0));
1644 EXPECT_EQ(RangeScale0->getOperand(1), OuterStep);
1645 auto *TileShift0 = cast<AddOperator>(RangeScale0->getOperand(0));
1646 EXPECT_EQ(cast<Instruction>(TileShift0)->getParent(), Tile1->getBody());
1647 EXPECT_EQ(TileShift0->getOperand(1), Tile0->getIndVar());
1648 auto *TileScale0 = cast<MulOperator>(TileShift0->getOperand(0));
1649 EXPECT_EQ(cast<Instruction>(TileScale0)->getParent(), Tile1->getBody());
1650 EXPECT_EQ(TileScale0->getOperand(0), TileSize0);
1651 EXPECT_EQ(TileScale0->getOperand(1), Floor0->getIndVar());
1652
1653 auto *RangeShift1 = cast<AddOperator>(Call->getOperand(2));
1654 EXPECT_EQ(cast<Instruction>(RangeShift1)->getParent(), BodyCode);
1655 EXPECT_EQ(RangeShift1->getOperand(1), InnerStartVal);
1656 auto *RangeScale1 = cast<MulOperator>(RangeShift1->getOperand(0));
1657 EXPECT_EQ(cast<Instruction>(RangeScale1)->getParent(), BodyCode);
1658 EXPECT_EQ(RangeScale1->getOperand(1), InnerStep);
1659 auto *TileShift1 = cast<AddOperator>(RangeScale1->getOperand(0));
1660 EXPECT_EQ(cast<Instruction>(TileShift1)->getParent(), Tile1->getBody());
1661 EXPECT_EQ(TileShift1->getOperand(1), Tile1->getIndVar());
1662 auto *TileScale1 = cast<MulOperator>(TileShift1->getOperand(0));
1663 EXPECT_EQ(cast<Instruction>(TileScale1)->getParent(), Tile1->getBody());
1664 EXPECT_EQ(TileScale1->getOperand(0), TileSize1);
1665 EXPECT_EQ(TileScale1->getOperand(1), Floor1->getIndVar());
1666 }
1667
1668 TEST_F(OpenMPIRBuilderTest, TileSingleLoopCounts) {
1669 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1670 OpenMPIRBuilder OMPBuilder(*M);
1671 OMPBuilder.initialize();
1672 IRBuilder<> Builder(BB);
1673
1674 // Create a loop, tile it, and extract its trip count. All input values are
1675 // constant and IRBuilder evaluates all-constant arithmetic inplace, such that
1676 // the floor trip count itself will be a ConstantInt. Unfortunately we cannot
1677 // do the same for the tile loop.
1678 auto GetFloorCount = [&](int64_t Start, int64_t Stop, int64_t Step,
1679 bool IsSigned, bool InclusiveStop,
1680 int64_t TileSize) -> uint64_t {
1681 OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL);
1682 Type *LCTy = Type::getInt16Ty(Ctx);
1683 Value *StartVal = ConstantInt::get(LCTy, Start);
1684 Value *StopVal = ConstantInt::get(LCTy, Stop);
1685 Value *StepVal = ConstantInt::get(LCTy, Step);
1686
1687 // Generate a loop.
1688 auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) {};
1689 CanonicalLoopInfo *Loop =
1690 OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, StartVal, StopVal,
1691 StepVal, IsSigned, InclusiveStop);
1692 InsertPointTy AfterIP = Loop->getAfterIP();
1693
1694 // Tile the loop.
1695 Value *TileSizeVal = ConstantInt::get(LCTy, TileSize);
1696 std::vector<CanonicalLoopInfo *> GenLoops =
1697 OMPBuilder.tileLoops(Loc.DL, {Loop}, {TileSizeVal});
1698
1699 // Set the insertion pointer to after loop, where the next loop will be
1700 // emitted.
1701 Builder.restoreIP(AfterIP);
1702
1703 // Extract the trip count.
1704 CanonicalLoopInfo *FloorLoop = GenLoops[0];
1705 Value *FloorTripCount = FloorLoop->getTripCount();
1706 return cast<ConstantInt>(FloorTripCount)->getValue().getZExtValue();
1707 };
1708
1709 // Empty iteration domain.
1710 EXPECT_EQ(GetFloorCount(0, 0, 1, false, false, 7), 0u);
1711 EXPECT_EQ(GetFloorCount(0, -1, 1, false, true, 7), 0u);
1712 EXPECT_EQ(GetFloorCount(-1, -1, -1, true, false, 7), 0u);
1713 EXPECT_EQ(GetFloorCount(-1, 0, -1, true, true, 7), 0u);
1714 EXPECT_EQ(GetFloorCount(-1, -1, 3, true, false, 7), 0u);
1715
1716 // Only complete tiles.
1717 EXPECT_EQ(GetFloorCount(0, 14, 1, false, false, 7), 2u);
1718 EXPECT_EQ(GetFloorCount(0, 14, 1, false, false, 7), 2u);
1719 EXPECT_EQ(GetFloorCount(1, 15, 1, false, false, 7), 2u);
1720 EXPECT_EQ(GetFloorCount(0, -14, -1, true, false, 7), 2u);
1721 EXPECT_EQ(GetFloorCount(-1, -14, -1, true, true, 7), 2u);
1722 EXPECT_EQ(GetFloorCount(0, 3 * 7 * 2, 3, false, false, 7), 2u);
1723
1724 // Only a partial tile.
1725 EXPECT_EQ(GetFloorCount(0, 1, 1, false, false, 7), 1u);
1726 EXPECT_EQ(GetFloorCount(0, 6, 1, false, false, 7), 1u);
1727 EXPECT_EQ(GetFloorCount(-1, 1, 3, true, false, 7), 1u);
1728 EXPECT_EQ(GetFloorCount(-1, -2, -1, true, false, 7), 1u);
1729 EXPECT_EQ(GetFloorCount(0, 2, 3, false, false, 7), 1u);
1730
1731 // Complete and partial tiles.
1732 EXPECT_EQ(GetFloorCount(0, 13, 1, false, false, 7), 2u);
1733 EXPECT_EQ(GetFloorCount(0, 15, 1, false, false, 7), 3u);
1734 EXPECT_EQ(GetFloorCount(-1, -14, -1, true, false, 7), 2u);
1735 EXPECT_EQ(GetFloorCount(0, 3 * 7 * 5 - 1, 3, false, false, 7), 5u);
1736 EXPECT_EQ(GetFloorCount(-1, -3 * 7 * 5, -3, true, false, 7), 5u);
1737
1738 // Close to 16-bit integer range.
1739 EXPECT_EQ(GetFloorCount(0, 0xFFFF, 1, false, false, 1), 0xFFFFu);
1740 EXPECT_EQ(GetFloorCount(0, 0xFFFF, 1, false, false, 7), 0xFFFFu / 7 + 1);
1741 EXPECT_EQ(GetFloorCount(0, 0xFFFE, 1, false, true, 7), 0xFFFFu / 7 + 1);
1742 EXPECT_EQ(GetFloorCount(-0x8000, 0x7FFF, 1, true, false, 7), 0xFFFFu / 7 + 1);
1743 EXPECT_EQ(GetFloorCount(-0x7FFF, 0x7FFF, 1, true, true, 7), 0xFFFFu / 7 + 1);
1744 EXPECT_EQ(GetFloorCount(0, 0xFFFE, 1, false, false, 0xFFFF), 1u);
1745 EXPECT_EQ(GetFloorCount(-0x8000, 0x7FFF, 1, true, false, 0xFFFF), 1u);
1746
1747 // Finalize the function.
1748 Builder.CreateRetVoid();
1749 OMPBuilder.finalize();
1750
1751 EXPECT_FALSE(verifyModule(*M, &errs()));
1752 }
1753
1754 TEST_F(OpenMPIRBuilderTest, ApplySimd) {
1755 OpenMPIRBuilder OMPBuilder(*M);
1756 MapVector<Value *, Value *> AlignedVars;
1757 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
1758
1759 // Simd-ize the loop.
1760 OMPBuilder.applySimd(CLI, AlignedVars, /* IfCond */ nullptr,
1761 OrderKind::OMP_ORDER_unknown,
1762 /* Simdlen */ nullptr,
1763 /* Safelen */ nullptr);
1764
1765 OMPBuilder.finalize();
1766 EXPECT_FALSE(verifyModule(*M, &errs()));
1767
1768 PassBuilder PB;
1769 FunctionAnalysisManager FAM;
1770 PB.registerFunctionAnalyses(FAM);
1771 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
1772
1773 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
1774 EXPECT_EQ(TopLvl.size(), 1u);
1775
1776 Loop *L = TopLvl.front();
1777 EXPECT_TRUE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));
1778 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));
1779
1780 // Check for llvm.access.group metadata attached to the printf
1781 // function in the loop body.
1782 BasicBlock *LoopBody = CLI->getBody();
1783 EXPECT_TRUE(any_of(*LoopBody, [](Instruction &I) {
1784 return I.getMetadata("llvm.access.group") != nullptr;
1785 }));
1786 }
1787
1788 TEST_F(OpenMPIRBuilderTest, ApplySimdCustomAligned) {
1789 OpenMPIRBuilder OMPBuilder(*M);
1790 IRBuilder<> Builder(BB);
1791 const int AlignmentValue = 32;
1792 AllocaInst *Alloc1 =
1793 Builder.CreateAlloca(Builder.getInt8PtrTy(), Builder.getInt64(1));
1794 LoadInst *Load1 = Builder.CreateLoad(Alloc1->getAllocatedType(), Alloc1);
1795 MapVector<Value *, Value *> AlignedVars;
1796 AlignedVars.insert({Load1, Builder.getInt64(AlignmentValue)});
1797
1798 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
1799
1800 // Simd-ize the loop.
1801 OMPBuilder.applySimd(CLI, AlignedVars, /* IfCond */ nullptr,
1802 OrderKind::OMP_ORDER_unknown,
1803 /* Simdlen */ nullptr,
1804 /* Safelen */ nullptr);
1805
1806 OMPBuilder.finalize();
1807 EXPECT_FALSE(verifyModule(*M, &errs()));
1808
1809 PassBuilder PB;
1810 FunctionAnalysisManager FAM;
1811 PB.registerFunctionAnalyses(FAM);
1812 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
1813
1814 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
1815 EXPECT_EQ(TopLvl.size(), 1u);
1816
1817 Loop *L = TopLvl.front();
1818 EXPECT_TRUE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));
1819 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));
1820
1821 // Check for llvm.access.group metadata attached to the printf
1822 // function in the loop body.
1823 BasicBlock *LoopBody = CLI->getBody();
1824 EXPECT_TRUE(any_of(*LoopBody, [](Instruction &I) {
1825 return I.getMetadata("llvm.access.group") != nullptr;
1826 }));
1827
1828 // Check if number of assumption instructions is equal to number of aligned
1829 // variables
1830 BasicBlock *LoopPreheader = CLI->getPreheader();
1831 size_t NumAssummptionCallsInPreheader = count_if(
1832 *LoopPreheader, [](Instruction &I) { return isa<AssumeInst>(I); });
1833 EXPECT_EQ(NumAssummptionCallsInPreheader, AlignedVars.size());
1834
1835 // Check if variables are correctly aligned
1836 for (Instruction &Instr : *LoopPreheader) {
1837 if (!isa<AssumeInst>(Instr))
1838 continue;
1839 AssumeInst *AssumeInstruction = cast<AssumeInst>(&Instr);
1840 if (AssumeInstruction->getNumTotalBundleOperands()) {
1841 auto Bundle = AssumeInstruction->getOperandBundleAt(0);
1842 if (Bundle.getTagName() == "align") {
1843 EXPECT_TRUE(isa<ConstantInt>(Bundle.Inputs[1]));
1844 auto ConstIntVal = dyn_cast<ConstantInt>(Bundle.Inputs[1]);
1845 EXPECT_EQ(ConstIntVal->getSExtValue(), AlignmentValue);
1846 }
1847 }
1848 }
1849 }
1850 TEST_F(OpenMPIRBuilderTest, ApplySimdlen) {
1851 OpenMPIRBuilder OMPBuilder(*M);
1852 MapVector<Value *, Value *> AlignedVars;
1853 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
1854
1855 // Simd-ize the loop.
1856 OMPBuilder.applySimd(CLI, AlignedVars,
1857 /* IfCond */ nullptr, OrderKind::OMP_ORDER_unknown,
1858 ConstantInt::get(Type::getInt32Ty(Ctx), 3),
1859 /* Safelen */ nullptr);
1860
1861 OMPBuilder.finalize();
1862 EXPECT_FALSE(verifyModule(*M, &errs()));
1863
1864 PassBuilder PB;
1865 FunctionAnalysisManager FAM;
1866 PB.registerFunctionAnalyses(FAM);
1867 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
1868
1869 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
1870 EXPECT_EQ(TopLvl.size(), 1u);
1871
1872 Loop *L = TopLvl.front();
1873 EXPECT_TRUE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));
1874 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));
1875 EXPECT_EQ(getIntLoopAttribute(L, "llvm.loop.vectorize.width"), 3);
1876
1877 // Check for llvm.access.group metadata attached to the printf
1878 // function in the loop body.
1879 BasicBlock *LoopBody = CLI->getBody();
1880 EXPECT_TRUE(any_of(*LoopBody, [](Instruction &I) {
1881 return I.getMetadata("llvm.access.group") != nullptr;
1882 }));
1883 }
1884
1885 TEST_F(OpenMPIRBuilderTest, ApplySafelenOrderConcurrent) {
1886 OpenMPIRBuilder OMPBuilder(*M);
1887 MapVector<Value *, Value *> AlignedVars;
1888
1889 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
1890
1891 // Simd-ize the loop.
1892 OMPBuilder.applySimd(
1893 CLI, AlignedVars, /* IfCond */ nullptr, OrderKind::OMP_ORDER_concurrent,
1894 /* Simdlen */ nullptr, ConstantInt::get(Type::getInt32Ty(Ctx), 3));
1895
1896 OMPBuilder.finalize();
1897 EXPECT_FALSE(verifyModule(*M, &errs()));
1898
1899 PassBuilder PB;
1900 FunctionAnalysisManager FAM;
1901 PB.registerFunctionAnalyses(FAM);
1902 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
1903
1904 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
1905 EXPECT_EQ(TopLvl.size(), 1u);
1906
1907 Loop *L = TopLvl.front();
1908 // Parallel metadata shoudl be attached because of presence of
1909 // the order(concurrent) OpenMP clause
1910 EXPECT_TRUE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));
1911 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));
1912 EXPECT_EQ(getIntLoopAttribute(L, "llvm.loop.vectorize.width"), 3);
1913
1914 // Check for llvm.access.group metadata attached to the printf
1915 // function in the loop body.
1916 BasicBlock *LoopBody = CLI->getBody();
1917 EXPECT_TRUE(any_of(*LoopBody, [](Instruction &I) {
1918 return I.getMetadata("llvm.access.group") != nullptr;
1919 }));
1920 }
1921
1922 TEST_F(OpenMPIRBuilderTest, ApplySafelen) {
1923 OpenMPIRBuilder OMPBuilder(*M);
1924 MapVector<Value *, Value *> AlignedVars;
1925
1926 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
1927
1928 OMPBuilder.applySimd(
1929 CLI, AlignedVars, /* IfCond */ nullptr, OrderKind::OMP_ORDER_unknown,
1930 /* Simdlen */ nullptr, ConstantInt::get(Type::getInt32Ty(Ctx), 3));
1931
1932 OMPBuilder.finalize();
1933 EXPECT_FALSE(verifyModule(*M, &errs()));
1934
1935 PassBuilder PB;
1936 FunctionAnalysisManager FAM;
1937 PB.registerFunctionAnalyses(FAM);
1938 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
1939
1940 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
1941 EXPECT_EQ(TopLvl.size(), 1u);
1942
1943 Loop *L = TopLvl.front();
1944 EXPECT_FALSE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));
1945 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));
1946 EXPECT_EQ(getIntLoopAttribute(L, "llvm.loop.vectorize.width"), 3);
1947
1948 // Check for llvm.access.group metadata attached to the printf
1949 // function in the loop body.
1950 BasicBlock *LoopBody = CLI->getBody();
1951 EXPECT_FALSE(any_of(*LoopBody, [](Instruction &I) {
1952 return I.getMetadata("llvm.access.group") != nullptr;
1953 }));
1954 }
1955
1956 TEST_F(OpenMPIRBuilderTest, ApplySimdlenSafelen) {
1957 OpenMPIRBuilder OMPBuilder(*M);
1958 MapVector<Value *, Value *> AlignedVars;
1959
1960 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
1961
1962 OMPBuilder.applySimd(CLI, AlignedVars, /* IfCond */ nullptr,
1963 OrderKind::OMP_ORDER_unknown,
1964 ConstantInt::get(Type::getInt32Ty(Ctx), 2),
1965 ConstantInt::get(Type::getInt32Ty(Ctx), 3));
1966
1967 OMPBuilder.finalize();
1968 EXPECT_FALSE(verifyModule(*M, &errs()));
1969
1970 PassBuilder PB;
1971 FunctionAnalysisManager FAM;
1972 PB.registerFunctionAnalyses(FAM);
1973 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
1974
1975 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
1976 EXPECT_EQ(TopLvl.size(), 1u);
1977
1978 Loop *L = TopLvl.front();
1979 EXPECT_FALSE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));
1980 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));
1981 EXPECT_EQ(getIntLoopAttribute(L, "llvm.loop.vectorize.width"), 2);
1982
1983 // Check for llvm.access.group metadata attached to the printf
1984 // function in the loop body.
1985 BasicBlock *LoopBody = CLI->getBody();
1986 EXPECT_FALSE(any_of(*LoopBody, [](Instruction &I) {
1987 return I.getMetadata("llvm.access.group") != nullptr;
1988 }));
1989 }
1990
1991 TEST_F(OpenMPIRBuilderTest, ApplySimdLoopIf) {
1992 OpenMPIRBuilder OMPBuilder(*M);
1993 IRBuilder<> Builder(BB);
1994 MapVector<Value *, Value *> AlignedVars;
1995 AllocaInst *Alloc1 = Builder.CreateAlloca(Builder.getInt32Ty());
1996 AllocaInst *Alloc2 = Builder.CreateAlloca(Builder.getInt32Ty());
1997
1998 // Generation of if condition
1999 Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0U), Alloc1);
2000 Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 1U), Alloc2);
2001 LoadInst *Load1 = Builder.CreateLoad(Alloc1->getAllocatedType(), Alloc1);
2002 LoadInst *Load2 = Builder.CreateLoad(Alloc2->getAllocatedType(), Alloc2);
2003
2004 Value *IfCmp = Builder.CreateICmpNE(Load1, Load2);
2005
2006 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
2007
2008 // Simd-ize the loop with if condition
2009 OMPBuilder.applySimd(CLI, AlignedVars, IfCmp, OrderKind::OMP_ORDER_unknown,
2010 ConstantInt::get(Type::getInt32Ty(Ctx), 3),
2011 /* Safelen */ nullptr);
2012
2013 OMPBuilder.finalize();
2014 EXPECT_FALSE(verifyModule(*M, &errs()));
2015
2016 PassBuilder PB;
2017 FunctionAnalysisManager FAM;
2018 PB.registerFunctionAnalyses(FAM);
2019 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
2020
2021 // Check if there are two loops (one with enabled vectorization)
2022 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
2023 EXPECT_EQ(TopLvl.size(), 2u);
2024
2025 Loop *L = TopLvl[0];
2026 EXPECT_TRUE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));
2027 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));
2028 EXPECT_EQ(getIntLoopAttribute(L, "llvm.loop.vectorize.width"), 3);
2029
2030 // The second loop should have disabled vectorization
2031 L = TopLvl[1];
2032 EXPECT_FALSE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));
2033 EXPECT_FALSE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));
2034 // Check for llvm.access.group metadata attached to the printf
2035 // function in the loop body.
2036 BasicBlock *LoopBody = CLI->getBody();
2037 EXPECT_TRUE(any_of(*LoopBody, [](Instruction &I) {
2038 return I.getMetadata("llvm.access.group") != nullptr;
2039 }));
2040 }
2041
2042 TEST_F(OpenMPIRBuilderTest, UnrollLoopFull) {
2043 OpenMPIRBuilder OMPBuilder(*M);
2044
2045 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
2046
2047 // Unroll the loop.
2048 OMPBuilder.unrollLoopFull(DL, CLI);
2049
2050 OMPBuilder.finalize();
2051 EXPECT_FALSE(verifyModule(*M, &errs()));
2052
2053 PassBuilder PB;
2054 FunctionAnalysisManager FAM;
2055 PB.registerFunctionAnalyses(FAM);
2056 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
2057
2058 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
2059 EXPECT_EQ(TopLvl.size(), 1u);
2060
2061 Loop *L = TopLvl.front();
2062 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.unroll.enable"));
2063 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.unroll.full"));
2064 }
2065
2066 TEST_F(OpenMPIRBuilderTest, UnrollLoopPartial) {
2067 OpenMPIRBuilder OMPBuilder(*M);
2068 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
2069
2070 // Unroll the loop.
2071 CanonicalLoopInfo *UnrolledLoop = nullptr;
2072 OMPBuilder.unrollLoopPartial(DL, CLI, 5, &UnrolledLoop);
2073 ASSERT_NE(UnrolledLoop, nullptr);
2074
2075 OMPBuilder.finalize();
2076 EXPECT_FALSE(verifyModule(*M, &errs()));
2077 UnrolledLoop->assertOK();
2078
2079 PassBuilder PB;
2080 FunctionAnalysisManager FAM;
2081 PB.registerFunctionAnalyses(FAM);
2082 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
2083
2084 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
2085 EXPECT_EQ(TopLvl.size(), 1u);
2086 Loop *Outer = TopLvl.front();
2087 EXPECT_EQ(Outer->getHeader(), UnrolledLoop->getHeader());
2088 EXPECT_EQ(Outer->getLoopLatch(), UnrolledLoop->getLatch());
2089 EXPECT_EQ(Outer->getExitingBlock(), UnrolledLoop->getCond());
2090 EXPECT_EQ(Outer->getExitBlock(), UnrolledLoop->getExit());
2091
2092 EXPECT_EQ(Outer->getSubLoops().size(), 1u);
2093 Loop *Inner = Outer->getSubLoops().front();
2094
2095 EXPECT_TRUE(getBooleanLoopAttribute(Inner, "llvm.loop.unroll.enable"));
2096 EXPECT_EQ(getIntLoopAttribute(Inner, "llvm.loop.unroll.count"), 5);
2097 }
2098
2099 TEST_F(OpenMPIRBuilderTest, UnrollLoopHeuristic) {
2100 OpenMPIRBuilder OMPBuilder(*M);
2101
2102 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
2103
2104 // Unroll the loop.
2105 OMPBuilder.unrollLoopHeuristic(DL, CLI);
2106
2107 OMPBuilder.finalize();
2108 EXPECT_FALSE(verifyModule(*M, &errs()));
2109
2110 PassBuilder PB;
2111 FunctionAnalysisManager FAM;
2112 PB.registerFunctionAnalyses(FAM);
2113 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
2114
2115 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
2116 EXPECT_EQ(TopLvl.size(), 1u);
2117
2118 Loop *L = TopLvl.front();
2119 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.unroll.enable"));
2120 }
2121
2122 TEST_F(OpenMPIRBuilderTest, StaticWorkShareLoop) {
2123 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2124 OpenMPIRBuilder OMPBuilder(*M);
2125 OMPBuilder.initialize();
2126 IRBuilder<> Builder(BB);
2127 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2128
2129 Type *LCTy = Type::getInt32Ty(Ctx);
2130 Value *StartVal = ConstantInt::get(LCTy, 10);
2131 Value *StopVal = ConstantInt::get(LCTy, 52);
2132 Value *StepVal = ConstantInt::get(LCTy, 2);
2133 auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {};
2134
2135 CanonicalLoopInfo *CLI = OMPBuilder.createCanonicalLoop(
2136 Loc, LoopBodyGen, StartVal, StopVal, StepVal,
2137 /*IsSigned=*/false, /*InclusiveStop=*/false);
2138 BasicBlock *Preheader = CLI->getPreheader();
2139 BasicBlock *Body = CLI->getBody();
2140 Value *IV = CLI->getIndVar();
2141 BasicBlock *ExitBlock = CLI->getExit();
2142
2143 Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());
2144 InsertPointTy AllocaIP = Builder.saveIP();
2145
2146 OMPBuilder.applyWorkshareLoop(DL, CLI, AllocaIP, /*NeedsBarrier=*/true,
2147 OMP_SCHEDULE_Static);
2148
2149 BasicBlock *Cond = Body->getSinglePredecessor();
2150 Instruction *Cmp = &*Cond->begin();
2151 Value *TripCount = Cmp->getOperand(1);
2152
2153 auto AllocaIter = BB->begin();
2154 ASSERT_GE(std::distance(BB->begin(), BB->end()), 4);
2155 AllocaInst *PLastIter = dyn_cast<AllocaInst>(&*(AllocaIter++));
2156 AllocaInst *PLowerBound = dyn_cast<AllocaInst>(&*(AllocaIter++));
2157 AllocaInst *PUpperBound = dyn_cast<AllocaInst>(&*(AllocaIter++));
2158 AllocaInst *PStride = dyn_cast<AllocaInst>(&*(AllocaIter++));
2159 EXPECT_NE(PLastIter, nullptr);
2160 EXPECT_NE(PLowerBound, nullptr);
2161 EXPECT_NE(PUpperBound, nullptr);
2162 EXPECT_NE(PStride, nullptr);
2163
2164 auto PreheaderIter = Preheader->begin();
2165 ASSERT_GE(std::distance(Preheader->begin(), Preheader->end()), 7);
2166 StoreInst *LowerBoundStore = dyn_cast<StoreInst>(&*(PreheaderIter++));
2167 StoreInst *UpperBoundStore = dyn_cast<StoreInst>(&*(PreheaderIter++));
2168 StoreInst *StrideStore = dyn_cast<StoreInst>(&*(PreheaderIter++));
2169 ASSERT_NE(LowerBoundStore, nullptr);
2170 ASSERT_NE(UpperBoundStore, nullptr);
2171 ASSERT_NE(StrideStore, nullptr);
2172
2173 auto *OrigLowerBound =
2174 dyn_cast<ConstantInt>(LowerBoundStore->getValueOperand());
2175 auto *OrigUpperBound =
2176 dyn_cast<ConstantInt>(UpperBoundStore->getValueOperand());
2177 auto *OrigStride = dyn_cast<ConstantInt>(StrideStore->getValueOperand());
2178 ASSERT_NE(OrigLowerBound, nullptr);
2179 ASSERT_NE(OrigUpperBound, nullptr);
2180 ASSERT_NE(OrigStride, nullptr);
2181 EXPECT_EQ(OrigLowerBound->getValue(), 0);
2182 EXPECT_EQ(OrigUpperBound->getValue(), 20);
2183 EXPECT_EQ(OrigStride->getValue(), 1);
2184
2185 // Check that the loop IV is updated to account for the lower bound returned
2186 // by the OpenMP runtime call.
2187 BinaryOperator *Add = dyn_cast<BinaryOperator>(&Body->front());
2188 EXPECT_EQ(Add->getOperand(0), IV);
2189 auto *LoadedLowerBound = dyn_cast<LoadInst>(Add->getOperand(1));
2190 ASSERT_NE(LoadedLowerBound, nullptr);
2191 EXPECT_EQ(LoadedLowerBound->getPointerOperand(), PLowerBound);
2192
2193 // Check that the trip count is updated to account for the lower and upper
2194 // bounds return by the OpenMP runtime call.
2195 auto *AddOne = dyn_cast<Instruction>(TripCount);
2196 ASSERT_NE(AddOne, nullptr);
2197 ASSERT_TRUE(AddOne->isBinaryOp());
2198 auto *One = dyn_cast<ConstantInt>(AddOne->getOperand(1));
2199 ASSERT_NE(One, nullptr);
2200 EXPECT_EQ(One->getValue(), 1);
2201 auto *Difference = dyn_cast<Instruction>(AddOne->getOperand(0));
2202 ASSERT_NE(Difference, nullptr);
2203 ASSERT_TRUE(Difference->isBinaryOp());
2204 EXPECT_EQ(Difference->getOperand(1), LoadedLowerBound);
2205 auto *LoadedUpperBound = dyn_cast<LoadInst>(Difference->getOperand(0));
2206 ASSERT_NE(LoadedUpperBound, nullptr);
2207 EXPECT_EQ(LoadedUpperBound->getPointerOperand(), PUpperBound);
2208
2209 // The original loop iterator should only be used in the condition, in the
2210 // increment and in the statement that adds the lower bound to it.
2211 EXPECT_EQ(std::distance(IV->use_begin(), IV->use_end()), 3);
2212
2213 // The exit block should contain the "fini" call and the barrier call,
2214 // plus the call to obtain the thread ID.
2215 size_t NumCallsInExitBlock =
2216 count_if(*ExitBlock, [](Instruction &I) { return isa<CallInst>(I); });
2217 EXPECT_EQ(NumCallsInExitBlock, 3u);
2218 }
2219
2220 TEST_P(OpenMPIRBuilderTestWithIVBits, StaticChunkedWorkshareLoop) {
2221 unsigned IVBits = GetParam();
2222
2223 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2224 OpenMPIRBuilder OMPBuilder(*M);
2225
2226 BasicBlock *Body;
2227 CallInst *Call;
2228 CanonicalLoopInfo *CLI =
2229 buildSingleLoopFunction(DL, OMPBuilder, IVBits, &Call, &Body);
2230
2231 Instruction *OrigIndVar = CLI->getIndVar();
2232 EXPECT_EQ(Call->getOperand(1), OrigIndVar);
2233
2234 Type *LCTy = Type::getInt32Ty(Ctx);
2235 Value *ChunkSize = ConstantInt::get(LCTy, 5);
2236 InsertPointTy AllocaIP{&F->getEntryBlock(),
2237 F->getEntryBlock().getFirstInsertionPt()};
2238 OMPBuilder.applyWorkshareLoop(DL, CLI, AllocaIP, /*NeedsBarrier=*/true,
2239 OMP_SCHEDULE_Static, ChunkSize);
2240
2241 OMPBuilder.finalize();
2242 EXPECT_FALSE(verifyModule(*M, &errs()));
2243
2244 BasicBlock *Entry = &F->getEntryBlock();
2245 BasicBlock *Preheader = Entry->getSingleSuccessor();
2246
2247 BasicBlock *DispatchPreheader = Preheader->getSingleSuccessor();
2248 BasicBlock *DispatchHeader = DispatchPreheader->getSingleSuccessor();
2249 BasicBlock *DispatchCond = DispatchHeader->getSingleSuccessor();
2250 BasicBlock *DispatchBody = succ_begin(DispatchCond)[0];
2251 BasicBlock *DispatchExit = succ_begin(DispatchCond)[1];
2252 BasicBlock *DispatchAfter = DispatchExit->getSingleSuccessor();
2253 BasicBlock *Return = DispatchAfter->getSingleSuccessor();
2254
2255 BasicBlock *ChunkPreheader = DispatchBody->getSingleSuccessor();
2256 BasicBlock *ChunkHeader = ChunkPreheader->getSingleSuccessor();
2257 BasicBlock *ChunkCond = ChunkHeader->getSingleSuccessor();
2258 BasicBlock *ChunkBody = succ_begin(ChunkCond)[0];
2259 BasicBlock *ChunkExit = succ_begin(ChunkCond)[1];
2260 BasicBlock *ChunkInc = ChunkBody->getSingleSuccessor();
2261 BasicBlock *ChunkAfter = ChunkExit->getSingleSuccessor();
2262
2263 BasicBlock *DispatchInc = ChunkAfter;
2264
2265 EXPECT_EQ(ChunkBody, Body);
2266 EXPECT_EQ(ChunkInc->getSingleSuccessor(), ChunkHeader);
2267 EXPECT_EQ(DispatchInc->getSingleSuccessor(), DispatchHeader);
2268
2269 EXPECT_TRUE(isa<ReturnInst>(Return->front()));
2270
2271 Value *NewIV = Call->getOperand(1);
2272 EXPECT_EQ(NewIV->getType()->getScalarSizeInBits(), IVBits);
2273
2274 CallInst *InitCall = findSingleCall(
2275 F,
2276 (IVBits > 32) ? omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_8u
2277 : omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_4u,
2278 OMPBuilder);
2279 EXPECT_EQ(InitCall->getParent(), Preheader);
2280 EXPECT_EQ(cast<ConstantInt>(InitCall->getArgOperand(2))->getSExtValue(), 33);
2281 EXPECT_EQ(cast<ConstantInt>(InitCall->getArgOperand(7))->getSExtValue(), 1);
2282 EXPECT_EQ(cast<ConstantInt>(InitCall->getArgOperand(8))->getSExtValue(), 5);
2283
2284 CallInst *FiniCall = findSingleCall(
2285 F, omp::RuntimeFunction::OMPRTL___kmpc_for_static_fini, OMPBuilder);
2286 EXPECT_EQ(FiniCall->getParent(), DispatchExit);
2287
2288 CallInst *BarrierCall = findSingleCall(
2289 F, omp::RuntimeFunction::OMPRTL___kmpc_barrier, OMPBuilder);
2290 EXPECT_EQ(BarrierCall->getParent(), DispatchExit);
2291 }
2292
2293 INSTANTIATE_TEST_SUITE_P(IVBits, OpenMPIRBuilderTestWithIVBits,
2294 ::testing::Values(8, 16, 32, 64));
2295
2296 TEST_P(OpenMPIRBuilderTestWithParams, DynamicWorkShareLoop) {
2297 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2298 OpenMPIRBuilder OMPBuilder(*M);
2299 OMPBuilder.initialize();
2300 IRBuilder<> Builder(BB);
2301 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2302
2303 omp::OMPScheduleType SchedType = GetParam();
2304 uint32_t ChunkSize = 1;
2305 switch (SchedType & ~OMPScheduleType::ModifierMask) {
2306 case omp::OMPScheduleType::BaseDynamicChunked:
2307 case omp::OMPScheduleType::BaseGuidedChunked:
2308 ChunkSize = 7;
2309 break;
2310 case omp::OMPScheduleType::BaseAuto:
2311 case omp::OMPScheduleType::BaseRuntime:
2312 ChunkSize = 1;
2313 break;
2314 default:
2315 assert(0 && "unknown type for this test");
2316 break;
2317 }
2318
2319 Type *LCTy = Type::getInt32Ty(Ctx);
2320 Value *StartVal = ConstantInt::get(LCTy, 10);
2321 Value *StopVal = ConstantInt::get(LCTy, 52);
2322 Value *StepVal = ConstantInt::get(LCTy, 2);
2323 Value *ChunkVal =
2324 (ChunkSize == 1) ? nullptr : ConstantInt::get(LCTy, ChunkSize);
2325 auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {};
2326
2327 CanonicalLoopInfo *CLI = OMPBuilder.createCanonicalLoop(
2328 Loc, LoopBodyGen, StartVal, StopVal, StepVal,
2329 /*IsSigned=*/false, /*InclusiveStop=*/false);
2330
2331 Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());
2332 InsertPointTy AllocaIP = Builder.saveIP();
2333
2334 // Collect all the info from CLI, as it isn't usable after the call to
2335 // createDynamicWorkshareLoop.
2336 InsertPointTy AfterIP = CLI->getAfterIP();
2337 BasicBlock *Preheader = CLI->getPreheader();
2338 BasicBlock *ExitBlock = CLI->getExit();
2339 BasicBlock *LatchBlock = CLI->getLatch();
2340 Value *IV = CLI->getIndVar();
2341
2342 InsertPointTy EndIP = OMPBuilder.applyWorkshareLoop(
2343 DL, CLI, AllocaIP, /*NeedsBarrier=*/true, getSchedKind(SchedType),
2344 ChunkVal, /*Simd=*/false,
2345 (SchedType & omp::OMPScheduleType::ModifierMonotonic) ==
2346 omp::OMPScheduleType::ModifierMonotonic,
2347 (SchedType & omp::OMPScheduleType::ModifierNonmonotonic) ==
2348 omp::OMPScheduleType::ModifierNonmonotonic,
2349 /*Ordered=*/false);
2350
2351 // The returned value should be the "after" point.
2352 ASSERT_EQ(EndIP.getBlock(), AfterIP.getBlock());
2353 ASSERT_EQ(EndIP.getPoint(), AfterIP.getPoint());
2354
2355 auto AllocaIter = BB->begin();
2356 ASSERT_GE(std::distance(BB->begin(), BB->end()), 4);
2357 AllocaInst *PLastIter = dyn_cast<AllocaInst>(&*(AllocaIter++));
2358 AllocaInst *PLowerBound = dyn_cast<AllocaInst>(&*(AllocaIter++));
2359 AllocaInst *PUpperBound = dyn_cast<AllocaInst>(&*(AllocaIter++));
2360 AllocaInst *PStride = dyn_cast<AllocaInst>(&*(AllocaIter++));
2361 EXPECT_NE(PLastIter, nullptr);
2362 EXPECT_NE(PLowerBound, nullptr);
2363 EXPECT_NE(PUpperBound, nullptr);
2364 EXPECT_NE(PStride, nullptr);
2365
2366 auto PreheaderIter = Preheader->begin();
2367 ASSERT_GE(std::distance(Preheader->begin(), Preheader->end()), 6);
2368 StoreInst *LowerBoundStore = dyn_cast<StoreInst>(&*(PreheaderIter++));
2369 StoreInst *UpperBoundStore = dyn_cast<StoreInst>(&*(PreheaderIter++));
2370 StoreInst *StrideStore = dyn_cast<StoreInst>(&*(PreheaderIter++));
2371 ASSERT_NE(LowerBoundStore, nullptr);
2372 ASSERT_NE(UpperBoundStore, nullptr);
2373 ASSERT_NE(StrideStore, nullptr);
2374
2375 CallInst *ThreadIdCall = dyn_cast<CallInst>(&*(PreheaderIter++));
2376 ASSERT_NE(ThreadIdCall, nullptr);
2377 EXPECT_EQ(ThreadIdCall->getCalledFunction()->getName(),
2378 "__kmpc_global_thread_num");
2379
2380 CallInst *InitCall = dyn_cast<CallInst>(&*PreheaderIter);
2381
2382 ASSERT_NE(InitCall, nullptr);
2383 EXPECT_EQ(InitCall->getCalledFunction()->getName(),
2384 "__kmpc_dispatch_init_4u");
2385 EXPECT_EQ(InitCall->arg_size(), 7U);
2386 EXPECT_EQ(InitCall->getArgOperand(6), ConstantInt::get(LCTy, ChunkSize));
2387 ConstantInt *SchedVal = cast<ConstantInt>(InitCall->getArgOperand(2));
2388 if ((SchedType & OMPScheduleType::MonotonicityMask) ==
2389 OMPScheduleType::None) {
2390 // Implementation is allowed to add default nonmonotonicity flag
2391 EXPECT_EQ(
2392 static_cast<OMPScheduleType>(SchedVal->getValue().getZExtValue()) |
2393 OMPScheduleType::ModifierNonmonotonic,
2394 SchedType | OMPScheduleType::ModifierNonmonotonic);
2395 } else {
2396 EXPECT_EQ(static_cast<OMPScheduleType>(SchedVal->getValue().getZExtValue()),
2397 SchedType);
2398 }
2399
2400 ConstantInt *OrigLowerBound =
2401 dyn_cast<ConstantInt>(LowerBoundStore->getValueOperand());
2402 ConstantInt *OrigUpperBound =
2403 dyn_cast<ConstantInt>(UpperBoundStore->getValueOperand());
2404 ConstantInt *OrigStride =
2405 dyn_cast<ConstantInt>(StrideStore->getValueOperand());
2406 ASSERT_NE(OrigLowerBound, nullptr);
2407 ASSERT_NE(OrigUpperBound, nullptr);
2408 ASSERT_NE(OrigStride, nullptr);
2409 EXPECT_EQ(OrigLowerBound->getValue(), 1);
2410 EXPECT_EQ(OrigUpperBound->getValue(), 21);
2411 EXPECT_EQ(OrigStride->getValue(), 1);
2412
2413 CallInst *FiniCall = dyn_cast<CallInst>(
2414 &*(LatchBlock->getTerminator()->getPrevNonDebugInstruction(true)));
2415 EXPECT_EQ(FiniCall, nullptr);
2416
2417 // The original loop iterator should only be used in the condition, in the
2418 // increment and in the statement that adds the lower bound to it.
2419 EXPECT_EQ(std::distance(IV->use_begin(), IV->use_end()), 3);
2420
2421 // The exit block should contain the barrier call, plus the call to obtain
2422 // the thread ID.
2423 size_t NumCallsInExitBlock =
2424 count_if(*ExitBlock, [](Instruction &I) { return isa<CallInst>(I); });
2425 EXPECT_EQ(NumCallsInExitBlock, 2u);
2426
2427 // Add a termination to our block and check that it is internally consistent.
2428 Builder.restoreIP(EndIP);
2429 Builder.CreateRetVoid();
2430 OMPBuilder.finalize();
2431 EXPECT_FALSE(verifyModule(*M, &errs()));
2432 }
2433
2434 INSTANTIATE_TEST_SUITE_P(
2435 OpenMPWSLoopSchedulingTypes, OpenMPIRBuilderTestWithParams,
2436 ::testing::Values(omp::OMPScheduleType::UnorderedDynamicChunked,
2437 omp::OMPScheduleType::UnorderedGuidedChunked,
2438 omp::OMPScheduleType::UnorderedAuto,
2439 omp::OMPScheduleType::UnorderedRuntime,
2440 omp::OMPScheduleType::UnorderedDynamicChunked |
2441 omp::OMPScheduleType::ModifierMonotonic,
2442 omp::OMPScheduleType::UnorderedDynamicChunked |
2443 omp::OMPScheduleType::ModifierNonmonotonic,
2444 omp::OMPScheduleType::UnorderedGuidedChunked |
2445 omp::OMPScheduleType::ModifierMonotonic,
2446 omp::OMPScheduleType::UnorderedGuidedChunked |
2447 omp::OMPScheduleType::ModifierNonmonotonic,
2448 omp::OMPScheduleType::UnorderedAuto |
2449 omp::OMPScheduleType::ModifierMonotonic,
2450 omp::OMPScheduleType::UnorderedRuntime |
2451 omp::OMPScheduleType::ModifierMonotonic));
2452
2453 TEST_F(OpenMPIRBuilderTest, DynamicWorkShareLoopOrdered) {
2454 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2455 OpenMPIRBuilder OMPBuilder(*M);
2456 OMPBuilder.initialize();
2457 IRBuilder<> Builder(BB);
2458 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2459
2460 uint32_t ChunkSize = 1;
2461 Type *LCTy = Type::getInt32Ty(Ctx);
2462 Value *StartVal = ConstantInt::get(LCTy, 10);
2463 Value *StopVal = ConstantInt::get(LCTy, 52);
2464 Value *StepVal = ConstantInt::get(LCTy, 2);
2465 Value *ChunkVal = ConstantInt::get(LCTy, ChunkSize);
2466 auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {};
2467
2468 CanonicalLoopInfo *CLI = OMPBuilder.createCanonicalLoop(
2469 Loc, LoopBodyGen, StartVal, StopVal, StepVal,
2470 /*IsSigned=*/false, /*InclusiveStop=*/false);
2471
2472 Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());
2473 InsertPointTy AllocaIP = Builder.saveIP();
2474
2475 // Collect all the info from CLI, as it isn't usable after the call to
2476 // createDynamicWorkshareLoop.
2477 BasicBlock *Preheader = CLI->getPreheader();
2478 BasicBlock *ExitBlock = CLI->getExit();
2479 BasicBlock *LatchBlock = CLI->getLatch();
2480 Value *IV = CLI->getIndVar();
2481
2482 InsertPointTy EndIP = OMPBuilder.applyWorkshareLoop(
2483 DL, CLI, AllocaIP, /*NeedsBarrier=*/true, OMP_SCHEDULE_Static, ChunkVal,
2484 /*HasSimdModifier=*/false, /*HasMonotonicModifier=*/false,
2485 /*HasNonmonotonicModifier=*/false,
2486 /*HasOrderedClause=*/true);
2487
2488 // Add a termination to our block and check that it is internally consistent.
2489 Builder.restoreIP(EndIP);
2490 Builder.CreateRetVoid();
2491 OMPBuilder.finalize();
2492 EXPECT_FALSE(verifyModule(*M, &errs()));
2493
2494 CallInst *InitCall = nullptr;
2495 for (Instruction &EI : *Preheader) {
2496 Instruction *Cur = &EI;
2497 if (isa<CallInst>(Cur)) {
2498 InitCall = cast<CallInst>(Cur);
2499 if (InitCall->getCalledFunction()->getName() == "__kmpc_dispatch_init_4u")
2500 break;
2501 InitCall = nullptr;
2502 }
2503 }
2504 EXPECT_NE(InitCall, nullptr);
2505 EXPECT_EQ(InitCall->arg_size(), 7U);
2506 ConstantInt *SchedVal = cast<ConstantInt>(InitCall->getArgOperand(2));
2507 EXPECT_EQ(SchedVal->getValue(),
2508 static_cast<uint64_t>(OMPScheduleType::OrderedStaticChunked));
2509
2510 CallInst *FiniCall = dyn_cast<CallInst>(
2511 &*(LatchBlock->getTerminator()->getPrevNonDebugInstruction(true)));
2512 ASSERT_NE(FiniCall, nullptr);
2513 EXPECT_EQ(FiniCall->getCalledFunction()->getName(),
2514 "__kmpc_dispatch_fini_4u");
2515 EXPECT_EQ(FiniCall->arg_size(), 2U);
2516 EXPECT_EQ(InitCall->getArgOperand(0), FiniCall->getArgOperand(0));
2517 EXPECT_EQ(InitCall->getArgOperand(1), FiniCall->getArgOperand(1));
2518
2519 // The original loop iterator should only be used in the condition, in the
2520 // increment and in the statement that adds the lower bound to it.
2521 EXPECT_EQ(std::distance(IV->use_begin(), IV->use_end()), 3);
2522
2523 // The exit block should contain the barrier call, plus the call to obtain
2524 // the thread ID.
2525 size_t NumCallsInExitBlock =
2526 count_if(*ExitBlock, [](Instruction &I) { return isa<CallInst>(I); });
2527 EXPECT_EQ(NumCallsInExitBlock, 2u);
2528 }
2529
2530 TEST_F(OpenMPIRBuilderTest, MasterDirective) {
2531 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2532 OpenMPIRBuilder OMPBuilder(*M);
2533 OMPBuilder.initialize();
2534 F->setName("func");
2535 IRBuilder<> Builder(BB);
2536
2537 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2538
2539 AllocaInst *PrivAI = nullptr;
2540
2541 BasicBlock *EntryBB = nullptr;
2542 BasicBlock *ThenBB = nullptr;
2543
2544 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
2545 if (AllocaIP.isSet())
2546 Builder.restoreIP(AllocaIP);
2547 else
2548 Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt()));
2549 PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
2550 Builder.CreateStore(F->arg_begin(), PrivAI);
2551
2552 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
2553 llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
2554 EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
2555
2556 Builder.restoreIP(CodeGenIP);
2557
2558 // collect some info for checks later
2559 ThenBB = Builder.GetInsertBlock();
2560 EntryBB = ThenBB->getUniquePredecessor();
2561
2562 // simple instructions for body
2563 Value *PrivLoad =
2564 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
2565 Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
2566 };
2567
2568 auto FiniCB = [&](InsertPointTy IP) {
2569 BasicBlock *IPBB = IP.getBlock();
2570 EXPECT_NE(IPBB->end(), IP.getPoint());
2571 };
2572
2573 Builder.restoreIP(OMPBuilder.createMaster(Builder, BodyGenCB, FiniCB));
2574 Value *EntryBBTI = EntryBB->getTerminator();
2575 EXPECT_NE(EntryBBTI, nullptr);
2576 EXPECT_TRUE(isa<BranchInst>(EntryBBTI));
2577 BranchInst *EntryBr = cast<BranchInst>(EntryBB->getTerminator());
2578 EXPECT_TRUE(EntryBr->isConditional());
2579 EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB);
2580 BasicBlock *ExitBB = ThenBB->getUniqueSuccessor();
2581 EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB);
2582
2583 CmpInst *CondInst = cast<CmpInst>(EntryBr->getCondition());
2584 EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0)));
2585
2586 CallInst *MasterEntryCI = cast<CallInst>(CondInst->getOperand(0));
2587 EXPECT_EQ(MasterEntryCI->arg_size(), 2U);
2588 EXPECT_EQ(MasterEntryCI->getCalledFunction()->getName(), "__kmpc_master");
2589 EXPECT_TRUE(isa<GlobalVariable>(MasterEntryCI->getArgOperand(0)));
2590
2591 CallInst *MasterEndCI = nullptr;
2592 for (auto &FI : *ThenBB) {
2593 Instruction *cur = &FI;
2594 if (isa<CallInst>(cur)) {
2595 MasterEndCI = cast<CallInst>(cur);
2596 if (MasterEndCI->getCalledFunction()->getName() == "__kmpc_end_master")
2597 break;
2598 MasterEndCI = nullptr;
2599 }
2600 }
2601 EXPECT_NE(MasterEndCI, nullptr);
2602 EXPECT_EQ(MasterEndCI->arg_size(), 2U);
2603 EXPECT_TRUE(isa<GlobalVariable>(MasterEndCI->getArgOperand(0)));
2604 EXPECT_EQ(MasterEndCI->getArgOperand(1), MasterEntryCI->getArgOperand(1));
2605 }
2606
2607 TEST_F(OpenMPIRBuilderTest, MaskedDirective) {
2608 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2609 OpenMPIRBuilder OMPBuilder(*M);
2610 OMPBuilder.initialize();
2611 F->setName("func");
2612 IRBuilder<> Builder(BB);
2613
2614 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2615
2616 AllocaInst *PrivAI = nullptr;
2617
2618 BasicBlock *EntryBB = nullptr;
2619 BasicBlock *ThenBB = nullptr;
2620
2621 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
2622 if (AllocaIP.isSet())
2623 Builder.restoreIP(AllocaIP);
2624 else
2625 Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt()));
2626 PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
2627 Builder.CreateStore(F->arg_begin(), PrivAI);
2628
2629 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
2630 llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
2631 EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
2632
2633 Builder.restoreIP(CodeGenIP);
2634
2635 // collect some info for checks later
2636 ThenBB = Builder.GetInsertBlock();
2637 EntryBB = ThenBB->getUniquePredecessor();
2638
2639 // simple instructions for body
2640 Value *PrivLoad =
2641 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
2642 Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
2643 };
2644
2645 auto FiniCB = [&](InsertPointTy IP) {
2646 BasicBlock *IPBB = IP.getBlock();
2647 EXPECT_NE(IPBB->end(), IP.getPoint());
2648 };
2649
2650 Constant *Filter = ConstantInt::get(Type::getInt32Ty(M->getContext()), 0);
2651 Builder.restoreIP(
2652 OMPBuilder.createMasked(Builder, BodyGenCB, FiniCB, Filter));
2653 Value *EntryBBTI = EntryBB->getTerminator();
2654 EXPECT_NE(EntryBBTI, nullptr);
2655 EXPECT_TRUE(isa<BranchInst>(EntryBBTI));
2656 BranchInst *EntryBr = cast<BranchInst>(EntryBB->getTerminator());
2657 EXPECT_TRUE(EntryBr->isConditional());
2658 EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB);
2659 BasicBlock *ExitBB = ThenBB->getUniqueSuccessor();
2660 EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB);
2661
2662 CmpInst *CondInst = cast<CmpInst>(EntryBr->getCondition());
2663 EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0)));
2664
2665 CallInst *MaskedEntryCI = cast<CallInst>(CondInst->getOperand(0));
2666 EXPECT_EQ(MaskedEntryCI->arg_size(), 3U);
2667 EXPECT_EQ(MaskedEntryCI->getCalledFunction()->getName(), "__kmpc_masked");
2668 EXPECT_TRUE(isa<GlobalVariable>(MaskedEntryCI->getArgOperand(0)));
2669
2670 CallInst *MaskedEndCI = nullptr;
2671 for (auto &FI : *ThenBB) {
2672 Instruction *cur = &FI;
2673 if (isa<CallInst>(cur)) {
2674 MaskedEndCI = cast<CallInst>(cur);
2675 if (MaskedEndCI->getCalledFunction()->getName() == "__kmpc_end_masked")
2676 break;
2677 MaskedEndCI = nullptr;
2678 }
2679 }
2680 EXPECT_NE(MaskedEndCI, nullptr);
2681 EXPECT_EQ(MaskedEndCI->arg_size(), 2U);
2682 EXPECT_TRUE(isa<GlobalVariable>(MaskedEndCI->getArgOperand(0)));
2683 EXPECT_EQ(MaskedEndCI->getArgOperand(1), MaskedEntryCI->getArgOperand(1));
2684 }
2685
2686 TEST_F(OpenMPIRBuilderTest, CriticalDirective) {
2687 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2688 OpenMPIRBuilder OMPBuilder(*M);
2689 OMPBuilder.initialize();
2690 F->setName("func");
2691 IRBuilder<> Builder(BB);
2692
2693 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2694
2695 AllocaInst *PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
2696
2697 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
2698 // actual start for bodyCB
2699 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
2700 llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
2701 EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
2702
2703 // body begin
2704 Builder.restoreIP(CodeGenIP);
2705 Builder.CreateStore(F->arg_begin(), PrivAI);
2706 Value *PrivLoad =
2707 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
2708 Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
2709 };
2710
2711 auto FiniCB = [&](InsertPointTy IP) {
2712 BasicBlock *IPBB = IP.getBlock();
2713 EXPECT_NE(IPBB->end(), IP.getPoint());
2714 };
2715 BasicBlock *EntryBB = Builder.GetInsertBlock();
2716
2717 Builder.restoreIP(OMPBuilder.createCritical(Builder, BodyGenCB, FiniCB,
2718 "testCRT", nullptr));
2719
2720 CallInst *CriticalEntryCI = nullptr;
2721 for (auto &EI : *EntryBB) {
2722 Instruction *cur = &EI;
2723 if (isa<CallInst>(cur)) {
2724 CriticalEntryCI = cast<CallInst>(cur);
2725 if (CriticalEntryCI->getCalledFunction()->getName() == "__kmpc_critical")
2726 break;
2727 CriticalEntryCI = nullptr;
2728 }
2729 }
2730 EXPECT_NE(CriticalEntryCI, nullptr);
2731 EXPECT_EQ(CriticalEntryCI->arg_size(), 3U);
2732 EXPECT_EQ(CriticalEntryCI->getCalledFunction()->getName(), "__kmpc_critical");
2733 EXPECT_TRUE(isa<GlobalVariable>(CriticalEntryCI->getArgOperand(0)));
2734
2735 CallInst *CriticalEndCI = nullptr;
2736 for (auto &FI : *EntryBB) {
2737 Instruction *cur = &FI;
2738 if (isa<CallInst>(cur)) {
2739 CriticalEndCI = cast<CallInst>(cur);
2740 if (CriticalEndCI->getCalledFunction()->getName() ==
2741 "__kmpc_end_critical")
2742 break;
2743 CriticalEndCI = nullptr;
2744 }
2745 }
2746 EXPECT_NE(CriticalEndCI, nullptr);
2747 EXPECT_EQ(CriticalEndCI->arg_size(), 3U);
2748 EXPECT_TRUE(isa<GlobalVariable>(CriticalEndCI->getArgOperand(0)));
2749 EXPECT_EQ(CriticalEndCI->getArgOperand(1), CriticalEntryCI->getArgOperand(1));
2750 PointerType *CriticalNamePtrTy =
2751 PointerType::getUnqual(ArrayType::get(Type::getInt32Ty(Ctx), 8));
2752 EXPECT_EQ(CriticalEndCI->getArgOperand(2), CriticalEntryCI->getArgOperand(2));
2753 EXPECT_EQ(CriticalEndCI->getArgOperand(2)->getType(), CriticalNamePtrTy);
2754 }
2755
2756 TEST_F(OpenMPIRBuilderTest, OrderedDirectiveDependSource) {
2757 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2758 OpenMPIRBuilder OMPBuilder(*M);
2759 OMPBuilder.initialize();
2760 F->setName("func");
2761 IRBuilder<> Builder(BB);
2762 LLVMContext &Ctx = M->getContext();
2763
2764 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2765
2766 InsertPointTy AllocaIP(&F->getEntryBlock(),
2767 F->getEntryBlock().getFirstInsertionPt());
2768
2769 unsigned NumLoops = 2;
2770 SmallVector<Value *, 2> StoreValues;
2771 Type *LCTy = Type::getInt64Ty(Ctx);
2772 StoreValues.emplace_back(ConstantInt::get(LCTy, 1));
2773 StoreValues.emplace_back(ConstantInt::get(LCTy, 2));
2774
2775 // Test for "#omp ordered depend(source)"
2776 Builder.restoreIP(OMPBuilder.createOrderedDepend(Builder, AllocaIP, NumLoops,
2777 StoreValues, ".cnt.addr",
2778 /*IsDependSource=*/true));
2779
2780 Builder.CreateRetVoid();
2781 OMPBuilder.finalize();
2782 EXPECT_FALSE(verifyModule(*M, &errs()));
2783
2784 AllocaInst *AllocInst = dyn_cast<AllocaInst>(&BB->front());
2785 ASSERT_NE(AllocInst, nullptr);
2786 ArrayType *ArrType = dyn_cast<ArrayType>(AllocInst->getAllocatedType());
2787 EXPECT_EQ(ArrType->getNumElements(), NumLoops);
2788 EXPECT_TRUE(
2789 AllocInst->getAllocatedType()->getArrayElementType()->isIntegerTy(64));
2790
2791 Instruction *IterInst = dyn_cast<Instruction>(AllocInst);
2792 for (unsigned Iter = 0; Iter < NumLoops; Iter++) {
2793 GetElementPtrInst *DependAddrGEPIter =
2794 dyn_cast<GetElementPtrInst>(IterInst->getNextNode());
2795 ASSERT_NE(DependAddrGEPIter, nullptr);
2796 EXPECT_EQ(DependAddrGEPIter->getPointerOperand(), AllocInst);
2797 EXPECT_EQ(DependAddrGEPIter->getNumIndices(), (unsigned)2);
2798 auto *FirstIdx = dyn_cast<ConstantInt>(DependAddrGEPIter->getOperand(1));
2799 auto *SecondIdx = dyn_cast<ConstantInt>(DependAddrGEPIter->getOperand(2));
2800 ASSERT_NE(FirstIdx, nullptr);
2801 ASSERT_NE(SecondIdx, nullptr);
2802 EXPECT_EQ(FirstIdx->getValue(), 0);
2803 EXPECT_EQ(SecondIdx->getValue(), Iter);
2804 StoreInst *StoreValue =
2805 dyn_cast<StoreInst>(DependAddrGEPIter->getNextNode());
2806 ASSERT_NE(StoreValue, nullptr);
2807 EXPECT_EQ(StoreValue->getValueOperand(), StoreValues[Iter]);
2808 EXPECT_EQ(StoreValue->getPointerOperand(), DependAddrGEPIter);
2809 EXPECT_EQ(StoreValue->getAlign(), Align(8));
2810 IterInst = dyn_cast<Instruction>(StoreValue);
2811 }
2812
2813 GetElementPtrInst *DependBaseAddrGEP =
2814 dyn_cast<GetElementPtrInst>(IterInst->getNextNode());
2815 ASSERT_NE(DependBaseAddrGEP, nullptr);
2816 EXPECT_EQ(DependBaseAddrGEP->getPointerOperand(), AllocInst);
2817 EXPECT_EQ(DependBaseAddrGEP->getNumIndices(), (unsigned)2);
2818 auto *FirstIdx = dyn_cast<ConstantInt>(DependBaseAddrGEP->getOperand(1));
2819 auto *SecondIdx = dyn_cast<ConstantInt>(DependBaseAddrGEP->getOperand(2));
2820 ASSERT_NE(FirstIdx, nullptr);
2821 ASSERT_NE(SecondIdx, nullptr);
2822 EXPECT_EQ(FirstIdx->getValue(), 0);
2823 EXPECT_EQ(SecondIdx->getValue(), 0);
2824
2825 CallInst *GTID = dyn_cast<CallInst>(DependBaseAddrGEP->getNextNode());
2826 ASSERT_NE(GTID, nullptr);
2827 EXPECT_EQ(GTID->arg_size(), 1U);
2828 EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
2829 EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
2830 EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
2831
2832 CallInst *Depend = dyn_cast<CallInst>(GTID->getNextNode());
2833 ASSERT_NE(Depend, nullptr);
2834 EXPECT_EQ(Depend->arg_size(), 3U);
2835 EXPECT_EQ(Depend->getCalledFunction()->getName(), "__kmpc_doacross_post");
2836 EXPECT_TRUE(isa<GlobalVariable>(Depend->getArgOperand(0)));
2837 EXPECT_EQ(Depend->getArgOperand(1), GTID);
2838 EXPECT_EQ(Depend->getArgOperand(2), DependBaseAddrGEP);
2839 }
2840
2841 TEST_F(OpenMPIRBuilderTest, OrderedDirectiveDependSink) {
2842 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2843 OpenMPIRBuilder OMPBuilder(*M);
2844 OMPBuilder.initialize();
2845 F->setName("func");
2846 IRBuilder<> Builder(BB);
2847 LLVMContext &Ctx = M->getContext();
2848
2849 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2850
2851 InsertPointTy AllocaIP(&F->getEntryBlock(),
2852 F->getEntryBlock().getFirstInsertionPt());
2853
2854 unsigned NumLoops = 2;
2855 SmallVector<Value *, 2> StoreValues;
2856 Type *LCTy = Type::getInt64Ty(Ctx);
2857 StoreValues.emplace_back(ConstantInt::get(LCTy, 1));
2858 StoreValues.emplace_back(ConstantInt::get(LCTy, 2));
2859
2860 // Test for "#omp ordered depend(sink: vec)"
2861 Builder.restoreIP(OMPBuilder.createOrderedDepend(Builder, AllocaIP, NumLoops,
2862 StoreValues, ".cnt.addr",
2863 /*IsDependSource=*/false));
2864
2865 Builder.CreateRetVoid();
2866 OMPBuilder.finalize();
2867 EXPECT_FALSE(verifyModule(*M, &errs()));
2868
2869 AllocaInst *AllocInst = dyn_cast<AllocaInst>(&BB->front());
2870 ASSERT_NE(AllocInst, nullptr);
2871 ArrayType *ArrType = dyn_cast<ArrayType>(AllocInst->getAllocatedType());
2872 EXPECT_EQ(ArrType->getNumElements(), NumLoops);
2873 EXPECT_TRUE(
2874 AllocInst->getAllocatedType()->getArrayElementType()->isIntegerTy(64));
2875
2876 Instruction *IterInst = dyn_cast<Instruction>(AllocInst);
2877 for (unsigned Iter = 0; Iter < NumLoops; Iter++) {
2878 GetElementPtrInst *DependAddrGEPIter =
2879 dyn_cast<GetElementPtrInst>(IterInst->getNextNode());
2880 ASSERT_NE(DependAddrGEPIter, nullptr);
2881 EXPECT_EQ(DependAddrGEPIter->getPointerOperand(), AllocInst);
2882 EXPECT_EQ(DependAddrGEPIter->getNumIndices(), (unsigned)2);
2883 auto *FirstIdx = dyn_cast<ConstantInt>(DependAddrGEPIter->getOperand(1));
2884 auto *SecondIdx = dyn_cast<ConstantInt>(DependAddrGEPIter->getOperand(2));
2885 ASSERT_NE(FirstIdx, nullptr);
2886 ASSERT_NE(SecondIdx, nullptr);
2887 EXPECT_EQ(FirstIdx->getValue(), 0);
2888 EXPECT_EQ(SecondIdx->getValue(), Iter);
2889 StoreInst *StoreValue =
2890 dyn_cast<StoreInst>(DependAddrGEPIter->getNextNode());
2891 ASSERT_NE(StoreValue, nullptr);
2892 EXPECT_EQ(StoreValue->getValueOperand(), StoreValues[Iter]);
2893 EXPECT_EQ(StoreValue->getPointerOperand(), DependAddrGEPIter);
2894 EXPECT_EQ(StoreValue->getAlign(), Align(8));
2895 IterInst = dyn_cast<Instruction>(StoreValue);
2896 }
2897
2898 GetElementPtrInst *DependBaseAddrGEP =
2899 dyn_cast<GetElementPtrInst>(IterInst->getNextNode());
2900 ASSERT_NE(DependBaseAddrGEP, nullptr);
2901 EXPECT_EQ(DependBaseAddrGEP->getPointerOperand(), AllocInst);
2902 EXPECT_EQ(DependBaseAddrGEP->getNumIndices(), (unsigned)2);
2903 auto *FirstIdx = dyn_cast<ConstantInt>(DependBaseAddrGEP->getOperand(1));
2904 auto *SecondIdx = dyn_cast<ConstantInt>(DependBaseAddrGEP->getOperand(2));
2905 ASSERT_NE(FirstIdx, nullptr);
2906 ASSERT_NE(SecondIdx, nullptr);
2907 EXPECT_EQ(FirstIdx->getValue(), 0);
2908 EXPECT_EQ(SecondIdx->getValue(), 0);
2909
2910 CallInst *GTID = dyn_cast<CallInst>(DependBaseAddrGEP->getNextNode());
2911 ASSERT_NE(GTID, nullptr);
2912 EXPECT_EQ(GTID->arg_size(), 1U);
2913 EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
2914 EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
2915 EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
2916
2917 CallInst *Depend = dyn_cast<CallInst>(GTID->getNextNode());
2918 ASSERT_NE(Depend, nullptr);
2919 EXPECT_EQ(Depend->arg_size(), 3U);
2920 EXPECT_EQ(Depend->getCalledFunction()->getName(), "__kmpc_doacross_wait");
2921 EXPECT_TRUE(isa<GlobalVariable>(Depend->getArgOperand(0)));
2922 EXPECT_EQ(Depend->getArgOperand(1), GTID);
2923 EXPECT_EQ(Depend->getArgOperand(2), DependBaseAddrGEP);
2924 }
2925
2926 TEST_F(OpenMPIRBuilderTest, OrderedDirectiveThreads) {
2927 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2928 OpenMPIRBuilder OMPBuilder(*M);
2929 OMPBuilder.initialize();
2930 F->setName("func");
2931 IRBuilder<> Builder(BB);
2932
2933 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2934
2935 AllocaInst *PrivAI =
2936 Builder.CreateAlloca(F->arg_begin()->getType(), nullptr, "priv.inst");
2937
2938 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
2939 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
2940 llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
2941 EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
2942
2943 Builder.restoreIP(CodeGenIP);
2944 Builder.CreateStore(F->arg_begin(), PrivAI);
2945 Value *PrivLoad =
2946 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
2947 Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
2948 };
2949
2950 auto FiniCB = [&](InsertPointTy IP) {
2951 BasicBlock *IPBB = IP.getBlock();
2952 EXPECT_NE(IPBB->end(), IP.getPoint());
2953 };
2954
2955 // Test for "#omp ordered [threads]"
2956 BasicBlock *EntryBB = Builder.GetInsertBlock();
2957 Builder.restoreIP(
2958 OMPBuilder.createOrderedThreadsSimd(Builder, BodyGenCB, FiniCB, true));
2959
2960 Builder.CreateRetVoid();
2961 OMPBuilder.finalize();
2962 EXPECT_FALSE(verifyModule(*M, &errs()));
2963
2964 EXPECT_NE(EntryBB->getTerminator(), nullptr);
2965
2966 CallInst *OrderedEntryCI = nullptr;
2967 for (auto &EI : *EntryBB) {
2968 Instruction *Cur = &EI;
2969 if (isa<CallInst>(Cur)) {
2970 OrderedEntryCI = cast<CallInst>(Cur);
2971 if (OrderedEntryCI->getCalledFunction()->getName() == "__kmpc_ordered")
2972 break;
2973 OrderedEntryCI = nullptr;
2974 }
2975 }
2976 EXPECT_NE(OrderedEntryCI, nullptr);
2977 EXPECT_EQ(OrderedEntryCI->arg_size(), 2U);
2978 EXPECT_EQ(OrderedEntryCI->getCalledFunction()->getName(), "__kmpc_ordered");
2979 EXPECT_TRUE(isa<GlobalVariable>(OrderedEntryCI->getArgOperand(0)));
2980
2981 CallInst *OrderedEndCI = nullptr;
2982 for (auto &FI : *EntryBB) {
2983 Instruction *Cur = &FI;
2984 if (isa<CallInst>(Cur)) {
2985 OrderedEndCI = cast<CallInst>(Cur);
2986 if (OrderedEndCI->getCalledFunction()->getName() == "__kmpc_end_ordered")
2987 break;
2988 OrderedEndCI = nullptr;
2989 }
2990 }
2991 EXPECT_NE(OrderedEndCI, nullptr);
2992 EXPECT_EQ(OrderedEndCI->arg_size(), 2U);
2993 EXPECT_TRUE(isa<GlobalVariable>(OrderedEndCI->getArgOperand(0)));
2994 EXPECT_EQ(OrderedEndCI->getArgOperand(1), OrderedEntryCI->getArgOperand(1));
2995 }
2996
2997 TEST_F(OpenMPIRBuilderTest, OrderedDirectiveSimd) {
2998 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2999 OpenMPIRBuilder OMPBuilder(*M);
3000 OMPBuilder.initialize();
3001 F->setName("func");
3002 IRBuilder<> Builder(BB);
3003
3004 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3005
3006 AllocaInst *PrivAI =
3007 Builder.CreateAlloca(F->arg_begin()->getType(), nullptr, "priv.inst");
3008
3009 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
3010 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
3011 llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
3012 EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
3013
3014 Builder.restoreIP(CodeGenIP);
3015 Builder.CreateStore(F->arg_begin(), PrivAI);
3016 Value *PrivLoad =
3017 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
3018 Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
3019 };
3020
3021 auto FiniCB = [&](InsertPointTy IP) {
3022 BasicBlock *IPBB = IP.getBlock();
3023 EXPECT_NE(IPBB->end(), IP.getPoint());
3024 };
3025
3026 // Test for "#omp ordered simd"
3027 BasicBlock *EntryBB = Builder.GetInsertBlock();
3028 Builder.restoreIP(
3029 OMPBuilder.createOrderedThreadsSimd(Builder, BodyGenCB, FiniCB, false));
3030
3031 Builder.CreateRetVoid();
3032 OMPBuilder.finalize();
3033 EXPECT_FALSE(verifyModule(*M, &errs()));
3034
3035 EXPECT_NE(EntryBB->getTerminator(), nullptr);
3036
3037 CallInst *OrderedEntryCI = nullptr;
3038 for (auto &EI : *EntryBB) {
3039 Instruction *Cur = &EI;
3040 if (isa<CallInst>(Cur)) {
3041 OrderedEntryCI = cast<CallInst>(Cur);
3042 if (OrderedEntryCI->getCalledFunction()->getName() == "__kmpc_ordered")
3043 break;
3044 OrderedEntryCI = nullptr;
3045 }
3046 }
3047 EXPECT_EQ(OrderedEntryCI, nullptr);
3048
3049 CallInst *OrderedEndCI = nullptr;
3050 for (auto &FI : *EntryBB) {
3051 Instruction *Cur = &FI;
3052 if (isa<CallInst>(Cur)) {
3053 OrderedEndCI = cast<CallInst>(Cur);
3054 if (OrderedEndCI->getCalledFunction()->getName() == "__kmpc_end_ordered")
3055 break;
3056 OrderedEndCI = nullptr;
3057 }
3058 }
3059 EXPECT_EQ(OrderedEndCI, nullptr);
3060 }
3061
3062 TEST_F(OpenMPIRBuilderTest, CopyinBlocks) {
3063 OpenMPIRBuilder OMPBuilder(*M);
3064 OMPBuilder.initialize();
3065 F->setName("func");
3066 IRBuilder<> Builder(BB);
3067
3068 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3069
3070 IntegerType *Int32 = Type::getInt32Ty(M->getContext());
3071 AllocaInst *MasterAddress = Builder.CreateAlloca(Int32->getPointerTo());
3072 AllocaInst *PrivAddress = Builder.CreateAlloca(Int32->getPointerTo());
3073
3074 BasicBlock *EntryBB = BB;
3075
3076 OMPBuilder.createCopyinClauseBlocks(Builder.saveIP(), MasterAddress,
3077 PrivAddress, Int32, /*BranchtoEnd*/ true);
3078
3079 BranchInst *EntryBr = dyn_cast_or_null<BranchInst>(EntryBB->getTerminator());
3080
3081 EXPECT_NE(EntryBr, nullptr);
3082 EXPECT_TRUE(EntryBr->isConditional());
3083
3084 BasicBlock *NotMasterBB = EntryBr->getSuccessor(0);
3085 BasicBlock *CopyinEnd = EntryBr->getSuccessor(1);
3086 CmpInst *CMP = dyn_cast_or_null<CmpInst>(EntryBr->getCondition());
3087
3088 EXPECT_NE(CMP, nullptr);
3089 EXPECT_NE(NotMasterBB, nullptr);
3090 EXPECT_NE(CopyinEnd, nullptr);
3091
3092 BranchInst *NotMasterBr =
3093 dyn_cast_or_null<BranchInst>(NotMasterBB->getTerminator());
3094 EXPECT_NE(NotMasterBr, nullptr);
3095 EXPECT_FALSE(NotMasterBr->isConditional());
3096 EXPECT_EQ(CopyinEnd, NotMasterBr->getSuccessor(0));
3097 }
3098
3099 TEST_F(OpenMPIRBuilderTest, SingleDirective) {
3100 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
3101 OpenMPIRBuilder OMPBuilder(*M);
3102 OMPBuilder.initialize();
3103 F->setName("func");
3104 IRBuilder<> Builder(BB);
3105
3106 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3107
3108 AllocaInst *PrivAI = nullptr;
3109
3110 BasicBlock *EntryBB = nullptr;
3111 BasicBlock *ThenBB = nullptr;
3112
3113 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
3114 if (AllocaIP.isSet())
3115 Builder.restoreIP(AllocaIP);
3116 else
3117 Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt()));
3118 PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
3119 Builder.CreateStore(F->arg_begin(), PrivAI);
3120
3121 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
3122 llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
3123 EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
3124
3125 Builder.restoreIP(CodeGenIP);
3126
3127 // collect some info for checks later
3128 ThenBB = Builder.GetInsertBlock();
3129 EntryBB = ThenBB->getUniquePredecessor();
3130
3131 // simple instructions for body
3132 Value *PrivLoad =
3133 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
3134 Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
3135 };
3136
3137 auto FiniCB = [&](InsertPointTy IP) {
3138 BasicBlock *IPBB = IP.getBlock();
3139 EXPECT_NE(IPBB->end(), IP.getPoint());
3140 };
3141
3142 Builder.restoreIP(OMPBuilder.createSingle(
3143 Builder, BodyGenCB, FiniCB, /*IsNowait*/ false, /*DidIt*/ nullptr));
3144 Value *EntryBBTI = EntryBB->getTerminator();
3145 EXPECT_NE(EntryBBTI, nullptr);
3146 EXPECT_TRUE(isa<BranchInst>(EntryBBTI));
3147 BranchInst *EntryBr = cast<BranchInst>(EntryBB->getTerminator());
3148 EXPECT_TRUE(EntryBr->isConditional());
3149 EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB);
3150 BasicBlock *ExitBB = ThenBB->getUniqueSuccessor();
3151 EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB);
3152
3153 CmpInst *CondInst = cast<CmpInst>(EntryBr->getCondition());
3154 EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0)));
3155
3156 CallInst *SingleEntryCI = cast<CallInst>(CondInst->getOperand(0));
3157 EXPECT_EQ(SingleEntryCI->arg_size(), 2U);
3158 EXPECT_EQ(SingleEntryCI->getCalledFunction()->getName(), "__kmpc_single");
3159 EXPECT_TRUE(isa<GlobalVariable>(SingleEntryCI->getArgOperand(0)));
3160
3161 CallInst *SingleEndCI = nullptr;
3162 for (auto &FI : *ThenBB) {
3163 Instruction *cur = &FI;
3164 if (isa<CallInst>(cur)) {
3165 SingleEndCI = cast<CallInst>(cur);
3166 if (SingleEndCI->getCalledFunction()->getName() == "__kmpc_end_single")
3167 break;
3168 SingleEndCI = nullptr;
3169 }
3170 }
3171 EXPECT_NE(SingleEndCI, nullptr);
3172 EXPECT_EQ(SingleEndCI->arg_size(), 2U);
3173 EXPECT_TRUE(isa<GlobalVariable>(SingleEndCI->getArgOperand(0)));
3174 EXPECT_EQ(SingleEndCI->getArgOperand(1), SingleEntryCI->getArgOperand(1));
3175
3176 bool FoundBarrier = false;
3177 for (auto &FI : *ExitBB) {
3178 Instruction *cur = &FI;
3179 if (auto CI = dyn_cast<CallInst>(cur)) {
3180 if (CI->getCalledFunction()->getName() == "__kmpc_barrier") {
3181 FoundBarrier = true;
3182 break;
3183 }
3184 }
3185 }
3186 EXPECT_TRUE(FoundBarrier);
3187 }
3188
3189 TEST_F(OpenMPIRBuilderTest, SingleDirectiveNowait) {
3190 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
3191 OpenMPIRBuilder OMPBuilder(*M);
3192 OMPBuilder.initialize();
3193 F->setName("func");
3194 IRBuilder<> Builder(BB);
3195
3196 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3197
3198 AllocaInst *PrivAI = nullptr;
3199
3200 BasicBlock *EntryBB = nullptr;
3201 BasicBlock *ThenBB = nullptr;
3202
3203 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
3204 if (AllocaIP.isSet())
3205 Builder.restoreIP(AllocaIP);
3206 else
3207 Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt()));
3208 PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
3209 Builder.CreateStore(F->arg_begin(), PrivAI);
3210
3211 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
3212 llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
3213 EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
3214
3215 Builder.restoreIP(CodeGenIP);
3216
3217 // collect some info for checks later
3218 ThenBB = Builder.GetInsertBlock();
3219 EntryBB = ThenBB->getUniquePredecessor();
3220
3221 // simple instructions for body
3222 Value *PrivLoad =
3223 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
3224 Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
3225 };
3226
3227 auto FiniCB = [&](InsertPointTy IP) {
3228 BasicBlock *IPBB = IP.getBlock();
3229 EXPECT_NE(IPBB->end(), IP.getPoint());
3230 };
3231
3232 Builder.restoreIP(OMPBuilder.createSingle(
3233 Builder, BodyGenCB, FiniCB, /*IsNowait*/ true, /*DidIt*/ nullptr));
3234 Value *EntryBBTI = EntryBB->getTerminator();
3235 EXPECT_NE(EntryBBTI, nullptr);
3236 EXPECT_TRUE(isa<BranchInst>(EntryBBTI));
3237 BranchInst *EntryBr = cast<BranchInst>(EntryBB->getTerminator());
3238 EXPECT_TRUE(EntryBr->isConditional());
3239 EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB);
3240 BasicBlock *ExitBB = ThenBB->getUniqueSuccessor();
3241 EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB);
3242
3243 CmpInst *CondInst = cast<CmpInst>(EntryBr->getCondition());
3244 EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0)));
3245
3246 CallInst *SingleEntryCI = cast<CallInst>(CondInst->getOperand(0));
3247 EXPECT_EQ(SingleEntryCI->arg_size(), 2U);
3248 EXPECT_EQ(SingleEntryCI->getCalledFunction()->getName(), "__kmpc_single");
3249 EXPECT_TRUE(isa<GlobalVariable>(SingleEntryCI->getArgOperand(0)));
3250
3251 CallInst *SingleEndCI = nullptr;
3252 for (auto &FI : *ThenBB) {
3253 Instruction *cur = &FI;
3254 if (isa<CallInst>(cur)) {
3255 SingleEndCI = cast<CallInst>(cur);
3256 if (SingleEndCI->getCalledFunction()->getName() == "__kmpc_end_single")
3257 break;
3258 SingleEndCI = nullptr;
3259 }
3260 }
3261 EXPECT_NE(SingleEndCI, nullptr);
3262 EXPECT_EQ(SingleEndCI->arg_size(), 2U);
3263 EXPECT_TRUE(isa<GlobalVariable>(SingleEndCI->getArgOperand(0)));
3264 EXPECT_EQ(SingleEndCI->getArgOperand(1), SingleEntryCI->getArgOperand(1));
3265
3266 CallInst *ExitBarrier = nullptr;
3267 for (auto &FI : *ExitBB) {
3268 Instruction *cur = &FI;
3269 if (auto CI = dyn_cast<CallInst>(cur)) {
3270 if (CI->getCalledFunction()->getName() == "__kmpc_barrier") {
3271 ExitBarrier = CI;
3272 break;
3273 }
3274 }
3275 }
3276 EXPECT_EQ(ExitBarrier, nullptr);
3277 }
3278
3279 TEST_F(OpenMPIRBuilderTest, OMPAtomicReadFlt) {
3280 OpenMPIRBuilder OMPBuilder(*M);
3281 OMPBuilder.initialize();
3282 F->setName("func");
3283 IRBuilder<> Builder(BB);
3284
3285 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3286
3287 Type *Float32 = Type::getFloatTy(M->getContext());
3288 AllocaInst *XVal = Builder.CreateAlloca(Float32);
3289 XVal->setName("AtomicVar");
3290 AllocaInst *VVal = Builder.CreateAlloca(Float32);
3291 VVal->setName("AtomicRead");
3292 AtomicOrdering AO = AtomicOrdering::Monotonic;
3293 OpenMPIRBuilder::AtomicOpValue X = {XVal, Float32, false, false};
3294 OpenMPIRBuilder::AtomicOpValue V = {VVal, Float32, false, false};
3295
3296 Builder.restoreIP(OMPBuilder.createAtomicRead(Loc, X, V, AO));
3297
3298 IntegerType *IntCastTy =
3299 IntegerType::get(M->getContext(), Float32->getScalarSizeInBits());
3300
3301 BitCastInst *CastFrmFlt = cast<BitCastInst>(VVal->getNextNode());
3302 EXPECT_EQ(CastFrmFlt->getSrcTy(), Float32->getPointerTo());
3303 EXPECT_EQ(CastFrmFlt->getDestTy(), IntCastTy->getPointerTo());
3304 EXPECT_EQ(CastFrmFlt->getOperand(0), XVal);
3305
3306 LoadInst *AtomicLoad = cast<LoadInst>(CastFrmFlt->getNextNode());
3307 EXPECT_TRUE(AtomicLoad->isAtomic());
3308 EXPECT_EQ(AtomicLoad->getPointerOperand(), CastFrmFlt);
3309
3310 BitCastInst *CastToFlt = cast<BitCastInst>(AtomicLoad->getNextNode());
3311 EXPECT_EQ(CastToFlt->getSrcTy(), IntCastTy);
3312 EXPECT_EQ(CastToFlt->getDestTy(), Float32);
3313 EXPECT_EQ(CastToFlt->getOperand(0), AtomicLoad);
3314
3315 StoreInst *StoreofAtomic = cast<StoreInst>(CastToFlt->getNextNode());
3316 EXPECT_EQ(StoreofAtomic->getValueOperand(), CastToFlt);
3317 EXPECT_EQ(StoreofAtomic->getPointerOperand(), VVal);
3318
3319 Builder.CreateRetVoid();
3320 OMPBuilder.finalize();
3321 EXPECT_FALSE(verifyModule(*M, &errs()));
3322 }
3323
3324 TEST_F(OpenMPIRBuilderTest, OMPAtomicReadInt) {
3325 OpenMPIRBuilder OMPBuilder(*M);
3326 OMPBuilder.initialize();
3327 F->setName("func");
3328 IRBuilder<> Builder(BB);
3329
3330 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3331
3332 IntegerType *Int32 = Type::getInt32Ty(M->getContext());
3333 AllocaInst *XVal = Builder.CreateAlloca(Int32);
3334 XVal->setName("AtomicVar");
3335 AllocaInst *VVal = Builder.CreateAlloca(Int32);
3336 VVal->setName("AtomicRead");
3337 AtomicOrdering AO = AtomicOrdering::Monotonic;
3338 OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, false, false};
3339 OpenMPIRBuilder::AtomicOpValue V = {VVal, Int32, false, false};
3340
3341 BasicBlock *EntryBB = BB;
3342
3343 Builder.restoreIP(OMPBuilder.createAtomicRead(Loc, X, V, AO));
3344 LoadInst *AtomicLoad = nullptr;
3345 StoreInst *StoreofAtomic = nullptr;
3346
3347 for (Instruction &Cur : *EntryBB) {
3348 if (isa<LoadInst>(Cur)) {
3349 AtomicLoad = cast<LoadInst>(&Cur);
3350 if (AtomicLoad->getPointerOperand() == XVal)
3351 continue;
3352 AtomicLoad = nullptr;
3353 } else if (isa<StoreInst>(Cur)) {
3354 StoreofAtomic = cast<StoreInst>(&Cur);
3355 if (StoreofAtomic->getPointerOperand() == VVal)
3356 continue;
3357 StoreofAtomic = nullptr;
3358 }
3359 }
3360
3361 EXPECT_NE(AtomicLoad, nullptr);
3362 EXPECT_TRUE(AtomicLoad->isAtomic());
3363
3364 EXPECT_NE(StoreofAtomic, nullptr);
3365 EXPECT_EQ(StoreofAtomic->getValueOperand(), AtomicLoad);
3366
3367 Builder.CreateRetVoid();
3368 OMPBuilder.finalize();
3369
3370 EXPECT_FALSE(verifyModule(*M, &errs()));
3371 }
3372
3373 TEST_F(OpenMPIRBuilderTest, OMPAtomicWriteFlt) {
3374 OpenMPIRBuilder OMPBuilder(*M);
3375 OMPBuilder.initialize();
3376 F->setName("func");
3377 IRBuilder<> Builder(BB);
3378
3379 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3380
3381 LLVMContext &Ctx = M->getContext();
3382 Type *Float32 = Type::getFloatTy(Ctx);
3383 AllocaInst *XVal = Builder.CreateAlloca(Float32);
3384 XVal->setName("AtomicVar");
3385 OpenMPIRBuilder::AtomicOpValue X = {XVal, Float32, false, false};
3386 AtomicOrdering AO = AtomicOrdering::Monotonic;
3387 Constant *ValToWrite = ConstantFP::get(Float32, 1.0);
3388
3389 Builder.restoreIP(OMPBuilder.createAtomicWrite(Loc, X, ValToWrite, AO));
3390
3391 IntegerType *IntCastTy =
3392 IntegerType::get(M->getContext(), Float32->getScalarSizeInBits());
3393
3394 BitCastInst *CastFrmFlt = cast<BitCastInst>(XVal->getNextNode());
3395 EXPECT_EQ(CastFrmFlt->getSrcTy(), Float32->getPointerTo());
3396 EXPECT_EQ(CastFrmFlt->getDestTy(), IntCastTy->getPointerTo());
3397 EXPECT_EQ(CastFrmFlt->getOperand(0), XVal);
3398
3399 Value *ExprCast = Builder.CreateBitCast(ValToWrite, IntCastTy);
3400
3401 StoreInst *StoreofAtomic = cast<StoreInst>(CastFrmFlt->getNextNode());
3402 EXPECT_EQ(StoreofAtomic->getValueOperand(), ExprCast);
3403 EXPECT_EQ(StoreofAtomic->getPointerOperand(), CastFrmFlt);
3404 EXPECT_TRUE(StoreofAtomic->isAtomic());
3405
3406 Builder.CreateRetVoid();
3407 OMPBuilder.finalize();
3408 EXPECT_FALSE(verifyModule(*M, &errs()));
3409 }
3410
3411 TEST_F(OpenMPIRBuilderTest, OMPAtomicWriteInt) {
3412 OpenMPIRBuilder OMPBuilder(*M);
3413 OMPBuilder.initialize();
3414 F->setName("func");
3415 IRBuilder<> Builder(BB);
3416
3417 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3418
3419 LLVMContext &Ctx = M->getContext();
3420 IntegerType *Int32 = Type::getInt32Ty(Ctx);
3421 AllocaInst *XVal = Builder.CreateAlloca(Int32);
3422 XVal->setName("AtomicVar");
3423 OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, false, false};
3424 AtomicOrdering AO = AtomicOrdering::Monotonic;
3425 ConstantInt *ValToWrite = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3426
3427 BasicBlock *EntryBB = BB;
3428
3429 Builder.restoreIP(OMPBuilder.createAtomicWrite(Loc, X, ValToWrite, AO));
3430
3431 StoreInst *StoreofAtomic = nullptr;
3432
3433 for (Instruction &Cur : *EntryBB) {
3434 if (isa<StoreInst>(Cur)) {
3435 StoreofAtomic = cast<StoreInst>(&Cur);
3436 if (StoreofAtomic->getPointerOperand() == XVal)
3437 continue;
3438 StoreofAtomic = nullptr;
3439 }
3440 }
3441
3442 EXPECT_NE(StoreofAtomic, nullptr);
3443 EXPECT_TRUE(StoreofAtomic->isAtomic());
3444 EXPECT_EQ(StoreofAtomic->getValueOperand(), ValToWrite);
3445
3446 Builder.CreateRetVoid();
3447 OMPBuilder.finalize();
3448 EXPECT_FALSE(verifyModule(*M, &errs()));
3449 }
3450
3451 TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdate) {
3452 OpenMPIRBuilder OMPBuilder(*M);
3453 OMPBuilder.initialize();
3454 F->setName("func");
3455 IRBuilder<> Builder(BB);
3456
3457 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3458
3459 IntegerType *Int32 = Type::getInt32Ty(M->getContext());
3460 AllocaInst *XVal = Builder.CreateAlloca(Int32);
3461 XVal->setName("AtomicVar");
3462 Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0U), XVal);
3463 OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, false, false};
3464 AtomicOrdering AO = AtomicOrdering::Monotonic;
3465 ConstantInt *ConstVal = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3466 Value *Expr = nullptr;
3467 AtomicRMWInst::BinOp RMWOp = AtomicRMWInst::Sub;
3468 bool IsXLHSInRHSPart = false;
3469
3470 BasicBlock *EntryBB = BB;
3471 OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,
3472 EntryBB->getFirstInsertionPt());
3473 Value *Sub = nullptr;
3474
3475 auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) {
3476 Sub = IRB.CreateSub(ConstVal, Atomic);
3477 return Sub;
3478 };
3479 Builder.restoreIP(OMPBuilder.createAtomicUpdate(
3480 Builder, AllocaIP, X, Expr, AO, RMWOp, UpdateOp, IsXLHSInRHSPart));
3481 BasicBlock *ContBB = EntryBB->getSingleSuccessor();
3482 BranchInst *ContTI = dyn_cast<BranchInst>(ContBB->getTerminator());
3483 EXPECT_NE(ContTI, nullptr);
3484 BasicBlock *EndBB = ContTI->getSuccessor(0);
3485 EXPECT_TRUE(ContTI->isConditional());
3486 EXPECT_EQ(ContTI->getSuccessor(1), ContBB);
3487 EXPECT_NE(EndBB, nullptr);
3488
3489 PHINode *Phi = dyn_cast<PHINode>(&ContBB->front());
3490 EXPECT_NE(Phi, nullptr);
3491 EXPECT_EQ(Phi->getNumIncomingValues(), 2U);
3492 EXPECT_EQ(Phi->getIncomingBlock(0), EntryBB);
3493 EXPECT_EQ(Phi->getIncomingBlock(1), ContBB);
3494
3495 EXPECT_EQ(Sub->getNumUses(), 1U);
3496 StoreInst *St = dyn_cast<StoreInst>(Sub->user_back());
3497 AllocaInst *UpdateTemp = dyn_cast<AllocaInst>(St->getPointerOperand());
3498
3499 ExtractValueInst *ExVI1 =
3500 dyn_cast<ExtractValueInst>(Phi->getIncomingValueForBlock(ContBB));
3501 EXPECT_NE(ExVI1, nullptr);
3502 AtomicCmpXchgInst *CmpExchg =
3503 dyn_cast<AtomicCmpXchgInst>(ExVI1->getAggregateOperand());
3504 EXPECT_NE(CmpExchg, nullptr);
3505 EXPECT_EQ(CmpExchg->getPointerOperand(), XVal);
3506 EXPECT_EQ(CmpExchg->getCompareOperand(), Phi);
3507 EXPECT_EQ(CmpExchg->getSuccessOrdering(), AtomicOrdering::Monotonic);
3508
3509 LoadInst *Ld = dyn_cast<LoadInst>(CmpExchg->getNewValOperand());
3510 EXPECT_NE(Ld, nullptr);
3511 EXPECT_EQ(UpdateTemp, Ld->getPointerOperand());
3512
3513 Builder.CreateRetVoid();
3514 OMPBuilder.finalize();
3515 EXPECT_FALSE(verifyModule(*M, &errs()));
3516 }
3517
3518 TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdateFloat) {
3519 OpenMPIRBuilder OMPBuilder(*M);
3520 OMPBuilder.initialize();
3521 F->setName("func");
3522 IRBuilder<> Builder(BB);
3523
3524 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3525
3526 Type *FloatTy = Type::getFloatTy(M->getContext());
3527 AllocaInst *XVal = Builder.CreateAlloca(FloatTy);
3528 XVal->setName("AtomicVar");
3529 Builder.CreateStore(ConstantFP::get(Type::getFloatTy(Ctx), 0.0), XVal);
3530 OpenMPIRBuilder::AtomicOpValue X = {XVal, FloatTy, false, false};
3531 AtomicOrdering AO = AtomicOrdering::Monotonic;
3532 Constant *ConstVal = ConstantFP::get(Type::getFloatTy(Ctx), 1.0);
3533 Value *Expr = nullptr;
3534 AtomicRMWInst::BinOp RMWOp = AtomicRMWInst::FSub;
3535 bool IsXLHSInRHSPart = false;
3536
3537 BasicBlock *EntryBB = BB;
3538 OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,
3539 EntryBB->getFirstInsertionPt());
3540 Value *Sub = nullptr;
3541
3542 auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) {
3543 Sub = IRB.CreateFSub(ConstVal, Atomic);
3544 return Sub;
3545 };
3546 Builder.restoreIP(OMPBuilder.createAtomicUpdate(
3547 Builder, AllocaIP, X, Expr, AO, RMWOp, UpdateOp, IsXLHSInRHSPart));
3548 BasicBlock *ContBB = EntryBB->getSingleSuccessor();
3549 BranchInst *ContTI = dyn_cast<BranchInst>(ContBB->getTerminator());
3550 EXPECT_NE(ContTI, nullptr);
3551 BasicBlock *EndBB = ContTI->getSuccessor(0);
3552 EXPECT_TRUE(ContTI->isConditional());
3553 EXPECT_EQ(ContTI->getSuccessor(1), ContBB);
3554 EXPECT_NE(EndBB, nullptr);
3555
3556 PHINode *Phi = dyn_cast<PHINode>(&ContBB->front());
3557 EXPECT_NE(Phi, nullptr);
3558 EXPECT_EQ(Phi->getNumIncomingValues(), 2U);
3559 EXPECT_EQ(Phi->getIncomingBlock(0), EntryBB);
3560 EXPECT_EQ(Phi->getIncomingBlock(1), ContBB);
3561
3562 EXPECT_EQ(Sub->getNumUses(), 1U);
3563 StoreInst *St = dyn_cast<StoreInst>(Sub->user_back());
3564 AllocaInst *UpdateTemp = dyn_cast<AllocaInst>(St->getPointerOperand());
3565
3566 ExtractValueInst *ExVI1 =
3567 dyn_cast<ExtractValueInst>(Phi->getIncomingValueForBlock(ContBB));
3568 EXPECT_NE(ExVI1, nullptr);
3569 AtomicCmpXchgInst *CmpExchg =
3570 dyn_cast<AtomicCmpXchgInst>(ExVI1->getAggregateOperand());
3571 EXPECT_NE(CmpExchg, nullptr);
3572 BitCastInst *BitCastNew =
3573 dyn_cast<BitCastInst>(CmpExchg->getPointerOperand());
3574 EXPECT_NE(BitCastNew, nullptr);
3575 EXPECT_EQ(BitCastNew->getOperand(0), XVal);
3576 EXPECT_EQ(CmpExchg->getCompareOperand(), Phi);
3577 EXPECT_EQ(CmpExchg->getSuccessOrdering(), AtomicOrdering::Monotonic);
3578
3579 LoadInst *Ld = dyn_cast<LoadInst>(CmpExchg->getNewValOperand());
3580 EXPECT_NE(Ld, nullptr);
3581 BitCastInst *BitCastOld = dyn_cast<BitCastInst>(Ld->getPointerOperand());
3582 EXPECT_NE(BitCastOld, nullptr);
3583 EXPECT_EQ(UpdateTemp, BitCastOld->getOperand(0));
3584
3585 Builder.CreateRetVoid();
3586 OMPBuilder.finalize();
3587 EXPECT_FALSE(verifyModule(*M, &errs()));
3588 }
3589
3590 TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdateIntr) {
3591 OpenMPIRBuilder OMPBuilder(*M);
3592 OMPBuilder.initialize();
3593 F->setName("func");
3594 IRBuilder<> Builder(BB);
3595
3596 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3597
3598 Type *IntTy = Type::getInt32Ty(M->getContext());
3599 AllocaInst *XVal = Builder.CreateAlloca(IntTy);
3600 XVal->setName("AtomicVar");
3601 Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0), XVal);
3602 OpenMPIRBuilder::AtomicOpValue X = {XVal, IntTy, false, false};
3603 AtomicOrdering AO = AtomicOrdering::Monotonic;
3604 Constant *ConstVal = ConstantInt::get(Type::getInt32Ty(Ctx), 1);
3605 Value *Expr = ConstantInt::get(Type::getInt32Ty(Ctx), 1);
3606 AtomicRMWInst::BinOp RMWOp = AtomicRMWInst::UMax;
3607 bool IsXLHSInRHSPart = false;
3608
3609 BasicBlock *EntryBB = BB;
3610 OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,
3611 EntryBB->getFirstInsertionPt());
3612 Value *Sub = nullptr;
3613
3614 auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) {
3615 Sub = IRB.CreateSub(ConstVal, Atomic);
3616 return Sub;
3617 };
3618 Builder.restoreIP(OMPBuilder.createAtomicUpdate(
3619 Builder, AllocaIP, X, Expr, AO, RMWOp, UpdateOp, IsXLHSInRHSPart));
3620 BasicBlock *ContBB = EntryBB->getSingleSuccessor();
3621 BranchInst *ContTI = dyn_cast<BranchInst>(ContBB->getTerminator());
3622 EXPECT_NE(ContTI, nullptr);
3623 BasicBlock *EndBB = ContTI->getSuccessor(0);
3624 EXPECT_TRUE(ContTI->isConditional());
3625 EXPECT_EQ(ContTI->getSuccessor(1), ContBB);
3626 EXPECT_NE(EndBB, nullptr);
3627
3628 PHINode *Phi = dyn_cast<PHINode>(&ContBB->front());
3629 EXPECT_NE(Phi, nullptr);
3630 EXPECT_EQ(Phi->getNumIncomingValues(), 2U);
3631 EXPECT_EQ(Phi->getIncomingBlock(0), EntryBB);
3632 EXPECT_EQ(Phi->getIncomingBlock(1), ContBB);
3633
3634 EXPECT_EQ(Sub->getNumUses(), 1U);
3635 StoreInst *St = dyn_cast<StoreInst>(Sub->user_back());
3636 AllocaInst *UpdateTemp = dyn_cast<AllocaInst>(St->getPointerOperand());
3637
3638 ExtractValueInst *ExVI1 =
3639 dyn_cast<ExtractValueInst>(Phi->getIncomingValueForBlock(ContBB));
3640 EXPECT_NE(ExVI1, nullptr);
3641 AtomicCmpXchgInst *CmpExchg =
3642 dyn_cast<AtomicCmpXchgInst>(ExVI1->getAggregateOperand());
3643 EXPECT_NE(CmpExchg, nullptr);
3644 EXPECT_EQ(CmpExchg->getPointerOperand(), XVal);
3645 EXPECT_EQ(CmpExchg->getCompareOperand(), Phi);
3646 EXPECT_EQ(CmpExchg->getSuccessOrdering(), AtomicOrdering::Monotonic);
3647
3648 LoadInst *Ld = dyn_cast<LoadInst>(CmpExchg->getNewValOperand());
3649 EXPECT_NE(Ld, nullptr);
3650 EXPECT_EQ(UpdateTemp, Ld->getPointerOperand());
3651
3652 Builder.CreateRetVoid();
3653 OMPBuilder.finalize();
3654 EXPECT_FALSE(verifyModule(*M, &errs()));
3655 }
3656
3657 TEST_F(OpenMPIRBuilderTest, OMPAtomicCapture) {
3658 OpenMPIRBuilder OMPBuilder(*M);
3659 OMPBuilder.initialize();
3660 F->setName("func");
3661 IRBuilder<> Builder(BB);
3662
3663 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3664
3665 LLVMContext &Ctx = M->getContext();
3666 IntegerType *Int32 = Type::getInt32Ty(Ctx);
3667 AllocaInst *XVal = Builder.CreateAlloca(Int32);
3668 XVal->setName("AtomicVar");
3669 AllocaInst *VVal = Builder.CreateAlloca(Int32);
3670 VVal->setName("AtomicCapTar");
3671 StoreInst *Init =
3672 Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0U), XVal);
3673
3674 OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, false, false};
3675 OpenMPIRBuilder::AtomicOpValue V = {VVal, Int32, false, false};
3676 AtomicOrdering AO = AtomicOrdering::Monotonic;
3677 ConstantInt *Expr = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3678 AtomicRMWInst::BinOp RMWOp = AtomicRMWInst::Add;
3679 bool IsXLHSInRHSPart = true;
3680 bool IsPostfixUpdate = true;
3681 bool UpdateExpr = true;
3682
3683 BasicBlock *EntryBB = BB;
3684 OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,
3685 EntryBB->getFirstInsertionPt());
3686
3687 // integer update - not used
3688 auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) { return nullptr; };
3689
3690 Builder.restoreIP(OMPBuilder.createAtomicCapture(
3691 Builder, AllocaIP, X, V, Expr, AO, RMWOp, UpdateOp, UpdateExpr,
3692 IsPostfixUpdate, IsXLHSInRHSPart));
3693 EXPECT_EQ(EntryBB->getParent()->size(), 1U);
3694 AtomicRMWInst *ARWM = dyn_cast<AtomicRMWInst>(Init->getNextNode());
3695 EXPECT_NE(ARWM, nullptr);
3696 EXPECT_EQ(ARWM->getPointerOperand(), XVal);
3697 EXPECT_EQ(ARWM->getOperation(), RMWOp);
3698 StoreInst *St = dyn_cast<StoreInst>(ARWM->user_back());
3699 EXPECT_NE(St, nullptr);
3700 EXPECT_EQ(St->getPointerOperand(), VVal);
3701
3702 Builder.CreateRetVoid();
3703 OMPBuilder.finalize();
3704 EXPECT_FALSE(verifyModule(*M, &errs()));
3705 }
3706
3707 TEST_F(OpenMPIRBuilderTest, OMPAtomicCompare) {
3708 OpenMPIRBuilder OMPBuilder(*M);
3709 OMPBuilder.initialize();
3710 F->setName("func");
3711 IRBuilder<> Builder(BB);
3712
3713 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3714
3715 LLVMContext &Ctx = M->getContext();
3716 IntegerType *Int32 = Type::getInt32Ty(Ctx);
3717 AllocaInst *XVal = Builder.CreateAlloca(Int32);
3718 XVal->setName("x");
3719 StoreInst *Init =
3720 Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0U), XVal);
3721
3722 OpenMPIRBuilder::AtomicOpValue XSigned = {XVal, Int32, true, false};
3723 OpenMPIRBuilder::AtomicOpValue XUnsigned = {XVal, Int32, false, false};
3724 // V and R are not used in atomic compare
3725 OpenMPIRBuilder::AtomicOpValue V = {nullptr, nullptr, false, false};
3726 OpenMPIRBuilder::AtomicOpValue R = {nullptr, nullptr, false, false};
3727 AtomicOrdering AO = AtomicOrdering::Monotonic;
3728 ConstantInt *Expr = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3729 ConstantInt *D = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3730 OMPAtomicCompareOp OpMax = OMPAtomicCompareOp::MAX;
3731 OMPAtomicCompareOp OpEQ = OMPAtomicCompareOp::EQ;
3732
3733 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3734 Builder, XSigned, V, R, Expr, nullptr, AO, OpMax, true, false, false));
3735 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3736 Builder, XUnsigned, V, R, Expr, nullptr, AO, OpMax, false, false, false));
3737 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3738 Builder, XSigned, V, R, Expr, D, AO, OpEQ, true, false, false));
3739
3740 BasicBlock *EntryBB = BB;
3741 EXPECT_EQ(EntryBB->getParent()->size(), 1U);
3742 EXPECT_EQ(EntryBB->size(), 5U);
3743
3744 AtomicRMWInst *ARWM1 = dyn_cast<AtomicRMWInst>(Init->getNextNode());
3745 EXPECT_NE(ARWM1, nullptr);
3746 EXPECT_EQ(ARWM1->getPointerOperand(), XVal);
3747 EXPECT_EQ(ARWM1->getValOperand(), Expr);
3748 EXPECT_EQ(ARWM1->getOperation(), AtomicRMWInst::Min);
3749
3750 AtomicRMWInst *ARWM2 = dyn_cast<AtomicRMWInst>(ARWM1->getNextNode());
3751 EXPECT_NE(ARWM2, nullptr);
3752 EXPECT_EQ(ARWM2->getPointerOperand(), XVal);
3753 EXPECT_EQ(ARWM2->getValOperand(), Expr);
3754 EXPECT_EQ(ARWM2->getOperation(), AtomicRMWInst::UMax);
3755
3756 AtomicCmpXchgInst *AXCHG = dyn_cast<AtomicCmpXchgInst>(ARWM2->getNextNode());
3757 EXPECT_NE(AXCHG, nullptr);
3758 EXPECT_EQ(AXCHG->getPointerOperand(), XVal);
3759 EXPECT_EQ(AXCHG->getCompareOperand(), Expr);
3760 EXPECT_EQ(AXCHG->getNewValOperand(), D);
3761
3762 Builder.CreateRetVoid();
3763 OMPBuilder.finalize();
3764 EXPECT_FALSE(verifyModule(*M, &errs()));
3765 }
3766
3767 TEST_F(OpenMPIRBuilderTest, OMPAtomicCompareCapture) {
3768 OpenMPIRBuilder OMPBuilder(*M);
3769 OMPBuilder.initialize();
3770 F->setName("func");
3771 IRBuilder<> Builder(BB);
3772
3773 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3774
3775 LLVMContext &Ctx = M->getContext();
3776 IntegerType *Int32 = Type::getInt32Ty(Ctx);
3777 AllocaInst *XVal = Builder.CreateAlloca(Int32);
3778 XVal->setName("x");
3779 AllocaInst *VVal = Builder.CreateAlloca(Int32);
3780 VVal->setName("v");
3781 AllocaInst *RVal = Builder.CreateAlloca(Int32);
3782 RVal->setName("r");
3783
3784 StoreInst *Init =
3785 Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0U), XVal);
3786
3787 OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, true, false};
3788 OpenMPIRBuilder::AtomicOpValue V = {VVal, Int32, false, false};
3789 OpenMPIRBuilder::AtomicOpValue NoV = {nullptr, nullptr, false, false};
3790 OpenMPIRBuilder::AtomicOpValue R = {RVal, Int32, false, false};
3791 OpenMPIRBuilder::AtomicOpValue NoR = {nullptr, nullptr, false, false};
3792
3793 AtomicOrdering AO = AtomicOrdering::Monotonic;
3794 ConstantInt *Expr = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3795 ConstantInt *D = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3796 OMPAtomicCompareOp OpMax = OMPAtomicCompareOp::MAX;
3797 OMPAtomicCompareOp OpEQ = OMPAtomicCompareOp::EQ;
3798
3799 // { cond-update-stmt v = x; }
3800 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3801 Builder, X, V, NoR, Expr, D, AO, OpEQ, /* IsXBinopExpr */ true,
3802 /* IsPostfixUpdate */ false,
3803 /* IsFailOnly */ false));
3804 // { v = x; cond-update-stmt }
3805 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3806 Builder, X, V, NoR, Expr, D, AO, OpEQ, /* IsXBinopExpr */ true,
3807 /* IsPostfixUpdate */ true,
3808 /* IsFailOnly */ false));
3809 // if(x == e) { x = d; } else { v = x; }
3810 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3811 Builder, X, V, NoR, Expr, D, AO, OpEQ, /* IsXBinopExpr */ true,
3812 /* IsPostfixUpdate */ false,
3813 /* IsFailOnly */ true));
3814 // { r = x == e; if(r) { x = d; } }
3815 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3816 Builder, X, NoV, R, Expr, D, AO, OpEQ, /* IsXBinopExpr */ true,
3817 /* IsPostfixUpdate */ false,
3818 /* IsFailOnly */ false));
3819 // { r = x == e; if(r) { x = d; } else { v = x; } }
3820 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3821 Builder, X, V, R, Expr, D, AO, OpEQ, /* IsXBinopExpr */ true,
3822 /* IsPostfixUpdate */ false,
3823 /* IsFailOnly */ true));
3824
3825 // { v = x; cond-update-stmt }
3826 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3827 Builder, X, V, NoR, Expr, nullptr, AO, OpMax, /* IsXBinopExpr */ true,
3828 /* IsPostfixUpdate */ true,
3829 /* IsFailOnly */ false));
3830 // { cond-update-stmt v = x; }
3831 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3832 Builder, X, V, NoR, Expr, nullptr, AO, OpMax, /* IsXBinopExpr */ false,
3833 /* IsPostfixUpdate */ false,
3834 /* IsFailOnly */ false));
3835
3836 BasicBlock *EntryBB = BB;
3837 EXPECT_EQ(EntryBB->getParent()->size(), 5U);
3838 BasicBlock *Cont1 = dyn_cast<BasicBlock>(EntryBB->getNextNode());
3839 EXPECT_NE(Cont1, nullptr);
3840 BasicBlock *Exit1 = dyn_cast<BasicBlock>(Cont1->getNextNode());
3841 EXPECT_NE(Exit1, nullptr);
3842 BasicBlock *Cont2 = dyn_cast<BasicBlock>(Exit1->getNextNode());
3843 EXPECT_NE(Cont2, nullptr);
3844 BasicBlock *Exit2 = dyn_cast<BasicBlock>(Cont2->getNextNode());
3845 EXPECT_NE(Exit2, nullptr);
3846
3847 AtomicCmpXchgInst *CmpXchg1 =
3848 dyn_cast<AtomicCmpXchgInst>(Init->getNextNode());
3849 EXPECT_NE(CmpXchg1, nullptr);
3850 EXPECT_EQ(CmpXchg1->getPointerOperand(), XVal);
3851 EXPECT_EQ(CmpXchg1->getCompareOperand(), Expr);
3852 EXPECT_EQ(CmpXchg1->getNewValOperand(), D);
3853 ExtractValueInst *ExtVal1 =
3854 dyn_cast<ExtractValueInst>(CmpXchg1->getNextNode());
3855 EXPECT_NE(ExtVal1, nullptr);
3856 EXPECT_EQ(ExtVal1->getAggregateOperand(), CmpXchg1);
3857 EXPECT_EQ(ExtVal1->getIndices(), ArrayRef<unsigned int>(0U));
3858 ExtractValueInst *ExtVal2 =
3859 dyn_cast<ExtractValueInst>(ExtVal1->getNextNode());
3860 EXPECT_NE(ExtVal2, nullptr);
3861 EXPECT_EQ(ExtVal2->getAggregateOperand(), CmpXchg1);
3862 EXPECT_EQ(ExtVal2->getIndices(), ArrayRef<unsigned int>(1U));
3863 SelectInst *Sel1 = dyn_cast<SelectInst>(ExtVal2->getNextNode());
3864 EXPECT_NE(Sel1, nullptr);
3865 EXPECT_EQ(Sel1->getCondition(), ExtVal2);
3866 EXPECT_EQ(Sel1->getTrueValue(), Expr);
3867 EXPECT_EQ(Sel1->getFalseValue(), ExtVal1);
3868 StoreInst *Store1 = dyn_cast<StoreInst>(Sel1->getNextNode());
3869 EXPECT_NE(Store1, nullptr);
3870 EXPECT_EQ(Store1->getPointerOperand(), VVal);
3871 EXPECT_EQ(Store1->getValueOperand(), Sel1);
3872
3873 AtomicCmpXchgInst *CmpXchg2 =
3874 dyn_cast<AtomicCmpXchgInst>(Store1->getNextNode());
3875 EXPECT_NE(CmpXchg2, nullptr);
3876 EXPECT_EQ(CmpXchg2->getPointerOperand(), XVal);
3877 EXPECT_EQ(CmpXchg2->getCompareOperand(), Expr);
3878 EXPECT_EQ(CmpXchg2->getNewValOperand(), D);
3879 ExtractValueInst *ExtVal3 =
3880 dyn_cast<ExtractValueInst>(CmpXchg2->getNextNode());
3881 EXPECT_NE(ExtVal3, nullptr);
3882 EXPECT_EQ(ExtVal3->getAggregateOperand(), CmpXchg2);
3883 EXPECT_EQ(ExtVal3->getIndices(), ArrayRef<unsigned int>(0U));
3884 StoreInst *Store2 = dyn_cast<StoreInst>(ExtVal3->getNextNode());
3885 EXPECT_NE(Store2, nullptr);
3886 EXPECT_EQ(Store2->getPointerOperand(), VVal);
3887 EXPECT_EQ(Store2->getValueOperand(), ExtVal3);
3888
3889 AtomicCmpXchgInst *CmpXchg3 =
3890 dyn_cast<AtomicCmpXchgInst>(Store2->getNextNode());
3891 EXPECT_NE(CmpXchg3, nullptr);
3892 EXPECT_EQ(CmpXchg3->getPointerOperand(), XVal);
3893 EXPECT_EQ(CmpXchg3->getCompareOperand(), Expr);
3894 EXPECT_EQ(CmpXchg3->getNewValOperand(), D);
3895 ExtractValueInst *ExtVal4 =
3896 dyn_cast<ExtractValueInst>(CmpXchg3->getNextNode());
3897 EXPECT_NE(ExtVal4, nullptr);
3898 EXPECT_EQ(ExtVal4->getAggregateOperand(), CmpXchg3);
3899 EXPECT_EQ(ExtVal4->getIndices(), ArrayRef<unsigned int>(0U));
3900 ExtractValueInst *ExtVal5 =
3901 dyn_cast<ExtractValueInst>(ExtVal4->getNextNode());
3902 EXPECT_NE(ExtVal5, nullptr);
3903 EXPECT_EQ(ExtVal5->getAggregateOperand(), CmpXchg3);
3904 EXPECT_EQ(ExtVal5->getIndices(), ArrayRef<unsigned int>(1U));
3905 BranchInst *Br1 = dyn_cast<BranchInst>(ExtVal5->getNextNode());
3906 EXPECT_NE(Br1, nullptr);
3907 EXPECT_EQ(Br1->isConditional(), true);
3908 EXPECT_EQ(Br1->getCondition(), ExtVal5);
3909 EXPECT_EQ(Br1->getSuccessor(0), Exit1);
3910 EXPECT_EQ(Br1->getSuccessor(1), Cont1);
3911
3912 StoreInst *Store3 = dyn_cast<StoreInst>(&Cont1->front());
3913 EXPECT_NE(Store3, nullptr);
3914 EXPECT_EQ(Store3->getPointerOperand(), VVal);
3915 EXPECT_EQ(Store3->getValueOperand(), ExtVal4);
3916 BranchInst *Br2 = dyn_cast<BranchInst>(Store3->getNextNode());
3917 EXPECT_NE(Br2, nullptr);
3918 EXPECT_EQ(Br2->isUnconditional(), true);
3919 EXPECT_EQ(Br2->getSuccessor(0), Exit1);
3920
3921 AtomicCmpXchgInst *CmpXchg4 = dyn_cast<AtomicCmpXchgInst>(&Exit1->front());
3922 EXPECT_NE(CmpXchg4, nullptr);
3923 EXPECT_EQ(CmpXchg4->getPointerOperand(), XVal);
3924 EXPECT_EQ(CmpXchg4->getCompareOperand(), Expr);
3925 EXPECT_EQ(CmpXchg4->getNewValOperand(), D);
3926 ExtractValueInst *ExtVal6 =
3927 dyn_cast<ExtractValueInst>(CmpXchg4->getNextNode());
3928 EXPECT_NE(ExtVal6, nullptr);
3929 EXPECT_EQ(ExtVal6->getAggregateOperand(), CmpXchg4);
3930 EXPECT_EQ(ExtVal6->getIndices(), ArrayRef<unsigned int>(1U));
3931 ZExtInst *ZExt1 = dyn_cast<ZExtInst>(ExtVal6->getNextNode());
3932 EXPECT_NE(ZExt1, nullptr);
3933 EXPECT_EQ(ZExt1->getDestTy(), Int32);
3934 StoreInst *Store4 = dyn_cast<StoreInst>(ZExt1->getNextNode());
3935 EXPECT_NE(Store4, nullptr);
3936 EXPECT_EQ(Store4->getPointerOperand(), RVal);
3937 EXPECT_EQ(Store4->getValueOperand(), ZExt1);
3938
3939 AtomicCmpXchgInst *CmpXchg5 =
3940 dyn_cast<AtomicCmpXchgInst>(Store4->getNextNode());
3941 EXPECT_NE(CmpXchg5, nullptr);
3942 EXPECT_EQ(CmpXchg5->getPointerOperand(), XVal);
3943 EXPECT_EQ(CmpXchg5->getCompareOperand(), Expr);
3944 EXPECT_EQ(CmpXchg5->getNewValOperand(), D);
3945 ExtractValueInst *ExtVal7 =
3946 dyn_cast<ExtractValueInst>(CmpXchg5->getNextNode());
3947 EXPECT_NE(ExtVal7, nullptr);
3948 EXPECT_EQ(ExtVal7->getAggregateOperand(), CmpXchg5);
3949 EXPECT_EQ(ExtVal7->getIndices(), ArrayRef<unsigned int>(0U));
3950 ExtractValueInst *ExtVal8 =
3951 dyn_cast<ExtractValueInst>(ExtVal7->getNextNode());
3952 EXPECT_NE(ExtVal8, nullptr);
3953 EXPECT_EQ(ExtVal8->getAggregateOperand(), CmpXchg5);
3954 EXPECT_EQ(ExtVal8->getIndices(), ArrayRef<unsigned int>(1U));
3955 BranchInst *Br3 = dyn_cast<BranchInst>(ExtVal8->getNextNode());
3956 EXPECT_NE(Br3, nullptr);
3957 EXPECT_EQ(Br3->isConditional(), true);
3958 EXPECT_EQ(Br3->getCondition(), ExtVal8);
3959 EXPECT_EQ(Br3->getSuccessor(0), Exit2);
3960 EXPECT_EQ(Br3->getSuccessor(1), Cont2);
3961
3962 StoreInst *Store5 = dyn_cast<StoreInst>(&Cont2->front());
3963 EXPECT_NE(Store5, nullptr);
3964 EXPECT_EQ(Store5->getPointerOperand(), VVal);
3965 EXPECT_EQ(Store5->getValueOperand(), ExtVal7);
3966 BranchInst *Br4 = dyn_cast<BranchInst>(Store5->getNextNode());
3967 EXPECT_NE(Br4, nullptr);
3968 EXPECT_EQ(Br4->isUnconditional(), true);
3969 EXPECT_EQ(Br4->getSuccessor(0), Exit2);
3970
3971 ExtractValueInst *ExtVal9 = dyn_cast<ExtractValueInst>(&Exit2->front());
3972 EXPECT_NE(ExtVal9, nullptr);
3973 EXPECT_EQ(ExtVal9->getAggregateOperand(), CmpXchg5);
3974 EXPECT_EQ(ExtVal9->getIndices(), ArrayRef<unsigned int>(1U));
3975 ZExtInst *ZExt2 = dyn_cast<ZExtInst>(ExtVal9->getNextNode());
3976 EXPECT_NE(ZExt2, nullptr);
3977 EXPECT_EQ(ZExt2->getDestTy(), Int32);
3978 StoreInst *Store6 = dyn_cast<StoreInst>(ZExt2->getNextNode());
3979 EXPECT_NE(Store6, nullptr);
3980 EXPECT_EQ(Store6->getPointerOperand(), RVal);
3981 EXPECT_EQ(Store6->getValueOperand(), ZExt2);
3982
3983 AtomicRMWInst *ARWM1 = dyn_cast<AtomicRMWInst>(Store6->getNextNode());
3984 EXPECT_NE(ARWM1, nullptr);
3985 EXPECT_EQ(ARWM1->getPointerOperand(), XVal);
3986 EXPECT_EQ(ARWM1->getValOperand(), Expr);
3987 EXPECT_EQ(ARWM1->getOperation(), AtomicRMWInst::Min);
3988 StoreInst *Store7 = dyn_cast<StoreInst>(ARWM1->getNextNode());
3989 EXPECT_NE(Store7, nullptr);
3990 EXPECT_EQ(Store7->getPointerOperand(), VVal);
3991 EXPECT_EQ(Store7->getValueOperand(), ARWM1);
3992
3993 AtomicRMWInst *ARWM2 = dyn_cast<AtomicRMWInst>(Store7->getNextNode());
3994 EXPECT_NE(ARWM2, nullptr);
3995 EXPECT_EQ(ARWM2->getPointerOperand(), XVal);
3996 EXPECT_EQ(ARWM2->getValOperand(), Expr);
3997 EXPECT_EQ(ARWM2->getOperation(), AtomicRMWInst::Max);
3998 CmpInst *Cmp1 = dyn_cast<CmpInst>(ARWM2->getNextNode());
3999 EXPECT_NE(Cmp1, nullptr);
4000 EXPECT_EQ(Cmp1->getPredicate(), CmpInst::ICMP_SGT);
4001 EXPECT_EQ(Cmp1->getOperand(0), ARWM2);
4002 EXPECT_EQ(Cmp1->getOperand(1), Expr);
4003 SelectInst *Sel2 = dyn_cast<SelectInst>(Cmp1->getNextNode());
4004 EXPECT_NE(Sel2, nullptr);
4005 EXPECT_EQ(Sel2->getCondition(), Cmp1);
4006 EXPECT_EQ(Sel2->getTrueValue(), Expr);
4007 EXPECT_EQ(Sel2->getFalseValue(), ARWM2);
4008 StoreInst *Store8 = dyn_cast<StoreInst>(Sel2->getNextNode());
4009 EXPECT_NE(Store8, nullptr);
4010 EXPECT_EQ(Store8->getPointerOperand(), VVal);
4011 EXPECT_EQ(Store8->getValueOperand(), Sel2);
4012
4013 Builder.CreateRetVoid();
4014 OMPBuilder.finalize();
4015 EXPECT_FALSE(verifyModule(*M, &errs()));
4016 }
4017
4018 /// Returns the single instruction of InstTy type in BB that uses the value V.
4019 /// If there is more than one such instruction, returns null.
4020 template <typename InstTy>
4021 static InstTy *findSingleUserInBlock(Value *V, BasicBlock *BB) {
4022 InstTy *Result = nullptr;
4023 for (User *U : V->users()) {
4024 auto *Inst = dyn_cast<InstTy>(U);
4025 if (!Inst || Inst->getParent() != BB)
4026 continue;
4027 if (Result)
4028 return nullptr;
4029 Result = Inst;
4030 }
4031 return Result;
4032 }
4033
4034 /// Returns true if BB contains a simple binary reduction that loads a value
4035 /// from Accum, performs some binary operation with it, and stores it back to
4036 /// Accum.
4037 static bool isSimpleBinaryReduction(Value *Accum, BasicBlock *BB,
4038 Instruction::BinaryOps *OpCode = nullptr) {
4039 StoreInst *Store = findSingleUserInBlock<StoreInst>(Accum, BB);
4040 if (!Store)
4041 return false;
4042 auto *Stored = dyn_cast<BinaryOperator>(Store->getOperand(0));
4043 if (!Stored)
4044 return false;
4045 if (OpCode && *OpCode != Stored->getOpcode())
4046 return false;
4047 auto *Load = dyn_cast<LoadInst>(Stored->getOperand(0));
4048 return Load && Load->getOperand(0) == Accum;
4049 }
4050
4051 /// Returns true if BB contains a binary reduction that reduces V using a binary
4052 /// operator into an accumulator that is a function argument.
4053 static bool isValueReducedToFuncArg(Value *V, BasicBlock *BB) {
4054 auto *ReductionOp = findSingleUserInBlock<BinaryOperator>(V, BB);
4055 if (!ReductionOp)
4056 return false;
4057
4058 auto *GlobalLoad = dyn_cast<LoadInst>(ReductionOp->getOperand(0));
4059 if (!GlobalLoad)
4060 return false;
4061
4062 auto *Store = findSingleUserInBlock<StoreInst>(ReductionOp, BB);
4063 if (!Store)
4064 return false;
4065
4066 return Store->getPointerOperand() == GlobalLoad->getPointerOperand() &&
4067 isa<Argument>(findAggregateFromValue(GlobalLoad->getPointerOperand()));
4068 }
4069
4070 /// Finds among users of Ptr a pair of GEP instructions with indices [0, 0] and
4071 /// [0, 1], respectively, and assigns results of these instructions to Zero and
4072 /// One. Returns true on success, false on failure or if such instructions are
4073 /// not unique among the users of Ptr.
4074 static bool findGEPZeroOne(Value *Ptr, Value *&Zero, Value *&One) {
4075 Zero = nullptr;
4076 One = nullptr;
4077 for (User *U : Ptr->users()) {
4078 if (auto *GEP = dyn_cast<GetElementPtrInst>(U)) {
4079 if (GEP->getNumIndices() != 2)
4080 continue;
4081 auto *FirstIdx = dyn_cast<ConstantInt>(GEP->getOperand(1));
4082 auto *SecondIdx = dyn_cast<ConstantInt>(GEP->getOperand(2));
4083 EXPECT_NE(FirstIdx, nullptr);
4084 EXPECT_NE(SecondIdx, nullptr);
4085
4086 EXPECT_TRUE(FirstIdx->isZero());
4087 if (SecondIdx->isZero()) {
4088 if (Zero)
4089 return false;
4090 Zero = GEP;
4091 } else if (SecondIdx->isOne()) {
4092 if (One)
4093 return false;
4094 One = GEP;
4095 } else {
4096 return false;
4097 }
4098 }
4099 }
4100 return Zero != nullptr && One != nullptr;
4101 }
4102
4103 static OpenMPIRBuilder::InsertPointTy
4104 sumReduction(OpenMPIRBuilder::InsertPointTy IP, Value *LHS, Value *RHS,
4105 Value *&Result) {
4106 IRBuilder<> Builder(IP.getBlock(), IP.getPoint());
4107 Result = Builder.CreateFAdd(LHS, RHS, "red.add");
4108 return Builder.saveIP();
4109 }
4110
4111 static OpenMPIRBuilder::InsertPointTy
4112 sumAtomicReduction(OpenMPIRBuilder::InsertPointTy IP, Type *Ty, Value *LHS,
4113 Value *RHS) {
4114 IRBuilder<> Builder(IP.getBlock(), IP.getPoint());
4115 Value *Partial = Builder.CreateLoad(Ty, RHS, "red.partial");
4116 Builder.CreateAtomicRMW(AtomicRMWInst::FAdd, LHS, Partial, std::nullopt,
4117 AtomicOrdering::Monotonic);
4118 return Builder.saveIP();
4119 }
4120
4121 static OpenMPIRBuilder::InsertPointTy
4122 xorReduction(OpenMPIRBuilder::InsertPointTy IP, Value *LHS, Value *RHS,
4123 Value *&Result) {
4124 IRBuilder<> Builder(IP.getBlock(), IP.getPoint());
4125 Result = Builder.CreateXor(LHS, RHS, "red.xor");
4126 return Builder.saveIP();
4127 }
4128
4129 static OpenMPIRBuilder::InsertPointTy
4130 xorAtomicReduction(OpenMPIRBuilder::InsertPointTy IP, Type *Ty, Value *LHS,
4131 Value *RHS) {
4132 IRBuilder<> Builder(IP.getBlock(), IP.getPoint());
4133 Value *Partial = Builder.CreateLoad(Ty, RHS, "red.partial");
4134 Builder.CreateAtomicRMW(AtomicRMWInst::Xor, LHS, Partial, std::nullopt,
4135 AtomicOrdering::Monotonic);
4136 return Builder.saveIP();
4137 }
4138
4139 TEST_F(OpenMPIRBuilderTest, CreateReductions) {
4140 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4141 OpenMPIRBuilder OMPBuilder(*M);
4142 OMPBuilder.initialize();
4143 F->setName("func");
4144 IRBuilder<> Builder(BB);
4145
4146 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
4147 Builder.CreateBr(EnterBB);
4148 Builder.SetInsertPoint(EnterBB);
4149 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4150
4151 // Create variables to be reduced.
4152 InsertPointTy OuterAllocaIP(&F->getEntryBlock(),
4153 F->getEntryBlock().getFirstInsertionPt());
4154 Type *SumType = Builder.getFloatTy();
4155 Type *XorType = Builder.getInt32Ty();
4156 Value *SumReduced;
4157 Value *XorReduced;
4158 {
4159 IRBuilderBase::InsertPointGuard Guard(Builder);
4160 Builder.restoreIP(OuterAllocaIP);
4161 SumReduced = Builder.CreateAlloca(SumType);
4162 XorReduced = Builder.CreateAlloca(XorType);
4163 }
4164
4165 // Store initial values of reductions into global variables.
4166 Builder.CreateStore(ConstantFP::get(Builder.getFloatTy(), 0.0), SumReduced);
4167 Builder.CreateStore(Builder.getInt32(1), XorReduced);
4168
4169 // The loop body computes two reductions:
4170 // sum of (float) thread-id;
4171 // xor of thread-id;
4172 // and store the result in global variables.
4173 InsertPointTy BodyIP, BodyAllocaIP;
4174 auto BodyGenCB = [&](InsertPointTy InnerAllocaIP, InsertPointTy CodeGenIP) {
4175 IRBuilderBase::InsertPointGuard Guard(Builder);
4176 Builder.restoreIP(CodeGenIP);
4177
4178 uint32_t StrSize;
4179 Constant *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(Loc, StrSize);
4180 Value *Ident = OMPBuilder.getOrCreateIdent(SrcLocStr, StrSize);
4181 Value *TID = OMPBuilder.getOrCreateThreadID(Ident);
4182 Value *SumLocal =
4183 Builder.CreateUIToFP(TID, Builder.getFloatTy(), "sum.local");
4184 Value *SumPartial = Builder.CreateLoad(SumType, SumReduced, "sum.partial");
4185 Value *XorPartial = Builder.CreateLoad(XorType, XorReduced, "xor.partial");
4186 Value *Sum = Builder.CreateFAdd(SumPartial, SumLocal, "sum");
4187 Value *Xor = Builder.CreateXor(XorPartial, TID, "xor");
4188 Builder.CreateStore(Sum, SumReduced);
4189 Builder.CreateStore(Xor, XorReduced);
4190
4191 BodyIP = Builder.saveIP();
4192 BodyAllocaIP = InnerAllocaIP;
4193 };
4194
4195 // Privatization for reduction creates local copies of reduction variables and
4196 // initializes them to reduction-neutral values.
4197 Value *SumPrivatized;
4198 Value *XorPrivatized;
4199 auto PrivCB = [&](InsertPointTy InnerAllocaIP, InsertPointTy CodeGenIP,
4200 Value &Original, Value &Inner, Value *&ReplVal) {
4201 IRBuilderBase::InsertPointGuard Guard(Builder);
4202 Builder.restoreIP(InnerAllocaIP);
4203 if (&Original == SumReduced) {
4204 SumPrivatized = Builder.CreateAlloca(Builder.getFloatTy());
4205 ReplVal = SumPrivatized;
4206 } else if (&Original == XorReduced) {
4207 XorPrivatized = Builder.CreateAlloca(Builder.getInt32Ty());
4208 ReplVal = XorPrivatized;
4209 } else {
4210 ReplVal = &Inner;
4211 return CodeGenIP;
4212 }
4213
4214 Builder.restoreIP(CodeGenIP);
4215 if (&Original == SumReduced)
4216 Builder.CreateStore(ConstantFP::get(Builder.getFloatTy(), 0.0),
4217 SumPrivatized);
4218 else if (&Original == XorReduced)
4219 Builder.CreateStore(Builder.getInt32(0), XorPrivatized);
4220
4221 return Builder.saveIP();
4222 };
4223
4224 // Do nothing in finalization.
4225 auto FiniCB = [&](InsertPointTy CodeGenIP) { return CodeGenIP; };
4226
4227 InsertPointTy AfterIP =
4228 OMPBuilder.createParallel(Loc, OuterAllocaIP, BodyGenCB, PrivCB, FiniCB,
4229 /* IfCondition */ nullptr,
4230 /* NumThreads */ nullptr, OMP_PROC_BIND_default,
4231 /* IsCancellable */ false);
4232 Builder.restoreIP(AfterIP);
4233
4234 OpenMPIRBuilder::ReductionInfo ReductionInfos[] = {
4235 {SumType, SumReduced, SumPrivatized, sumReduction, sumAtomicReduction},
4236 {XorType, XorReduced, XorPrivatized, xorReduction, xorAtomicReduction}};
4237
4238 OMPBuilder.createReductions(BodyIP, BodyAllocaIP, ReductionInfos);
4239
4240 Builder.restoreIP(AfterIP);
4241 Builder.CreateRetVoid();
4242
4243 OMPBuilder.finalize(F);
4244
4245 // The IR must be valid.
4246 EXPECT_FALSE(verifyModule(*M));
4247
4248 // Outlining must have happened.
4249 SmallVector<CallInst *> ForkCalls;
4250 findCalls(F, omp::RuntimeFunction::OMPRTL___kmpc_fork_call, OMPBuilder,
4251 ForkCalls);
4252 ASSERT_EQ(ForkCalls.size(), 1u);
4253 Value *CalleeVal = cast<Constant>(ForkCalls[0]->getOperand(2))->getOperand(0);
4254 Function *Outlined = dyn_cast<Function>(CalleeVal);
4255 EXPECT_NE(Outlined, nullptr);
4256
4257 // Check that the lock variable was created with the expected name.
4258 GlobalVariable *LockVar =
4259 M->getGlobalVariable(".gomp_critical_user_.reduction.var");
4260 EXPECT_NE(LockVar, nullptr);
4261
4262 // Find the allocation of a local array that will be used to call the runtime
4263 // reduciton function.
4264 BasicBlock &AllocBlock = Outlined->getEntryBlock();
4265 Value *LocalArray = nullptr;
4266 for (Instruction &I : AllocBlock) {
4267 if (AllocaInst *Alloc = dyn_cast<AllocaInst>(&I)) {
4268 if (!Alloc->getAllocatedType()->isArrayTy() ||
4269 !Alloc->getAllocatedType()->getArrayElementType()->isPointerTy())
4270 continue;
4271 LocalArray = Alloc;
4272 break;
4273 }
4274 }
4275 ASSERT_NE(LocalArray, nullptr);
4276
4277 // Find the call to the runtime reduction function.
4278 BasicBlock *BB = AllocBlock.getUniqueSuccessor();
4279 Value *LocalArrayPtr = nullptr;
4280 Value *ReductionFnVal = nullptr;
4281 Value *SwitchArg = nullptr;
4282 for (Instruction &I : *BB) {
4283 if (CallInst *Call = dyn_cast<CallInst>(&I)) {
4284 if (Call->getCalledFunction() !=
4285 OMPBuilder.getOrCreateRuntimeFunctionPtr(
4286 RuntimeFunction::OMPRTL___kmpc_reduce))
4287 continue;
4288 LocalArrayPtr = Call->getOperand(4);
4289 ReductionFnVal = Call->getOperand(5);
4290 SwitchArg = Call;
4291 break;
4292 }
4293 }
4294
4295 // Check that the local array is passed to the function.
4296 ASSERT_NE(LocalArrayPtr, nullptr);
4297 BitCastInst *BitCast = dyn_cast<BitCastInst>(LocalArrayPtr);
4298 ASSERT_NE(BitCast, nullptr);
4299 EXPECT_EQ(BitCast->getOperand(0), LocalArray);
4300
4301 // Find the GEP instructions preceding stores to the local array.
4302 Value *FirstArrayElemPtr = nullptr;
4303 Value *SecondArrayElemPtr = nullptr;
4304 EXPECT_EQ(LocalArray->getNumUses(), 3u);
4305 ASSERT_TRUE(
4306 findGEPZeroOne(LocalArray, FirstArrayElemPtr, SecondArrayElemPtr));
4307
4308 // Check that the values stored into the local array are privatized reduction
4309 // variables.
4310 auto *FirstStored = dyn_cast_or_null<BitCastInst>(
4311 findStoredValue<GetElementPtrInst>(FirstArrayElemPtr));
4312 auto *SecondStored = dyn_cast_or_null<BitCastInst>(
4313 findStoredValue<GetElementPtrInst>(SecondArrayElemPtr));
4314 ASSERT_NE(FirstStored, nullptr);
4315 ASSERT_NE(SecondStored, nullptr);
4316 Value *FirstPrivatized = FirstStored->getOperand(0);
4317 Value *SecondPrivatized = SecondStored->getOperand(0);
4318 EXPECT_TRUE(
4319 isSimpleBinaryReduction(FirstPrivatized, FirstStored->getParent()));
4320 EXPECT_TRUE(
4321 isSimpleBinaryReduction(SecondPrivatized, SecondStored->getParent()));
4322
4323 // Check that the result of the runtime reduction call is used for further
4324 // dispatch.
4325 ASSERT_EQ(SwitchArg->getNumUses(), 1u);
4326 SwitchInst *Switch = dyn_cast<SwitchInst>(*SwitchArg->user_begin());
4327 ASSERT_NE(Switch, nullptr);
4328 EXPECT_EQ(Switch->getNumSuccessors(), 3u);
4329 BasicBlock *NonAtomicBB = Switch->case_begin()->getCaseSuccessor();
4330 BasicBlock *AtomicBB = std::next(Switch->case_begin())->getCaseSuccessor();
4331
4332 // Non-atomic block contains reductions to the global reduction variable,
4333 // which is passed into the outlined function as an argument.
4334 Value *FirstLoad =
4335 findSingleUserInBlock<LoadInst>(FirstPrivatized, NonAtomicBB);
4336 Value *SecondLoad =
4337 findSingleUserInBlock<LoadInst>(SecondPrivatized, NonAtomicBB);
4338 EXPECT_TRUE(isValueReducedToFuncArg(FirstLoad, NonAtomicBB));
4339 EXPECT_TRUE(isValueReducedToFuncArg(SecondLoad, NonAtomicBB));
4340
4341 // Atomic block also constains reductions to the global reduction variable.
4342 FirstLoad = findSingleUserInBlock<LoadInst>(FirstPrivatized, AtomicBB);
4343 SecondLoad = findSingleUserInBlock<LoadInst>(SecondPrivatized, AtomicBB);
4344 auto *FirstAtomic = findSingleUserInBlock<AtomicRMWInst>(FirstLoad, AtomicBB);
4345 auto *SecondAtomic =
4346 findSingleUserInBlock<AtomicRMWInst>(SecondLoad, AtomicBB);
4347 ASSERT_NE(FirstAtomic, nullptr);
4348 Value *AtomicStorePointer = FirstAtomic->getPointerOperand();
4349 EXPECT_TRUE(isa<Argument>(findAggregateFromValue(AtomicStorePointer)));
4350 ASSERT_NE(SecondAtomic, nullptr);
4351 AtomicStorePointer = SecondAtomic->getPointerOperand();
4352 EXPECT_TRUE(isa<Argument>(findAggregateFromValue(AtomicStorePointer)));
4353
4354 // Check that the separate reduction function also performs (non-atomic)
4355 // reductions after extracting reduction variables from its arguments.
4356 Function *ReductionFn = cast<Function>(ReductionFnVal);
4357 BasicBlock *FnReductionBB = &ReductionFn->getEntryBlock();
4358 auto *Bitcast =
4359 findSingleUserInBlock<BitCastInst>(ReductionFn->getArg(0), FnReductionBB);
4360 Value *FirstLHSPtr;
4361 Value *SecondLHSPtr;
4362 ASSERT_TRUE(findGEPZeroOne(Bitcast, FirstLHSPtr, SecondLHSPtr));
4363 Value *Opaque = findSingleUserInBlock<LoadInst>(FirstLHSPtr, FnReductionBB);
4364 ASSERT_NE(Opaque, nullptr);
4365 Bitcast = findSingleUserInBlock<BitCastInst>(Opaque, FnReductionBB);
4366 ASSERT_NE(Bitcast, nullptr);
4367 EXPECT_TRUE(isSimpleBinaryReduction(Bitcast, FnReductionBB));
4368 Opaque = findSingleUserInBlock<LoadInst>(SecondLHSPtr, FnReductionBB);
4369 ASSERT_NE(Opaque, nullptr);
4370 Bitcast = findSingleUserInBlock<BitCastInst>(Opaque, FnReductionBB);
4371 ASSERT_NE(Bitcast, nullptr);
4372 EXPECT_TRUE(isSimpleBinaryReduction(Bitcast, FnReductionBB));
4373
4374 Bitcast =
4375 findSingleUserInBlock<BitCastInst>(ReductionFn->getArg(1), FnReductionBB);
4376 Value *FirstRHS;
4377 Value *SecondRHS;
4378 EXPECT_TRUE(findGEPZeroOne(Bitcast, FirstRHS, SecondRHS));
4379 }
4380
4381 TEST_F(OpenMPIRBuilderTest, CreateTwoReductions) {
4382 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4383 OpenMPIRBuilder OMPBuilder(*M);
4384 OMPBuilder.initialize();
4385 F->setName("func");
4386 IRBuilder<> Builder(BB);
4387
4388 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
4389 Builder.CreateBr(EnterBB);
4390 Builder.SetInsertPoint(EnterBB);
4391 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4392
4393 // Create variables to be reduced.
4394 InsertPointTy OuterAllocaIP(&F->getEntryBlock(),
4395 F->getEntryBlock().getFirstInsertionPt());
4396 Type *SumType = Builder.getFloatTy();
4397 Type *XorType = Builder.getInt32Ty();
4398 Value *SumReduced;
4399 Value *XorReduced;
4400 {
4401 IRBuilderBase::InsertPointGuard Guard(Builder);
4402 Builder.restoreIP(OuterAllocaIP);
4403 SumReduced = Builder.CreateAlloca(SumType);
4404 XorReduced = Builder.CreateAlloca(XorType);
4405 }
4406
4407 // Store initial values of reductions into global variables.
4408 Builder.CreateStore(ConstantFP::get(Builder.getFloatTy(), 0.0), SumReduced);
4409 Builder.CreateStore(Builder.getInt32(1), XorReduced);
4410
4411 InsertPointTy FirstBodyIP, FirstBodyAllocaIP;
4412 auto FirstBodyGenCB = [&](InsertPointTy InnerAllocaIP,
4413 InsertPointTy CodeGenIP) {
4414 IRBuilderBase::InsertPointGuard Guard(Builder);
4415 Builder.restoreIP(CodeGenIP);
4416
4417 uint32_t StrSize;
4418 Constant *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(Loc, StrSize);
4419 Value *Ident = OMPBuilder.getOrCreateIdent(SrcLocStr, StrSize);
4420 Value *TID = OMPBuilder.getOrCreateThreadID(Ident);
4421 Value *SumLocal =
4422 Builder.CreateUIToFP(TID, Builder.getFloatTy(), "sum.local");
4423 Value *SumPartial = Builder.CreateLoad(SumType, SumReduced, "sum.partial");
4424 Value *Sum = Builder.CreateFAdd(SumPartial, SumLocal, "sum");
4425 Builder.CreateStore(Sum, SumReduced);
4426
4427 FirstBodyIP = Builder.saveIP();
4428 FirstBodyAllocaIP = InnerAllocaIP;
4429 };
4430
4431 InsertPointTy SecondBodyIP, SecondBodyAllocaIP;
4432 auto SecondBodyGenCB = [&](InsertPointTy InnerAllocaIP,
4433 InsertPointTy CodeGenIP) {
4434 IRBuilderBase::InsertPointGuard Guard(Builder);
4435 Builder.restoreIP(CodeGenIP);
4436
4437 uint32_t StrSize;
4438 Constant *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(Loc, StrSize);
4439 Value *Ident = OMPBuilder.getOrCreateIdent(SrcLocStr, StrSize);
4440 Value *TID = OMPBuilder.getOrCreateThreadID(Ident);
4441 Value *XorPartial = Builder.CreateLoad(XorType, XorReduced, "xor.partial");
4442 Value *Xor = Builder.CreateXor(XorPartial, TID, "xor");
4443 Builder.CreateStore(Xor, XorReduced);
4444
4445 SecondBodyIP = Builder.saveIP();
4446 SecondBodyAllocaIP = InnerAllocaIP;
4447 };
4448
4449 // Privatization for reduction creates local copies of reduction variables and
4450 // initializes them to reduction-neutral values. The same privatization
4451 // callback is used for both loops, with dispatch based on the value being
4452 // privatized.
4453 Value *SumPrivatized;
4454 Value *XorPrivatized;
4455 auto PrivCB = [&](InsertPointTy InnerAllocaIP, InsertPointTy CodeGenIP,
4456 Value &Original, Value &Inner, Value *&ReplVal) {
4457 IRBuilderBase::InsertPointGuard Guard(Builder);
4458 Builder.restoreIP(InnerAllocaIP);
4459 if (&Original == SumReduced) {
4460 SumPrivatized = Builder.CreateAlloca(Builder.getFloatTy());
4461 ReplVal = SumPrivatized;
4462 } else if (&Original == XorReduced) {
4463 XorPrivatized = Builder.CreateAlloca(Builder.getInt32Ty());
4464 ReplVal = XorPrivatized;
4465 } else {
4466 ReplVal = &Inner;
4467 return CodeGenIP;
4468 }
4469
4470 Builder.restoreIP(CodeGenIP);
4471 if (&Original == SumReduced)
4472 Builder.CreateStore(ConstantFP::get(Builder.getFloatTy(), 0.0),
4473 SumPrivatized);
4474 else if (&Original == XorReduced)
4475 Builder.CreateStore(Builder.getInt32(0), XorPrivatized);
4476
4477 return Builder.saveIP();
4478 };
4479
4480 // Do nothing in finalization.
4481 auto FiniCB = [&](InsertPointTy CodeGenIP) { return CodeGenIP; };
4482
4483 Builder.restoreIP(
4484 OMPBuilder.createParallel(Loc, OuterAllocaIP, FirstBodyGenCB, PrivCB,
4485 FiniCB, /* IfCondition */ nullptr,
4486 /* NumThreads */ nullptr, OMP_PROC_BIND_default,
4487 /* IsCancellable */ false));
4488 InsertPointTy AfterIP = OMPBuilder.createParallel(
4489 {Builder.saveIP(), DL}, OuterAllocaIP, SecondBodyGenCB, PrivCB, FiniCB,
4490 /* IfCondition */ nullptr,
4491 /* NumThreads */ nullptr, OMP_PROC_BIND_default,
4492 /* IsCancellable */ false);
4493
4494 OMPBuilder.createReductions(
4495 FirstBodyIP, FirstBodyAllocaIP,
4496 {{SumType, SumReduced, SumPrivatized, sumReduction, sumAtomicReduction}});
4497 OMPBuilder.createReductions(
4498 SecondBodyIP, SecondBodyAllocaIP,
4499 {{XorType, XorReduced, XorPrivatized, xorReduction, xorAtomicReduction}});
4500
4501 Builder.restoreIP(AfterIP);
4502 Builder.CreateRetVoid();
4503
4504 OMPBuilder.finalize(F);
4505
4506 // The IR must be valid.
4507 EXPECT_FALSE(verifyModule(*M));
4508
4509 // Two different outlined functions must have been created.
4510 SmallVector<CallInst *> ForkCalls;
4511 findCalls(F, omp::RuntimeFunction::OMPRTL___kmpc_fork_call, OMPBuilder,
4512 ForkCalls);
4513 ASSERT_EQ(ForkCalls.size(), 2u);
4514 Value *CalleeVal = cast<Constant>(ForkCalls[0]->getOperand(2))->getOperand(0);
4515 Function *FirstCallee = cast<Function>(CalleeVal);
4516 CalleeVal = cast<Constant>(ForkCalls[1]->getOperand(2))->getOperand(0);
4517 Function *SecondCallee = cast<Function>(CalleeVal);
4518 EXPECT_NE(FirstCallee, SecondCallee);
4519
4520 // Two different reduction functions must have been created.
4521 SmallVector<CallInst *> ReduceCalls;
4522 findCalls(FirstCallee, omp::RuntimeFunction::OMPRTL___kmpc_reduce, OMPBuilder,
4523 ReduceCalls);
4524 ASSERT_EQ(ReduceCalls.size(), 1u);
4525 auto *AddReduction = cast<Function>(ReduceCalls[0]->getOperand(5));
4526 ReduceCalls.clear();
4527 findCalls(SecondCallee, omp::RuntimeFunction::OMPRTL___kmpc_reduce,
4528 OMPBuilder, ReduceCalls);
4529 auto *XorReduction = cast<Function>(ReduceCalls[0]->getOperand(5));
4530 EXPECT_NE(AddReduction, XorReduction);
4531
4532 // Each reduction function does its own kind of reduction.
4533 BasicBlock *FnReductionBB = &AddReduction->getEntryBlock();
4534 auto *Bitcast = findSingleUserInBlock<BitCastInst>(AddReduction->getArg(0),
4535 FnReductionBB);
4536 ASSERT_NE(Bitcast, nullptr);
4537 Value *FirstLHSPtr =
4538 findSingleUserInBlock<GetElementPtrInst>(Bitcast, FnReductionBB);
4539 ASSERT_NE(FirstLHSPtr, nullptr);
4540 Value *Opaque = findSingleUserInBlock<LoadInst>(FirstLHSPtr, FnReductionBB);
4541 ASSERT_NE(Opaque, nullptr);
4542 Bitcast = findSingleUserInBlock<BitCastInst>(Opaque, FnReductionBB);
4543 ASSERT_NE(Bitcast, nullptr);
4544 Instruction::BinaryOps Opcode = Instruction::FAdd;
4545 EXPECT_TRUE(isSimpleBinaryReduction(Bitcast, FnReductionBB, &Opcode));
4546
4547 FnReductionBB = &XorReduction->getEntryBlock();
4548 Bitcast = findSingleUserInBlock<BitCastInst>(XorReduction->getArg(0),
4549 FnReductionBB);
4550 ASSERT_NE(Bitcast, nullptr);
4551 Value *SecondLHSPtr =
4552 findSingleUserInBlock<GetElementPtrInst>(Bitcast, FnReductionBB);
4553 ASSERT_NE(FirstLHSPtr, nullptr);
4554 Opaque = findSingleUserInBlock<LoadInst>(SecondLHSPtr, FnReductionBB);
4555 ASSERT_NE(Opaque, nullptr);
4556 Bitcast = findSingleUserInBlock<BitCastInst>(Opaque, FnReductionBB);
4557 ASSERT_NE(Bitcast, nullptr);
4558 Opcode = Instruction::Xor;
4559 EXPECT_TRUE(isSimpleBinaryReduction(Bitcast, FnReductionBB, &Opcode));
4560 }
4561
4562 TEST_F(OpenMPIRBuilderTest, CreateSectionsSimple) {
4563 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4564 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4565 OpenMPIRBuilder OMPBuilder(*M);
4566 OMPBuilder.initialize();
4567 F->setName("func");
4568 IRBuilder<> Builder(BB);
4569
4570 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "sections.enter", F);
4571 Builder.CreateBr(EnterBB);
4572 Builder.SetInsertPoint(EnterBB);
4573 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4574
4575 llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector;
4576 llvm::SmallVector<BasicBlock *, 4> CaseBBs;
4577
4578 auto FiniCB = [&](InsertPointTy IP) {};
4579 auto SectionCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
4580 SectionCBVector.push_back(SectionCB);
4581
4582 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
4583 llvm::Value &, llvm::Value &Val,
4584 llvm::Value *&ReplVal) { return CodeGenIP; };
4585 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
4586 F->getEntryBlock().getFirstInsertionPt());
4587 Builder.restoreIP(OMPBuilder.createSections(Loc, AllocaIP, SectionCBVector,
4588 PrivCB, FiniCB, false, false));
4589 Builder.CreateRetVoid(); // Required at the end of the function
4590 EXPECT_NE(F->getEntryBlock().getTerminator(), nullptr);
4591 EXPECT_FALSE(verifyModule(*M, &errs()));
4592 }
4593
4594 TEST_F(OpenMPIRBuilderTest, CreateSections) {
4595 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4596 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4597 OpenMPIRBuilder OMPBuilder(*M);
4598 OMPBuilder.initialize();
4599 F->setName("func");
4600 IRBuilder<> Builder(BB);
4601
4602 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4603 llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector;
4604 llvm::SmallVector<BasicBlock *, 4> CaseBBs;
4605
4606 BasicBlock *SwitchBB = nullptr;
4607 AllocaInst *PrivAI = nullptr;
4608 SwitchInst *Switch = nullptr;
4609
4610 unsigned NumBodiesGenerated = 0;
4611 unsigned NumFiniCBCalls = 0;
4612 PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
4613
4614 auto FiniCB = [&](InsertPointTy IP) {
4615 ++NumFiniCBCalls;
4616 BasicBlock *IPBB = IP.getBlock();
4617 EXPECT_NE(IPBB->end(), IP.getPoint());
4618 };
4619
4620 auto SectionCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
4621 ++NumBodiesGenerated;
4622 CaseBBs.push_back(CodeGenIP.getBlock());
4623 SwitchBB = CodeGenIP.getBlock()->getSinglePredecessor();
4624 Builder.restoreIP(CodeGenIP);
4625 Builder.CreateStore(F->arg_begin(), PrivAI);
4626 Value *PrivLoad =
4627 Builder.CreateLoad(F->arg_begin()->getType(), PrivAI, "local.alloca");
4628 Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
4629 };
4630 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
4631 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
4632 // TODO: Privatization not implemented yet
4633 return CodeGenIP;
4634 };
4635
4636 SectionCBVector.push_back(SectionCB);
4637 SectionCBVector.push_back(SectionCB);
4638
4639 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
4640 F->getEntryBlock().getFirstInsertionPt());
4641 Builder.restoreIP(OMPBuilder.createSections(Loc, AllocaIP, SectionCBVector,
4642 PrivCB, FiniCB, false, false));
4643 Builder.CreateRetVoid(); // Required at the end of the function
4644
4645 // Switch BB's predecessor is loop condition BB, whose successor at index 1 is
4646 // loop's exit BB
4647 BasicBlock *ForExitBB =
4648 SwitchBB->getSinglePredecessor()->getTerminator()->getSuccessor(1);
4649 EXPECT_NE(ForExitBB, nullptr);
4650
4651 EXPECT_NE(PrivAI, nullptr);
4652 Function *OutlinedFn = PrivAI->getFunction();
4653 EXPECT_EQ(F, OutlinedFn);
4654 EXPECT_FALSE(verifyModule(*M, &errs()));
4655 EXPECT_EQ(OutlinedFn->arg_size(), 1U);
4656
4657 BasicBlock *LoopPreheaderBB =
4658 OutlinedFn->getEntryBlock().getSingleSuccessor();
4659 // loop variables are 5 - lower bound, upper bound, stride, islastiter, and
4660 // iterator/counter
4661 bool FoundForInit = false;
4662 for (Instruction &Inst : *LoopPreheaderBB) {
4663 if (isa<CallInst>(Inst)) {
4664 if (cast<CallInst>(&Inst)->getCalledFunction()->getName() ==
4665 "__kmpc_for_static_init_4u") {
4666 FoundForInit = true;
4667 }
4668 }
4669 }
4670 EXPECT_EQ(FoundForInit, true);
4671
4672 bool FoundForExit = false;
4673 bool FoundBarrier = false;
4674 for (Instruction &Inst : *ForExitBB) {
4675 if (isa<CallInst>(Inst)) {
4676 if (cast<CallInst>(&Inst)->getCalledFunction()->getName() ==
4677 "__kmpc_for_static_fini") {
4678 FoundForExit = true;
4679 }
4680 if (cast<CallInst>(&Inst)->getCalledFunction()->getName() ==
4681 "__kmpc_barrier") {
4682 FoundBarrier = true;
4683 }
4684 if (FoundForExit && FoundBarrier)
4685 break;
4686 }
4687 }
4688 EXPECT_EQ(FoundForExit, true);
4689 EXPECT_EQ(FoundBarrier, true);
4690
4691 EXPECT_NE(SwitchBB, nullptr);
4692 EXPECT_NE(SwitchBB->getTerminator(), nullptr);
4693 EXPECT_EQ(isa<SwitchInst>(SwitchBB->getTerminator()), true);
4694 Switch = cast<SwitchInst>(SwitchBB->getTerminator());
4695 EXPECT_EQ(Switch->getNumCases(), 2U);
4696
4697 EXPECT_EQ(CaseBBs.size(), 2U);
4698 for (auto *&CaseBB : CaseBBs) {
4699 EXPECT_EQ(CaseBB->getParent(), OutlinedFn);
4700 }
4701
4702 ASSERT_EQ(NumBodiesGenerated, 2U);
4703 ASSERT_EQ(NumFiniCBCalls, 1U);
4704 EXPECT_FALSE(verifyModule(*M, &errs()));
4705 }
4706
4707 TEST_F(OpenMPIRBuilderTest, CreateSectionsNoWait) {
4708 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4709 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4710 OpenMPIRBuilder OMPBuilder(*M);
4711 OMPBuilder.initialize();
4712 F->setName("func");
4713 IRBuilder<> Builder(BB);
4714
4715 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "sections.enter", F);
4716 Builder.CreateBr(EnterBB);
4717 Builder.SetInsertPoint(EnterBB);
4718 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4719
4720 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
4721 F->getEntryBlock().getFirstInsertionPt());
4722 llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector;
4723 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
4724 llvm::Value &, llvm::Value &Val,
4725 llvm::Value *&ReplVal) { return CodeGenIP; };
4726 auto FiniCB = [&](InsertPointTy IP) {};
4727
4728 Builder.restoreIP(OMPBuilder.createSections(Loc, AllocaIP, SectionCBVector,
4729 PrivCB, FiniCB, false, true));
4730 Builder.CreateRetVoid(); // Required at the end of the function
4731 for (auto &Inst : instructions(*F)) {
4732 EXPECT_FALSE(isa<CallInst>(Inst) &&
4733 cast<CallInst>(&Inst)->getCalledFunction()->getName() ==
4734 "__kmpc_barrier" &&
4735 "call to function __kmpc_barrier found with nowait");
4736 }
4737 }
4738
4739 TEST_F(OpenMPIRBuilderTest, CreateOffloadMaptypes) {
4740 OpenMPIRBuilder OMPBuilder(*M);
4741 OMPBuilder.initialize();
4742
4743 IRBuilder<> Builder(BB);
4744
4745 SmallVector<uint64_t> Mappings = {0, 1};
4746 GlobalVariable *OffloadMaptypesGlobal =
4747 OMPBuilder.createOffloadMaptypes(Mappings, "offload_maptypes");
4748 EXPECT_FALSE(M->global_empty());
4749 EXPECT_EQ(OffloadMaptypesGlobal->getName(), "offload_maptypes");
4750 EXPECT_TRUE(OffloadMaptypesGlobal->isConstant());
4751 EXPECT_TRUE(OffloadMaptypesGlobal->hasGlobalUnnamedAddr());
4752 EXPECT_TRUE(OffloadMaptypesGlobal->hasPrivateLinkage());
4753 EXPECT_TRUE(OffloadMaptypesGlobal->hasInitializer());
4754 Constant *Initializer = OffloadMaptypesGlobal->getInitializer();
4755 EXPECT_TRUE(isa<ConstantDataArray>(Initializer));
4756 ConstantDataArray *MappingInit = dyn_cast<ConstantDataArray>(Initializer);
4757 EXPECT_EQ(MappingInit->getNumElements(), Mappings.size());
4758 EXPECT_TRUE(MappingInit->getType()->getElementType()->isIntegerTy(64));
4759 Constant *CA = ConstantDataArray::get(Builder.getContext(), Mappings);
4760 EXPECT_EQ(MappingInit, CA);
4761 }
4762
4763 TEST_F(OpenMPIRBuilderTest, CreateOffloadMapnames) {
4764 OpenMPIRBuilder OMPBuilder(*M);
4765 OMPBuilder.initialize();
4766
4767 IRBuilder<> Builder(BB);
4768
4769 uint32_t StrSize;
4770 Constant *Cst1 =
4771 OMPBuilder.getOrCreateSrcLocStr("array1", "file1", 2, 5, StrSize);
4772 Constant *Cst2 =
4773 OMPBuilder.getOrCreateSrcLocStr("array2", "file1", 3, 5, StrSize);
4774 SmallVector<llvm::Constant *> Names = {Cst1, Cst2};
4775
4776 GlobalVariable *OffloadMaptypesGlobal =
4777 OMPBuilder.createOffloadMapnames(Names, "offload_mapnames");
4778 EXPECT_FALSE(M->global_empty());
4779 EXPECT_EQ(OffloadMaptypesGlobal->getName(), "offload_mapnames");
4780 EXPECT_TRUE(OffloadMaptypesGlobal->isConstant());
4781 EXPECT_FALSE(OffloadMaptypesGlobal->hasGlobalUnnamedAddr());
4782 EXPECT_TRUE(OffloadMaptypesGlobal->hasPrivateLinkage());
4783 EXPECT_TRUE(OffloadMaptypesGlobal->hasInitializer());
4784 Constant *Initializer = OffloadMaptypesGlobal->getInitializer();
4785 EXPECT_TRUE(isa<Constant>(Initializer->getOperand(0)->stripPointerCasts()));
4786 EXPECT_TRUE(isa<Constant>(Initializer->getOperand(1)->stripPointerCasts()));
4787
4788 GlobalVariable *Name1Gbl =
4789 cast<GlobalVariable>(Initializer->getOperand(0)->stripPointerCasts());
4790 EXPECT_TRUE(isa<ConstantDataArray>(Name1Gbl->getInitializer()));
4791 ConstantDataArray *Name1GblCA =
4792 dyn_cast<ConstantDataArray>(Name1Gbl->getInitializer());
4793 EXPECT_EQ(Name1GblCA->getAsCString(), ";file1;array1;2;5;;");
4794
4795 GlobalVariable *Name2Gbl =
4796 cast<GlobalVariable>(Initializer->getOperand(1)->stripPointerCasts());
4797 EXPECT_TRUE(isa<ConstantDataArray>(Name2Gbl->getInitializer()));
4798 ConstantDataArray *Name2GblCA =
4799 dyn_cast<ConstantDataArray>(Name2Gbl->getInitializer());
4800 EXPECT_EQ(Name2GblCA->getAsCString(), ";file1;array2;3;5;;");
4801
4802 EXPECT_TRUE(Initializer->getType()->getArrayElementType()->isPointerTy());
4803 EXPECT_EQ(Initializer->getType()->getArrayNumElements(), Names.size());
4804 }
4805
4806 TEST_F(OpenMPIRBuilderTest, CreateMapperAllocas) {
4807 OpenMPIRBuilder OMPBuilder(*M);
4808 OMPBuilder.initialize();
4809 F->setName("func");
4810 IRBuilder<> Builder(BB);
4811
4812 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4813
4814 unsigned TotalNbOperand = 2;
4815
4816 OpenMPIRBuilder::MapperAllocas MapperAllocas;
4817 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
4818 F->getEntryBlock().getFirstInsertionPt());
4819 OMPBuilder.createMapperAllocas(Loc, AllocaIP, TotalNbOperand, MapperAllocas);
4820 EXPECT_NE(MapperAllocas.ArgsBase, nullptr);
4821 EXPECT_NE(MapperAllocas.Args, nullptr);
4822 EXPECT_NE(MapperAllocas.ArgSizes, nullptr);
4823 EXPECT_TRUE(MapperAllocas.ArgsBase->getAllocatedType()->isArrayTy());
4824 ArrayType *ArrType =
4825 dyn_cast<ArrayType>(MapperAllocas.ArgsBase->getAllocatedType());
4826 EXPECT_EQ(ArrType->getNumElements(), TotalNbOperand);
4827 EXPECT_TRUE(MapperAllocas.ArgsBase->getAllocatedType()
4828 ->getArrayElementType()
4829 ->isPointerTy());
4830 EXPECT_TRUE(
4831 cast<PointerType>(
4832 MapperAllocas.ArgsBase->getAllocatedType()->getArrayElementType())
4833 ->isOpaqueOrPointeeTypeMatches(Builder.getInt8Ty()));
4834
4835 EXPECT_TRUE(MapperAllocas.Args->getAllocatedType()->isArrayTy());
4836 ArrType = dyn_cast<ArrayType>(MapperAllocas.Args->getAllocatedType());
4837 EXPECT_EQ(ArrType->getNumElements(), TotalNbOperand);
4838 EXPECT_TRUE(MapperAllocas.Args->getAllocatedType()
4839 ->getArrayElementType()
4840 ->isPointerTy());
4841 EXPECT_TRUE(cast<PointerType>(
4842 MapperAllocas.Args->getAllocatedType()->getArrayElementType())
4843 ->isOpaqueOrPointeeTypeMatches(Builder.getInt8Ty()));
4844
4845 EXPECT_TRUE(MapperAllocas.ArgSizes->getAllocatedType()->isArrayTy());
4846 ArrType = dyn_cast<ArrayType>(MapperAllocas.ArgSizes->getAllocatedType());
4847 EXPECT_EQ(ArrType->getNumElements(), TotalNbOperand);
4848 EXPECT_TRUE(MapperAllocas.ArgSizes->getAllocatedType()
4849 ->getArrayElementType()
4850 ->isIntegerTy(64));
4851 }
4852
4853 TEST_F(OpenMPIRBuilderTest, EmitMapperCall) {
4854 OpenMPIRBuilder OMPBuilder(*M);
4855 OMPBuilder.initialize();
4856 F->setName("func");
4857 IRBuilder<> Builder(BB);
4858 LLVMContext &Ctx = M->getContext();
4859
4860 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4861
4862 unsigned TotalNbOperand = 2;
4863
4864 OpenMPIRBuilder::MapperAllocas MapperAllocas;
4865 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
4866 F->getEntryBlock().getFirstInsertionPt());
4867 OMPBuilder.createMapperAllocas(Loc, AllocaIP, TotalNbOperand, MapperAllocas);
4868
4869 auto *BeginMapperFunc = OMPBuilder.getOrCreateRuntimeFunctionPtr(
4870 omp::OMPRTL___tgt_target_data_begin_mapper);
4871
4872 SmallVector<uint64_t> Flags = {0, 2};
4873
4874 uint32_t StrSize;
4875 Constant *SrcLocCst =
4876 OMPBuilder.getOrCreateSrcLocStr("", "file1", 2, 5, StrSize);
4877 Value *SrcLocInfo = OMPBuilder.getOrCreateIdent(SrcLocCst, StrSize);
4878
4879 Constant *Cst1 =
4880 OMPBuilder.getOrCreateSrcLocStr("array1", "file1", 2, 5, StrSize);
4881 Constant *Cst2 =
4882 OMPBuilder.getOrCreateSrcLocStr("array2", "file1", 3, 5, StrSize);
4883 SmallVector<llvm::Constant *> Names = {Cst1, Cst2};
4884
4885 GlobalVariable *Maptypes =
4886 OMPBuilder.createOffloadMaptypes(Flags, ".offload_maptypes");
4887 Value *MaptypesArg = Builder.CreateConstInBoundsGEP2_32(
4888 ArrayType::get(Type::getInt64Ty(Ctx), TotalNbOperand), Maptypes,
4889 /*Idx0=*/0, /*Idx1=*/0);
4890
4891 GlobalVariable *Mapnames =
4892 OMPBuilder.createOffloadMapnames(Names, ".offload_mapnames");
4893 Value *MapnamesArg = Builder.CreateConstInBoundsGEP2_32(
4894 ArrayType::get(Type::getInt8PtrTy(Ctx), TotalNbOperand), Mapnames,
4895 /*Idx0=*/0, /*Idx1=*/0);
4896
4897 OMPBuilder.emitMapperCall(Builder.saveIP(), BeginMapperFunc, SrcLocInfo,
4898 MaptypesArg, MapnamesArg, MapperAllocas, -1,
4899 TotalNbOperand);
4900
4901 CallInst *MapperCall = dyn_cast<CallInst>(&BB->back());
4902 EXPECT_NE(MapperCall, nullptr);
4903 EXPECT_EQ(MapperCall->arg_size(), 9U);
4904 EXPECT_EQ(MapperCall->getCalledFunction()->getName(),
4905 "__tgt_target_data_begin_mapper");
4906 EXPECT_EQ(MapperCall->getOperand(0), SrcLocInfo);
4907 EXPECT_TRUE(MapperCall->getOperand(1)->getType()->isIntegerTy(64));
4908 EXPECT_TRUE(MapperCall->getOperand(2)->getType()->isIntegerTy(32));
4909
4910 EXPECT_EQ(MapperCall->getOperand(6), MaptypesArg);
4911 EXPECT_EQ(MapperCall->getOperand(7), MapnamesArg);
4912 EXPECT_TRUE(MapperCall->getOperand(8)->getType()->isPointerTy());
4913 }
4914
4915 TEST_F(OpenMPIRBuilderTest, CreateTask) {
4916 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4917 OpenMPIRBuilder OMPBuilder(*M);
4918 OMPBuilder.initialize();
4919 F->setName("func");
4920 IRBuilder<> Builder(BB);
4921
4922 AllocaInst *ValPtr32 = Builder.CreateAlloca(Builder.getInt32Ty());
4923 AllocaInst *ValPtr128 = Builder.CreateAlloca(Builder.getInt128Ty());
4924 Value *Val128 =
4925 Builder.CreateLoad(Builder.getInt128Ty(), ValPtr128, "bodygen.load");
4926
4927 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
4928 Builder.restoreIP(AllocaIP);
4929 AllocaInst *Local128 = Builder.CreateAlloca(Builder.getInt128Ty(), nullptr,
4930 "bodygen.alloca128");
4931
4932 Builder.restoreIP(CodeGenIP);
4933 // Loading and storing captured pointer and values
4934 Builder.CreateStore(Val128, Local128);
4935 Value *Val32 = Builder.CreateLoad(ValPtr32->getAllocatedType(), ValPtr32,
4936 "bodygen.load32");
4937
4938 LoadInst *PrivLoad128 = Builder.CreateLoad(
4939 Local128->getAllocatedType(), Local128, "bodygen.local.load128");
4940 Value *Cmp = Builder.CreateICmpNE(
4941 Val32, Builder.CreateTrunc(PrivLoad128, Val32->getType()));
4942 Instruction *ThenTerm, *ElseTerm;
4943 SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(),
4944 &ThenTerm, &ElseTerm);
4945 };
4946
4947 BasicBlock *AllocaBB = Builder.GetInsertBlock();
4948 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
4949 OpenMPIRBuilder::LocationDescription Loc(
4950 InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
4951 Builder.restoreIP(OMPBuilder.createTask(
4952 Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
4953 BodyGenCB));
4954 OMPBuilder.finalize();
4955 Builder.CreateRetVoid();
4956
4957 EXPECT_FALSE(verifyModule(*M, &errs()));
4958
4959 CallInst *TaskAllocCall = dyn_cast<CallInst>(
4960 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc)
4961 ->user_back());
4962
4963 // Verify the Ident argument
4964 GlobalVariable *Ident = cast<GlobalVariable>(TaskAllocCall->getArgOperand(0));
4965 ASSERT_NE(Ident, nullptr);
4966 EXPECT_TRUE(Ident->hasInitializer());
4967 Constant *Initializer = Ident->getInitializer();
4968 GlobalVariable *SrcStrGlob =
4969 cast<GlobalVariable>(Initializer->getOperand(4)->stripPointerCasts());
4970 ASSERT_NE(SrcStrGlob, nullptr);
4971 ConstantDataArray *SrcSrc =
4972 dyn_cast<ConstantDataArray>(SrcStrGlob->getInitializer());
4973 ASSERT_NE(SrcSrc, nullptr);
4974
4975 // Verify the num_threads argument.
4976 CallInst *GTID = dyn_cast<CallInst>(TaskAllocCall->getArgOperand(1));
4977 ASSERT_NE(GTID, nullptr);
4978 EXPECT_EQ(GTID->arg_size(), 1U);
4979 EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
4980
4981 // Verify the flags
4982 // TODO: Check for others flags. Currently testing only for tiedness.
4983 ConstantInt *Flags = dyn_cast<ConstantInt>(TaskAllocCall->getArgOperand(2));
4984 ASSERT_NE(Flags, nullptr);
4985 EXPECT_EQ(Flags->getSExtValue(), 1);
4986
4987 // Verify the data size
4988 ConstantInt *DataSize =
4989 dyn_cast<ConstantInt>(TaskAllocCall->getArgOperand(3));
4990 ASSERT_NE(DataSize, nullptr);
4991 EXPECT_EQ(DataSize->getSExtValue(), 24); // 64-bit pointer + 128-bit integer
4992
4993 // TODO: Verify size of shared clause variables
4994
4995 // Verify Wrapper function
4996 Function *WrapperFunc =
4997 dyn_cast<Function>(TaskAllocCall->getArgOperand(5)->stripPointerCasts());
4998 ASSERT_NE(WrapperFunc, nullptr);
4999 EXPECT_FALSE(WrapperFunc->isDeclaration());
5000 CallInst *OutlinedFnCall = dyn_cast<CallInst>(WrapperFunc->begin()->begin());
5001 ASSERT_NE(OutlinedFnCall, nullptr);
5002 EXPECT_EQ(WrapperFunc->getArg(0)->getType(), Builder.getInt32Ty());
5003 EXPECT_EQ(OutlinedFnCall->getArgOperand(0), WrapperFunc->getArg(1));
5004
5005 // Verify the presence of `trunc` and `icmp` instructions in Outlined function
5006 Function *OutlinedFn = OutlinedFnCall->getCalledFunction();
5007 ASSERT_NE(OutlinedFn, nullptr);
5008 EXPECT_TRUE(any_of(instructions(OutlinedFn),
5009 [](Instruction &inst) { return isa<TruncInst>(&inst); }));
5010 EXPECT_TRUE(any_of(instructions(OutlinedFn),
5011 [](Instruction &inst) { return isa<ICmpInst>(&inst); }));
5012
5013 // Verify the execution of the task
5014 CallInst *TaskCall = dyn_cast<CallInst>(
5015 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task)
5016 ->user_back());
5017 ASSERT_NE(TaskCall, nullptr);
5018 EXPECT_EQ(TaskCall->getArgOperand(0), Ident);
5019 EXPECT_EQ(TaskCall->getArgOperand(1), GTID);
5020 EXPECT_EQ(TaskCall->getArgOperand(2), TaskAllocCall);
5021
5022 // Verify that the argument data has been copied
5023 for (User *in : TaskAllocCall->users()) {
5024 if (MemCpyInst *memCpyInst = dyn_cast<MemCpyInst>(in)) {
5025 EXPECT_EQ(memCpyInst->getDest(), TaskAllocCall);
5026 }
5027 }
5028 }
5029
5030 TEST_F(OpenMPIRBuilderTest, CreateTaskNoArgs) {
5031 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
5032 OpenMPIRBuilder OMPBuilder(*M);
5033 OMPBuilder.initialize();
5034 F->setName("func");
5035 IRBuilder<> Builder(BB);
5036
5037 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
5038
5039 BasicBlock *AllocaBB = Builder.GetInsertBlock();
5040 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
5041 OpenMPIRBuilder::LocationDescription Loc(
5042 InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
5043 Builder.restoreIP(OMPBuilder.createTask(
5044 Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
5045 BodyGenCB));
5046 OMPBuilder.finalize();
5047 Builder.CreateRetVoid();
5048
5049 EXPECT_FALSE(verifyModule(*M, &errs()));
5050 }
5051
5052 TEST_F(OpenMPIRBuilderTest, CreateTaskUntied) {
5053 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
5054 OpenMPIRBuilder OMPBuilder(*M);
5055 OMPBuilder.initialize();
5056 F->setName("func");
5057 IRBuilder<> Builder(BB);
5058 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
5059 BasicBlock *AllocaBB = Builder.GetInsertBlock();
5060 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
5061 OpenMPIRBuilder::LocationDescription Loc(
5062 InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
5063 Builder.restoreIP(OMPBuilder.createTask(
5064 Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()), BodyGenCB,
5065 /*Tied=*/false));
5066 OMPBuilder.finalize();
5067 Builder.CreateRetVoid();
5068
5069 // Check for the `Tied` argument
5070 CallInst *TaskAllocCall = dyn_cast<CallInst>(
5071 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc)
5072 ->user_back());
5073 ASSERT_NE(TaskAllocCall, nullptr);
5074 ConstantInt *Flags = dyn_cast<ConstantInt>(TaskAllocCall->getArgOperand(2));
5075 ASSERT_NE(Flags, nullptr);
5076 EXPECT_EQ(Flags->getZExtValue() & 1U, 0U);
5077
5078 EXPECT_FALSE(verifyModule(*M, &errs()));
5079 }
5080
5081 TEST_F(OpenMPIRBuilderTest, CreateTaskDepend) {
5082 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
5083 OpenMPIRBuilder OMPBuilder(*M);
5084 OMPBuilder.initialize();
5085 F->setName("func");
5086 IRBuilder<> Builder(BB);
5087 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
5088 BasicBlock *AllocaBB = Builder.GetInsertBlock();
5089 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
5090 OpenMPIRBuilder::LocationDescription Loc(
5091 InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
5092 AllocaInst *InDep = Builder.CreateAlloca(Type::getInt32Ty(M->getContext()));
5093 SmallVector<OpenMPIRBuilder::DependData> DDS;
5094 {
5095 OpenMPIRBuilder::DependData DDIn(RTLDependenceKindTy::DepIn,
5096 Type::getInt32Ty(M->getContext()), InDep);
5097 DDS.push_back(DDIn);
5098 }
5099 Builder.restoreIP(OMPBuilder.createTask(
5100 Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()), BodyGenCB,
5101 /*Tied=*/false, /*Final*/ nullptr, /*IfCondition*/ nullptr, DDS));
5102 OMPBuilder.finalize();
5103 Builder.CreateRetVoid();
5104
5105 // Check for the `NumDeps` argument
5106 CallInst *TaskAllocCall = dyn_cast<CallInst>(
5107 OMPBuilder
5108 .getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_with_deps)
5109 ->user_back());
5110 ASSERT_NE(TaskAllocCall, nullptr);
5111 ConstantInt *NumDeps = dyn_cast<ConstantInt>(TaskAllocCall->getArgOperand(3));
5112 ASSERT_NE(NumDeps, nullptr);
5113 EXPECT_EQ(NumDeps->getZExtValue(), 1U);
5114
5115 // Check for the `DepInfo` array argument
5116 BitCastInst *DepArrayPtr =
5117 dyn_cast<BitCastInst>(TaskAllocCall->getOperand(4));
5118 ASSERT_NE(DepArrayPtr, nullptr);
5119 AllocaInst *DepArray = dyn_cast<AllocaInst>(DepArrayPtr->getOperand(0));
5120 ASSERT_NE(DepArray, nullptr);
5121 Value::user_iterator DepArrayI = DepArray->user_begin();
5122 EXPECT_EQ(*DepArrayI, DepArrayPtr);
5123 ++DepArrayI;
5124 Value::user_iterator DepInfoI = DepArrayI->user_begin();
5125 // Check for the `DependKind` flag in the `DepInfo` array
5126 Value *Flag = findStoredValue<GetElementPtrInst>(*DepInfoI);
5127 ASSERT_NE(Flag, nullptr);
5128 ConstantInt *FlagInt = dyn_cast<ConstantInt>(Flag);
5129 ASSERT_NE(FlagInt, nullptr);
5130 EXPECT_EQ(FlagInt->getZExtValue(),
5131 static_cast<unsigned int>(RTLDependenceKindTy::DepIn));
5132 ++DepInfoI;
5133 // Check for the size in the `DepInfo` array
5134 Value *Size = findStoredValue<GetElementPtrInst>(*DepInfoI);
5135 ASSERT_NE(Size, nullptr);
5136 ConstantInt *SizeInt = dyn_cast<ConstantInt>(Size);
5137 ASSERT_NE(SizeInt, nullptr);
5138 EXPECT_EQ(SizeInt->getZExtValue(), 4U);
5139 ++DepInfoI;
5140 // Check for the variable address in the `DepInfo` array
5141 Value *AddrStored = findStoredValue<GetElementPtrInst>(*DepInfoI);
5142 ASSERT_NE(AddrStored, nullptr);
5143 PtrToIntInst *AddrInt = dyn_cast<PtrToIntInst>(AddrStored);
5144 ASSERT_NE(AddrInt, nullptr);
5145 Value *Addr = AddrInt->getPointerOperand();
5146 EXPECT_EQ(Addr, InDep);
5147
5148 ConstantInt *NumDepsNoAlias =
5149 dyn_cast<ConstantInt>(TaskAllocCall->getArgOperand(5));
5150 ASSERT_NE(NumDepsNoAlias, nullptr);
5151 EXPECT_EQ(NumDepsNoAlias->getZExtValue(), 0U);
5152 EXPECT_EQ(TaskAllocCall->getOperand(6),
5153 ConstantPointerNull::get(Type::getInt8PtrTy(M->getContext())));
5154
5155 EXPECT_FALSE(verifyModule(*M, &errs()));
5156 }
5157
5158 TEST_F(OpenMPIRBuilderTest, CreateTaskFinal) {
5159 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
5160 OpenMPIRBuilder OMPBuilder(*M);
5161 OMPBuilder.initialize();
5162 F->setName("func");
5163 IRBuilder<> Builder(BB);
5164 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
5165 IRBuilderBase::InsertPoint AllocaIP = Builder.saveIP();
5166 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
5167 Builder.SetInsertPoint(BodyBB);
5168 Value *Final = Builder.CreateICmp(
5169 CmpInst::Predicate::ICMP_EQ, F->getArg(0),
5170 ConstantInt::get(Type::getInt32Ty(M->getContext()), 0U));
5171 OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL);
5172 Builder.restoreIP(OMPBuilder.createTask(Loc, AllocaIP, BodyGenCB,
5173 /*Tied=*/false, Final));
5174 OMPBuilder.finalize();
5175 Builder.CreateRetVoid();
5176
5177 // Check for the `Tied` argument
5178 CallInst *TaskAllocCall = dyn_cast<CallInst>(
5179 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc)
5180 ->user_back());
5181 ASSERT_NE(TaskAllocCall, nullptr);
5182 BinaryOperator *OrInst =
5183 dyn_cast<BinaryOperator>(TaskAllocCall->getArgOperand(2));
5184 ASSERT_NE(OrInst, nullptr);
5185 EXPECT_EQ(OrInst->getOpcode(), BinaryOperator::BinaryOps::Or);
5186
5187 // One of the arguments to `or` instruction is the tied flag, which is equal
5188 // to zero.
5189 EXPECT_TRUE(any_of(OrInst->operands(), [](Value *op) {
5190 if (ConstantInt *TiedValue = dyn_cast<ConstantInt>(op))
5191 return TiedValue->getSExtValue() == 0;
5192 return false;
5193 }));
5194
5195 // One of the arguments to `or` instruction is the final condition.
5196 EXPECT_TRUE(any_of(OrInst->operands(), [Final](Value *op) {
5197 if (SelectInst *Select = dyn_cast<SelectInst>(op)) {
5198 ConstantInt *TrueValue = dyn_cast<ConstantInt>(Select->getTrueValue());
5199 ConstantInt *FalseValue = dyn_cast<ConstantInt>(Select->getFalseValue());
5200 if (!TrueValue || !FalseValue)
5201 return false;
5202 return Select->getCondition() == Final &&
5203 TrueValue->getSExtValue() == 2 && FalseValue->getSExtValue() == 0;
5204 }
5205 return false;
5206 }));
5207
5208 EXPECT_FALSE(verifyModule(*M, &errs()));
5209 }
5210
5211 TEST_F(OpenMPIRBuilderTest, CreateTaskIfCondition) {
5212 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
5213 OpenMPIRBuilder OMPBuilder(*M);
5214 OMPBuilder.initialize();
5215 F->setName("func");
5216 IRBuilder<> Builder(BB);
5217 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
5218 IRBuilderBase::InsertPoint AllocaIP = Builder.saveIP();
5219 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
5220 Builder.SetInsertPoint(BodyBB);
5221 Value *IfCondition = Builder.CreateICmp(
5222 CmpInst::Predicate::ICMP_EQ, F->getArg(0),
5223 ConstantInt::get(Type::getInt32Ty(M->getContext()), 0U));
5224 OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL);
5225 Builder.restoreIP(OMPBuilder.createTask(Loc, AllocaIP, BodyGenCB,
5226 /*Tied=*/false, /*Final=*/nullptr,
5227 IfCondition));
5228 OMPBuilder.finalize();
5229 Builder.CreateRetVoid();
5230
5231 EXPECT_FALSE(verifyModule(*M, &errs()));
5232
5233 CallInst *TaskAllocCall = dyn_cast<CallInst>(
5234 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc)
5235 ->user_back());
5236 ASSERT_NE(TaskAllocCall, nullptr);
5237
5238 // Check the branching is based on the if condition argument.
5239 BranchInst *IfConditionBranchInst =
5240 dyn_cast<BranchInst>(TaskAllocCall->getParent()->getTerminator());
5241 ASSERT_NE(IfConditionBranchInst, nullptr);
5242 ASSERT_TRUE(IfConditionBranchInst->isConditional());
5243 EXPECT_EQ(IfConditionBranchInst->getCondition(), IfCondition);
5244
5245 // Check that the `__kmpc_omp_task` executes only in the then branch.
5246 CallInst *TaskCall = dyn_cast<CallInst>(
5247 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task)
5248 ->user_back());
5249 ASSERT_NE(TaskCall, nullptr);
5250 EXPECT_EQ(TaskCall->getParent(), IfConditionBranchInst->getSuccessor(0));
5251
5252 // Check that the OpenMP Runtime Functions specific to `if` clause execute
5253 // only in the else branch. Also check that the function call is between the
5254 // `__kmpc_omp_task_begin_if0` and `__kmpc_omp_task_complete_if0` calls.
5255 CallInst *TaskBeginIfCall = dyn_cast<CallInst>(
5256 OMPBuilder
5257 .getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_begin_if0)
5258 ->user_back());
5259 CallInst *TaskCompleteCall = dyn_cast<CallInst>(
5260 OMPBuilder
5261 .getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_complete_if0)
5262 ->user_back());
5263 ASSERT_NE(TaskBeginIfCall, nullptr);
5264 ASSERT_NE(TaskCompleteCall, nullptr);
5265 Function *WrapperFunc =
5266 dyn_cast<Function>(TaskAllocCall->getArgOperand(5)->stripPointerCasts());
5267 ASSERT_NE(WrapperFunc, nullptr);
5268 CallInst *WrapperFuncCall = dyn_cast<CallInst>(WrapperFunc->user_back());
5269 ASSERT_NE(WrapperFuncCall, nullptr);
5270 EXPECT_EQ(TaskBeginIfCall->getParent(),
5271 IfConditionBranchInst->getSuccessor(1));
5272 EXPECT_EQ(TaskBeginIfCall->getNextNonDebugInstruction(), WrapperFuncCall);
5273 EXPECT_EQ(WrapperFuncCall->getNextNonDebugInstruction(), TaskCompleteCall);
5274 }
5275
5276 TEST_F(OpenMPIRBuilderTest, CreateTaskgroup) {
5277 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
5278 OpenMPIRBuilder OMPBuilder(*M);
5279 OMPBuilder.initialize();
5280 F->setName("func");
5281 IRBuilder<> Builder(BB);
5282
5283 AllocaInst *ValPtr32 = Builder.CreateAlloca(Builder.getInt32Ty());
5284 AllocaInst *ValPtr128 = Builder.CreateAlloca(Builder.getInt128Ty());
5285 Value *Val128 =
5286 Builder.CreateLoad(Builder.getInt128Ty(), ValPtr128, "bodygen.load");
5287 Instruction *ThenTerm, *ElseTerm;
5288
5289 Value *InternalStoreInst, *InternalLoad32, *InternalLoad128, *InternalIfCmp;
5290
5291 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
5292 Builder.restoreIP(AllocaIP);
5293 AllocaInst *Local128 = Builder.CreateAlloca(Builder.getInt128Ty(), nullptr,
5294 "bodygen.alloca128");
5295
5296 Builder.restoreIP(CodeGenIP);
5297 // Loading and storing captured pointer and values
5298 InternalStoreInst = Builder.CreateStore(Val128, Local128);
5299 InternalLoad32 = Builder.CreateLoad(ValPtr32->getAllocatedType(), ValPtr32,
5300 "bodygen.load32");
5301
5302 InternalLoad128 = Builder.CreateLoad(Local128->getAllocatedType(), Local128,
5303 "bodygen.local.load128");
5304 InternalIfCmp = Builder.CreateICmpNE(
5305 InternalLoad32,
5306 Builder.CreateTrunc(InternalLoad128, InternalLoad32->getType()));
5307 SplitBlockAndInsertIfThenElse(InternalIfCmp,
5308 CodeGenIP.getBlock()->getTerminator(),
5309 &ThenTerm, &ElseTerm);
5310 };
5311
5312 BasicBlock *AllocaBB = Builder.GetInsertBlock();
5313 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
5314 OpenMPIRBuilder::LocationDescription Loc(
5315 InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
5316 Builder.restoreIP(OMPBuilder.createTaskgroup(
5317 Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
5318 BodyGenCB));
5319 OMPBuilder.finalize();
5320 Builder.CreateRetVoid();
5321
5322 EXPECT_FALSE(verifyModule(*M, &errs()));
5323
5324 CallInst *TaskgroupCall = dyn_cast<CallInst>(
5325 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_taskgroup)
5326 ->user_back());
5327 ASSERT_NE(TaskgroupCall, nullptr);
5328 CallInst *EndTaskgroupCall = dyn_cast<CallInst>(
5329 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_taskgroup)
5330 ->user_back());
5331 ASSERT_NE(EndTaskgroupCall, nullptr);
5332
5333 // Verify the Ident argument
5334 GlobalVariable *Ident = cast<GlobalVariable>(TaskgroupCall->getArgOperand(0));
5335 ASSERT_NE(Ident, nullptr);
5336 EXPECT_TRUE(Ident->hasInitializer());
5337 Constant *Initializer = Ident->getInitializer();
5338 GlobalVariable *SrcStrGlob =
5339 cast<GlobalVariable>(Initializer->getOperand(4)->stripPointerCasts());
5340 ASSERT_NE(SrcStrGlob, nullptr);
5341 ConstantDataArray *SrcSrc =
5342 dyn_cast<ConstantDataArray>(SrcStrGlob->getInitializer());
5343 ASSERT_NE(SrcSrc, nullptr);
5344
5345 // Verify the num_threads argument.
5346 CallInst *GTID = dyn_cast<CallInst>(TaskgroupCall->getArgOperand(1));
5347 ASSERT_NE(GTID, nullptr);
5348 EXPECT_EQ(GTID->arg_size(), 1U);
5349 EXPECT_EQ(GTID->getCalledFunction(), OMPBuilder.getOrCreateRuntimeFunctionPtr(
5350 OMPRTL___kmpc_global_thread_num));
5351
5352 // Checking the general structure of the IR generated is same as expected.
5353 Instruction *GeneratedStoreInst = TaskgroupCall->getNextNonDebugInstruction();
5354 EXPECT_EQ(GeneratedStoreInst, InternalStoreInst);
5355 Instruction *GeneratedLoad32 =
5356 GeneratedStoreInst->getNextNonDebugInstruction();
5357 EXPECT_EQ(GeneratedLoad32, InternalLoad32);
5358 Instruction *GeneratedLoad128 = GeneratedLoad32->getNextNonDebugInstruction();
5359 EXPECT_EQ(GeneratedLoad128, InternalLoad128);
5360
5361 // Checking the ordering because of the if statements and that
5362 // `__kmp_end_taskgroup` call is after the if branching.
5363 BasicBlock *RefOrder[] = {TaskgroupCall->getParent(), ThenTerm->getParent(),
5364 ThenTerm->getSuccessor(0),
5365 EndTaskgroupCall->getParent(),
5366 ElseTerm->getParent()};
5367 verifyDFSOrder(F, RefOrder);
5368 }
5369
5370 TEST_F(OpenMPIRBuilderTest, CreateTaskgroupWithTasks) {
5371 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
5372 OpenMPIRBuilder OMPBuilder(*M);
5373 OMPBuilder.initialize();
5374 F->setName("func");
5375 IRBuilder<> Builder(BB);
5376
5377 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
5378 Builder.restoreIP(AllocaIP);
5379 AllocaInst *Alloca32 =
5380 Builder.CreateAlloca(Builder.getInt32Ty(), nullptr, "bodygen.alloca32");
5381 AllocaInst *Alloca64 =
5382 Builder.CreateAlloca(Builder.getInt64Ty(), nullptr, "bodygen.alloca64");
5383 Builder.restoreIP(CodeGenIP);
5384 auto TaskBodyGenCB1 = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
5385 Builder.restoreIP(CodeGenIP);
5386 LoadInst *LoadValue =
5387 Builder.CreateLoad(Alloca64->getAllocatedType(), Alloca64);
5388 Value *AddInst = Builder.CreateAdd(LoadValue, Builder.getInt64(64));
5389 Builder.CreateStore(AddInst, Alloca64);
5390 };
5391 OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL);
5392 Builder.restoreIP(OMPBuilder.createTask(Loc, AllocaIP, TaskBodyGenCB1));
5393
5394 auto TaskBodyGenCB2 = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
5395 Builder.restoreIP(CodeGenIP);
5396 LoadInst *LoadValue =
5397 Builder.CreateLoad(Alloca32->getAllocatedType(), Alloca32);
5398 Value *AddInst = Builder.CreateAdd(LoadValue, Builder.getInt32(32));
5399 Builder.CreateStore(AddInst, Alloca32);
5400 };
5401 OpenMPIRBuilder::LocationDescription Loc2(Builder.saveIP(), DL);
5402 Builder.restoreIP(OMPBuilder.createTask(Loc2, AllocaIP, TaskBodyGenCB2));
5403 };
5404
5405 BasicBlock *AllocaBB = Builder.GetInsertBlock();
5406 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
5407 OpenMPIRBuilder::LocationDescription Loc(
5408 InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
5409 Builder.restoreIP(OMPBuilder.createTaskgroup(
5410 Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
5411 BodyGenCB));
5412 OMPBuilder.finalize();
5413 Builder.CreateRetVoid();
5414
5415 EXPECT_FALSE(verifyModule(*M, &errs()));
5416
5417 CallInst *TaskgroupCall = dyn_cast<CallInst>(
5418 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_taskgroup)
5419 ->user_back());
5420 ASSERT_NE(TaskgroupCall, nullptr);
5421 CallInst *EndTaskgroupCall = dyn_cast<CallInst>(
5422 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_taskgroup)
5423 ->user_back());
5424 ASSERT_NE(EndTaskgroupCall, nullptr);
5425
5426 Function *TaskAllocFn =
5427 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc);
5428 ASSERT_EQ(TaskAllocFn->getNumUses(), 2u);
5429
5430 CallInst *FirstTaskAllocCall =
5431 dyn_cast_or_null<CallInst>(*TaskAllocFn->users().begin());
5432 CallInst *SecondTaskAllocCall =
5433 dyn_cast_or_null<CallInst>(*TaskAllocFn->users().begin()++);
5434 ASSERT_NE(FirstTaskAllocCall, nullptr);
5435 ASSERT_NE(SecondTaskAllocCall, nullptr);
5436
5437 // Verify that the tasks have been generated in order and inside taskgroup
5438 // construct.
5439 BasicBlock *RefOrder[] = {
5440 TaskgroupCall->getParent(), FirstTaskAllocCall->getParent(),
5441 SecondTaskAllocCall->getParent(), EndTaskgroupCall->getParent()};
5442 verifyDFSOrder(F, RefOrder);
5443 }
5444
5445 TEST_F(OpenMPIRBuilderTest, EmitOffloadingArraysArguments) {
5446 OpenMPIRBuilder OMPBuilder(*M);
5447 OMPBuilder.initialize();
5448
5449 IRBuilder<> Builder(BB);
5450
5451 OpenMPIRBuilder::TargetDataRTArgs RTArgs;
5452 OpenMPIRBuilder::TargetDataInfo Info(true, false);
5453
5454 auto VoidPtrTy = Type::getInt8PtrTy(Builder.getContext());
5455 auto VoidPtrPtrTy = VoidPtrTy->getPointerTo(0);
5456 auto Int64Ty = Type::getInt64Ty(Builder.getContext());
5457 auto Int64PtrTy = Type::getInt64PtrTy(Builder.getContext());
5458 auto Array4VoidPtrTy = ArrayType::get(VoidPtrTy, 4);
5459 auto Array4Int64PtrTy = ArrayType::get(Int64Ty, 4);
5460
5461 Info.RTArgs.BasePointersArray =
5462 ConstantPointerNull::get(Array4VoidPtrTy->getPointerTo(0));
5463 Info.RTArgs.PointersArray =
5464 ConstantPointerNull::get(Array4VoidPtrTy->getPointerTo());
5465 Info.RTArgs.SizesArray =
5466 ConstantPointerNull::get(Array4Int64PtrTy->getPointerTo());
5467 Info.RTArgs.MapTypesArray =
5468 ConstantPointerNull::get(Array4Int64PtrTy->getPointerTo());
5469 Info.RTArgs.MapNamesArray =
5470 ConstantPointerNull::get(Array4VoidPtrTy->getPointerTo());
5471 Info.RTArgs.MappersArray =
5472 ConstantPointerNull::get(Array4VoidPtrTy->getPointerTo());
5473 Info.NumberOfPtrs = 4;
5474
5475 OMPBuilder.emitOffloadingArraysArgument(Builder, RTArgs, Info, false, false);
5476
5477 EXPECT_NE(RTArgs.BasePointersArray, nullptr);
5478 EXPECT_NE(RTArgs.PointersArray, nullptr);
5479 EXPECT_NE(RTArgs.SizesArray, nullptr);
5480 EXPECT_NE(RTArgs.MapTypesArray, nullptr);
5481 EXPECT_NE(RTArgs.MappersArray, nullptr);
5482 EXPECT_NE(RTArgs.MapNamesArray, nullptr);
5483 EXPECT_EQ(RTArgs.MapTypesArrayEnd, nullptr);
5484
5485 EXPECT_EQ(RTArgs.BasePointersArray->getType(), VoidPtrPtrTy);
5486 EXPECT_EQ(RTArgs.PointersArray->getType(), VoidPtrPtrTy);
5487 EXPECT_EQ(RTArgs.SizesArray->getType(), Int64PtrTy);
5488 EXPECT_EQ(RTArgs.MapTypesArray->getType(), Int64PtrTy);
5489 EXPECT_EQ(RTArgs.MappersArray->getType(), VoidPtrPtrTy);
5490 EXPECT_EQ(RTArgs.MapNamesArray->getType(), VoidPtrPtrTy);
5491 }
5492
5493 TEST_F(OpenMPIRBuilderTest, OffloadEntriesInfoManager) {
5494 OffloadEntriesInfoManager InfoManager;
5495 InfoManager.setConfig(OpenMPIRBuilderConfig(true, false, false, false));
5496 TargetRegionEntryInfo EntryInfo("parent", 1, 2, 4, 0);
5497 InfoManager.initializeTargetRegionEntryInfo(EntryInfo, 0);
5498 EXPECT_TRUE(InfoManager.hasTargetRegionEntryInfo(EntryInfo));
5499 InfoManager.initializeDeviceGlobalVarEntryInfo(
5500 "gvar", OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo, 0);
5501 InfoManager.registerTargetRegionEntryInfo(
5502 EntryInfo, nullptr, nullptr,
5503 OffloadEntriesInfoManager::OMPTargetRegionEntryTargetRegion);
5504 InfoManager.registerDeviceGlobalVarEntryInfo(
5505 "gvar", 0x0, 8, OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo,
5506 GlobalValue::WeakAnyLinkage);
5507 EXPECT_TRUE(InfoManager.hasDeviceGlobalVarEntryInfo("gvar"));
5508 }
5509 } // namespace