Debian Codegraph @

llvm-project/llvm/unittests/ProfileData/InstrProfTest.cpp

raw
1 //===- unittest/ProfileData/InstrProfTest.cpp -------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/IR/Function.h"
10 #include "llvm/IR/IRBuilder.h"
11 #include "llvm/IR/LLVMContext.h"
12 #include "llvm/IR/Module.h"
13 #include "llvm/ProfileData/InstrProfReader.h"
14 #include "llvm/ProfileData/InstrProfWriter.h"
15 #include "llvm/ProfileData/MemProf.h"
16 #include "llvm/ProfileData/MemProfData.inc"
17 #include "llvm/Support/Compression.h"
18 #include "llvm/Support/raw_ostream.h"
19 #include "llvm/Testing/Support/Error.h"
20 #include "llvm/Testing/Support/SupportHelpers.h"
21 #include "gtest/gtest.h"
22 #include <cstdarg>
23
24 using namespace llvm;
25
26 [[nodiscard]] static ::testing::AssertionResult
27 ErrorEquals(instrprof_error Expected, Error E) {
28 instrprof_error Found;
29 std::string FoundMsg;
30 handleAllErrors(std::move(E), [&](const InstrProfError &IPE) {
31 Found = IPE.get();
32 FoundMsg = IPE.message();
33 });
34 if (Expected == Found)
35 return ::testing::AssertionSuccess();
36 return ::testing::AssertionFailure() << "error: " << FoundMsg << "\n";
37 }
38
39 namespace {
40
41 struct InstrProfTest : ::testing::Test {
42 InstrProfWriter Writer;
43 std::unique_ptr<IndexedInstrProfReader> Reader;
44
45 void SetUp() override { Writer.setOutputSparse(false); }
46
47 void readProfile(std::unique_ptr<MemoryBuffer> Profile,
48 std::unique_ptr<MemoryBuffer> Remapping = nullptr) {
49 auto ReaderOrErr = IndexedInstrProfReader::create(std::move(Profile),
50 std::move(Remapping));
51 EXPECT_THAT_ERROR(ReaderOrErr.takeError(), Succeeded());
52 Reader = std::move(ReaderOrErr.get());
53 }
54 };
55
56 struct SparseInstrProfTest : public InstrProfTest {
57 void SetUp() override { Writer.setOutputSparse(true); }
58 };
59
60 struct MaybeSparseInstrProfTest : public InstrProfTest,
61 public ::testing::WithParamInterface<bool> {
62 void SetUp() override { Writer.setOutputSparse(GetParam()); }
63 };
64
65 TEST_P(MaybeSparseInstrProfTest, write_and_read_empty_profile) {
66 auto Profile = Writer.writeBuffer();
67 readProfile(std::move(Profile));
68 ASSERT_TRUE(Reader->begin() == Reader->end());
69 }
70
71 static const auto Err = [](Error E) {
72 consumeError(std::move(E));
73 FAIL();
74 };
75
76 TEST_P(MaybeSparseInstrProfTest, write_and_read_one_function) {
77 Writer.addRecord({"foo", 0x1234, {1, 2, 3, 4}}, Err);
78 auto Profile = Writer.writeBuffer();
79 readProfile(std::move(Profile));
80
81 auto I = Reader->begin(), E = Reader->end();
82 ASSERT_TRUE(I != E);
83 ASSERT_EQ(StringRef("foo"), I->Name);
84 ASSERT_EQ(0x1234U, I->Hash);
85 ASSERT_EQ(4U, I->Counts.size());
86 ASSERT_EQ(1U, I->Counts[0]);
87 ASSERT_EQ(2U, I->Counts[1]);
88 ASSERT_EQ(3U, I->Counts[2]);
89 ASSERT_EQ(4U, I->Counts[3]);
90 ASSERT_TRUE(++I == E);
91 }
92
93 TEST_P(MaybeSparseInstrProfTest, get_instr_prof_record) {
94 Writer.addRecord({"foo", 0x1234, {1, 2}}, Err);
95 Writer.addRecord({"foo", 0x1235, {3, 4}}, Err);
96 auto Profile = Writer.writeBuffer();
97 readProfile(std::move(Profile));
98
99 Expected<InstrProfRecord> R = Reader->getInstrProfRecord("foo", 0x1234);
100 EXPECT_THAT_ERROR(R.takeError(), Succeeded());
101 ASSERT_EQ(2U, R->Counts.size());
102 ASSERT_EQ(1U, R->Counts[0]);
103 ASSERT_EQ(2U, R->Counts[1]);
104
105 R = Reader->getInstrProfRecord("foo", 0x1235);
106 EXPECT_THAT_ERROR(R.takeError(), Succeeded());
107 ASSERT_EQ(2U, R->Counts.size());
108 ASSERT_EQ(3U, R->Counts[0]);
109 ASSERT_EQ(4U, R->Counts[1]);
110
111 R = Reader->getInstrProfRecord("foo", 0x5678);
112 ASSERT_TRUE(ErrorEquals(instrprof_error::hash_mismatch, R.takeError()));
113
114 R = Reader->getInstrProfRecord("bar", 0x1234);
115 ASSERT_TRUE(ErrorEquals(instrprof_error::unknown_function, R.takeError()));
116 }
117
118 TEST_P(MaybeSparseInstrProfTest, get_function_counts) {
119 Writer.addRecord({"foo", 0x1234, {1, 2}}, Err);
120 Writer.addRecord({"foo", 0x1235, {3, 4}}, Err);
121 auto Profile = Writer.writeBuffer();
122 readProfile(std::move(Profile));
123
124 std::vector<uint64_t> Counts;
125 EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo", 0x1234, Counts),
126 Succeeded());
127 ASSERT_EQ(2U, Counts.size());
128 ASSERT_EQ(1U, Counts[0]);
129 ASSERT_EQ(2U, Counts[1]);
130
131 EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo", 0x1235, Counts),
132 Succeeded());
133 ASSERT_EQ(2U, Counts.size());
134 ASSERT_EQ(3U, Counts[0]);
135 ASSERT_EQ(4U, Counts[1]);
136
137 Error E1 = Reader->getFunctionCounts("foo", 0x5678, Counts);
138 ASSERT_TRUE(ErrorEquals(instrprof_error::hash_mismatch, std::move(E1)));
139
140 Error E2 = Reader->getFunctionCounts("bar", 0x1234, Counts);
141 ASSERT_TRUE(ErrorEquals(instrprof_error::unknown_function, std::move(E2)));
142 }
143
144 // Profile data is copied from general.proftext
145 TEST_F(InstrProfTest, get_profile_summary) {
146 Writer.addRecord({"func1", 0x1234, {97531}}, Err);
147 Writer.addRecord({"func2", 0x1234, {0, 0}}, Err);
148 Writer.addRecord(
149 {"func3",
150 0x1234,
151 {2305843009213693952, 1152921504606846976, 576460752303423488,
152 288230376151711744, 144115188075855872, 72057594037927936}},
153 Err);
154 Writer.addRecord({"func4", 0x1234, {0}}, Err);
155 auto Profile = Writer.writeBuffer();
156 readProfile(std::move(Profile));
157
158 auto VerifySummary = [](ProfileSummary &IPS) mutable {
159 ASSERT_EQ(ProfileSummary::PSK_Instr, IPS.getKind());
160 ASSERT_EQ(2305843009213693952U, IPS.getMaxFunctionCount());
161 ASSERT_EQ(2305843009213693952U, IPS.getMaxCount());
162 ASSERT_EQ(10U, IPS.getNumCounts());
163 ASSERT_EQ(4539628424389557499U, IPS.getTotalCount());
164 const std::vector<ProfileSummaryEntry> &Details = IPS.getDetailedSummary();
165 uint32_t Cutoff = 800000;
166 auto Predicate = [&Cutoff](const ProfileSummaryEntry &PE) {
167 return PE.Cutoff == Cutoff;
168 };
169 auto EightyPerc = find_if(Details, Predicate);
170 Cutoff = 900000;
171 auto NinetyPerc = find_if(Details, Predicate);
172 Cutoff = 950000;
173 auto NinetyFivePerc = find_if(Details, Predicate);
174 Cutoff = 990000;
175 auto NinetyNinePerc = find_if(Details, Predicate);
176 ASSERT_EQ(576460752303423488U, EightyPerc->MinCount);
177 ASSERT_EQ(288230376151711744U, NinetyPerc->MinCount);
178 ASSERT_EQ(288230376151711744U, NinetyFivePerc->MinCount);
179 ASSERT_EQ(72057594037927936U, NinetyNinePerc->MinCount);
180 };
181 ProfileSummary &PS = Reader->getSummary(/* IsCS */ false);
182 VerifySummary(PS);
183
184 // Test that conversion of summary to and from Metadata works.
185 LLVMContext Context;
186 Metadata *MD = PS.getMD(Context);
187 ASSERT_TRUE(MD);
188 ProfileSummary *PSFromMD = ProfileSummary::getFromMD(MD);
189 ASSERT_TRUE(PSFromMD);
190 VerifySummary(*PSFromMD);
191 delete PSFromMD;
192
193 // Test that summary can be attached to and read back from module.
194 Module M("my_module", Context);
195 M.setProfileSummary(MD, ProfileSummary::PSK_Instr);
196 MD = M.getProfileSummary(/* IsCS */ false);
197 ASSERT_TRUE(MD);
198 PSFromMD = ProfileSummary::getFromMD(MD);
199 ASSERT_TRUE(PSFromMD);
200 VerifySummary(*PSFromMD);
201 delete PSFromMD;
202 }
203
204 TEST_F(InstrProfTest, test_writer_merge) {
205 Writer.addRecord({"func1", 0x1234, {42}}, Err);
206
207 InstrProfWriter Writer2;
208 Writer2.addRecord({"func2", 0x1234, {0, 0}}, Err);
209
210 Writer.mergeRecordsFromWriter(std::move(Writer2), Err);
211
212 auto Profile = Writer.writeBuffer();
213 readProfile(std::move(Profile));
214
215 Expected<InstrProfRecord> R = Reader->getInstrProfRecord("func1", 0x1234);
216 EXPECT_THAT_ERROR(R.takeError(), Succeeded());
217 ASSERT_EQ(1U, R->Counts.size());
218 ASSERT_EQ(42U, R->Counts[0]);
219
220 R = Reader->getInstrProfRecord("func2", 0x1234);
221 EXPECT_THAT_ERROR(R.takeError(), Succeeded());
222 ASSERT_EQ(2U, R->Counts.size());
223 ASSERT_EQ(0U, R->Counts[0]);
224 ASSERT_EQ(0U, R->Counts[1]);
225 }
226
227 using ::llvm::memprof::IndexedMemProfRecord;
228 using ::llvm::memprof::MemInfoBlock;
229 using FrameIdMapTy =
230 llvm::DenseMap<::llvm::memprof::FrameId, ::llvm::memprof::Frame>;
231
232 static FrameIdMapTy getFrameMapping() {
233 FrameIdMapTy Mapping;
234 Mapping.insert({0, {0x123, 1, 2, false}});
235 Mapping.insert({1, {0x345, 3, 4, true}});
236 Mapping.insert({2, {0x125, 5, 6, false}});
237 Mapping.insert({3, {0x567, 7, 8, true}});
238 Mapping.insert({4, {0x124, 5, 6, false}});
239 Mapping.insert({5, {0x789, 8, 9, true}});
240 return Mapping;
241 }
242
243 IndexedMemProfRecord makeRecord(
244 std::initializer_list<std::initializer_list<::llvm::memprof::FrameId>>
245 AllocFrames,
246 std::initializer_list<std::initializer_list<::llvm::memprof::FrameId>>
247 CallSiteFrames,
248 const MemInfoBlock &Block = MemInfoBlock()) {
249 llvm::memprof::IndexedMemProfRecord MR;
250 for (const auto &Frames : AllocFrames)
251 MR.AllocSites.emplace_back(Frames, Block);
252 for (const auto &Frames : CallSiteFrames)
253 MR.CallSites.push_back(Frames);
254 return MR;
255 }
256
257 MATCHER_P(EqualsRecord, Want, "") {
258 const memprof::MemProfRecord &Got = arg;
259
260 auto PrintAndFail = [&]() {
261 std::string Buffer;
262 llvm::raw_string_ostream OS(Buffer);
263 OS << "Want:\n";
264 Want.print(OS);
265 OS << "Got:\n";
266 Got.print(OS);
267 OS.flush();
268 *result_listener << "MemProf Record differs!\n" << Buffer;
269 return false;
270 };
271
272 if (Want.AllocSites.size() != Got.AllocSites.size())
273 return PrintAndFail();
274 if (Want.CallSites.size() != Got.CallSites.size())
275 return PrintAndFail();
276
277 for (size_t I = 0; I < Got.AllocSites.size(); I++) {
278 if (Want.AllocSites[I].Info != Got.AllocSites[I].Info)
279 return PrintAndFail();
280 if (Want.AllocSites[I].CallStack != Got.AllocSites[I].CallStack)
281 return PrintAndFail();
282 }
283
284 for (size_t I = 0; I < Got.CallSites.size(); I++) {
285 if (Want.CallSites[I] != Got.CallSites[I])
286 return PrintAndFail();
287 }
288 return true;
289 }
290
291 TEST_F(InstrProfTest, test_memprof) {
292 ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::MemProf),
293 Succeeded());
294
295 const IndexedMemProfRecord IndexedMR = makeRecord(
296 /*AllocFrames=*/
297 {
298 {0, 1},
299 {2, 3},
300 },
301 /*CallSiteFrames=*/{
302 {4, 5},
303 });
304 const FrameIdMapTy IdToFrameMap = getFrameMapping();
305 for (const auto &I : IdToFrameMap) {
306 Writer.addMemProfFrame(I.first, I.getSecond(), Err);
307 }
308 Writer.addMemProfRecord(/*Id=*/0x9999, IndexedMR);
309
310 auto Profile = Writer.writeBuffer();
311 readProfile(std::move(Profile));
312
313 auto RecordOr = Reader->getMemProfRecord(0x9999);
314 ASSERT_THAT_ERROR(RecordOr.takeError(), Succeeded());
315 const memprof::MemProfRecord &Record = RecordOr.get();
316
317 memprof::FrameId LastUnmappedFrameId = 0;
318 bool HasFrameMappingError = false;
319 auto IdToFrameCallback = [&](const memprof::FrameId Id) {
320 auto Iter = IdToFrameMap.find(Id);
321 if (Iter == IdToFrameMap.end()) {
322 LastUnmappedFrameId = Id;
323 HasFrameMappingError = true;
324 return memprof::Frame(0, 0, 0, false);
325 }
326 return Iter->second;
327 };
328
329 const memprof::MemProfRecord WantRecord(IndexedMR, IdToFrameCallback);
330 ASSERT_FALSE(HasFrameMappingError)
331 << "could not map frame id: " << LastUnmappedFrameId;
332 EXPECT_THAT(WantRecord, EqualsRecord(Record));
333 }
334
335 TEST_F(InstrProfTest, test_memprof_getrecord_error) {
336 ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::MemProf),
337 Succeeded());
338
339 const IndexedMemProfRecord IndexedMR = makeRecord(
340 /*AllocFrames=*/
341 {
342 {0, 1},
343 {2, 3},
344 },
345 /*CallSiteFrames=*/{
346 {4, 5},
347 });
348 // We skip adding the frame mappings here unlike the test_memprof unit test
349 // above to exercise the failure path when getMemProfRecord is invoked.
350 Writer.addMemProfRecord(/*Id=*/0x9999, IndexedMR);
351
352 auto Profile = Writer.writeBuffer();
353 readProfile(std::move(Profile));
354
355 // Missing frames give a hash_mismatch error.
356 auto RecordOr = Reader->getMemProfRecord(0x9999);
357 ASSERT_TRUE(
358 ErrorEquals(instrprof_error::hash_mismatch, RecordOr.takeError()));
359
360 // Missing functions give a unknown_function error.
361 RecordOr = Reader->getMemProfRecord(0x1111);
362 ASSERT_TRUE(
363 ErrorEquals(instrprof_error::unknown_function, RecordOr.takeError()));
364 }
365
366 TEST_F(InstrProfTest, test_memprof_merge) {
367 Writer.addRecord({"func1", 0x1234, {42}}, Err);
368
369 InstrProfWriter Writer2;
370 ASSERT_THAT_ERROR(Writer2.mergeProfileKind(InstrProfKind::MemProf),
371 Succeeded());
372
373 const IndexedMemProfRecord IndexedMR = makeRecord(
374 /*AllocFrames=*/
375 {
376 {0, 1},
377 {2, 3},
378 },
379 /*CallSiteFrames=*/{
380 {4, 5},
381 });
382
383 const FrameIdMapTy IdToFrameMap = getFrameMapping();
384 for (const auto &I : IdToFrameMap) {
385 Writer.addMemProfFrame(I.first, I.getSecond(), Err);
386 }
387 Writer2.addMemProfRecord(/*Id=*/0x9999, IndexedMR);
388
389 ASSERT_THAT_ERROR(Writer.mergeProfileKind(Writer2.getProfileKind()),
390 Succeeded());
391 Writer.mergeRecordsFromWriter(std::move(Writer2), Err);
392
393 auto Profile = Writer.writeBuffer();
394 readProfile(std::move(Profile));
395
396 Expected<InstrProfRecord> R = Reader->getInstrProfRecord("func1", 0x1234);
397 EXPECT_THAT_ERROR(R.takeError(), Succeeded());
398 ASSERT_EQ(1U, R->Counts.size());
399 ASSERT_EQ(42U, R->Counts[0]);
400
401 auto RecordOr = Reader->getMemProfRecord(0x9999);
402 ASSERT_THAT_ERROR(RecordOr.takeError(), Succeeded());
403 const memprof::MemProfRecord &Record = RecordOr.get();
404
405 memprof::FrameId LastUnmappedFrameId = 0;
406 bool HasFrameMappingError = false;
407
408 auto IdToFrameCallback = [&](const memprof::FrameId Id) {
409 auto Iter = IdToFrameMap.find(Id);
410 if (Iter == IdToFrameMap.end()) {
411 LastUnmappedFrameId = Id;
412 HasFrameMappingError = true;
413 return memprof::Frame(0, 0, 0, false);
414 }
415 return Iter->second;
416 };
417
418 const memprof::MemProfRecord WantRecord(IndexedMR, IdToFrameCallback);
419 ASSERT_FALSE(HasFrameMappingError)
420 << "could not map frame id: " << LastUnmappedFrameId;
421 EXPECT_THAT(WantRecord, EqualsRecord(Record));
422 }
423
424 static const char callee1[] = "callee1";
425 static const char callee2[] = "callee2";
426 static const char callee3[] = "callee3";
427 static const char callee4[] = "callee4";
428 static const char callee5[] = "callee5";
429 static const char callee6[] = "callee6";
430
431 TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write) {
432 NamedInstrProfRecord Record1("caller", 0x1234, {1, 2});
433
434 // 4 value sites.
435 Record1.reserveSites(IPVK_IndirectCallTarget, 4);
436 InstrProfValueData VD0[] = {
437 {(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}, {(uint64_t)callee3, 3}};
438 Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr);
439 // No value profile data at the second site.
440 Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
441 InstrProfValueData VD2[] = {{(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}};
442 Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, nullptr);
443 InstrProfValueData VD3[] = {{(uint64_t)callee1, 1}};
444 Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr);
445
446 Writer.addRecord(std::move(Record1), Err);
447 Writer.addRecord({"callee1", 0x1235, {3, 4}}, Err);
448 Writer.addRecord({"callee2", 0x1235, {3, 4}}, Err);
449 Writer.addRecord({"callee3", 0x1235, {3, 4}}, Err);
450 auto Profile = Writer.writeBuffer();
451 readProfile(std::move(Profile));
452
453 Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
454 EXPECT_THAT_ERROR(R.takeError(), Succeeded());
455 ASSERT_EQ(4U, R->getNumValueSites(IPVK_IndirectCallTarget));
456 ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
457 ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
458 ASSERT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
459 ASSERT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
460
461 uint64_t TotalC;
462 std::unique_ptr<InstrProfValueData[]> VD =
463 R->getValueForSite(IPVK_IndirectCallTarget, 0, &TotalC);
464
465 ASSERT_EQ(3U, VD[0].Count);
466 ASSERT_EQ(2U, VD[1].Count);
467 ASSERT_EQ(1U, VD[2].Count);
468 ASSERT_EQ(6U, TotalC);
469
470 ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3"));
471 ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2"));
472 ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1"));
473 }
474
475 TEST_P(MaybeSparseInstrProfTest, annotate_vp_data) {
476 NamedInstrProfRecord Record("caller", 0x1234, {1, 2});
477 Record.reserveSites(IPVK_IndirectCallTarget, 1);
478 InstrProfValueData VD0[] = {{1000, 1}, {2000, 2}, {3000, 3}, {5000, 5},
479 {4000, 4}, {6000, 6}};
480 Record.addValueData(IPVK_IndirectCallTarget, 0, VD0, 6, nullptr);
481 Writer.addRecord(std::move(Record), Err);
482 auto Profile = Writer.writeBuffer();
483 readProfile(std::move(Profile));
484 Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
485 EXPECT_THAT_ERROR(R.takeError(), Succeeded());
486
487 LLVMContext Ctx;
488 std::unique_ptr<Module> M(new Module("MyModule", Ctx));
489 FunctionType *FTy = FunctionType::get(Type::getVoidTy(Ctx),
490 /*isVarArg=*/false);
491 Function *F =
492 Function::Create(FTy, Function::ExternalLinkage, "caller", M.get());
493 BasicBlock *BB = BasicBlock::Create(Ctx, "", F);
494
495 IRBuilder<> Builder(BB);
496 BasicBlock *TBB = BasicBlock::Create(Ctx, "", F);
497 BasicBlock *FBB = BasicBlock::Create(Ctx, "", F);
498
499 // Use branch instruction to annotate with value profile data for simplicity
500 Instruction *Inst = Builder.CreateCondBr(Builder.getTrue(), TBB, FBB);
501 Instruction *Inst2 = Builder.CreateCondBr(Builder.getTrue(), TBB, FBB);
502 annotateValueSite(*M, *Inst, R.get(), IPVK_IndirectCallTarget, 0);
503
504 InstrProfValueData ValueData[5];
505 uint32_t N;
506 uint64_t T;
507 bool Res = getValueProfDataFromInst(*Inst, IPVK_IndirectCallTarget, 5,
508 ValueData, N, T);
509 ASSERT_TRUE(Res);
510 ASSERT_EQ(3U, N);
511 ASSERT_EQ(21U, T);
512 // The result should be sorted already:
513 ASSERT_EQ(6000U, ValueData[0].Value);
514 ASSERT_EQ(6U, ValueData[0].Count);
515 ASSERT_EQ(5000U, ValueData[1].Value);
516 ASSERT_EQ(5U, ValueData[1].Count);
517 ASSERT_EQ(4000U, ValueData[2].Value);
518 ASSERT_EQ(4U, ValueData[2].Count);
519 Res = getValueProfDataFromInst(*Inst, IPVK_IndirectCallTarget, 1, ValueData,
520 N, T);
521 ASSERT_TRUE(Res);
522 ASSERT_EQ(1U, N);
523 ASSERT_EQ(21U, T);
524
525 Res = getValueProfDataFromInst(*Inst2, IPVK_IndirectCallTarget, 5, ValueData,
526 N, T);
527 ASSERT_FALSE(Res);
528
529 // Remove the MD_prof metadata
530 Inst->setMetadata(LLVMContext::MD_prof, 0);
531 // Annotate 5 records this time.
532 annotateValueSite(*M, *Inst, R.get(), IPVK_IndirectCallTarget, 0, 5);
533 Res = getValueProfDataFromInst(*Inst, IPVK_IndirectCallTarget, 5,
534 ValueData, N, T);
535 ASSERT_TRUE(Res);
536 ASSERT_EQ(5U, N);
537 ASSERT_EQ(21U, T);
538 ASSERT_EQ(6000U, ValueData[0].Value);
539 ASSERT_EQ(6U, ValueData[0].Count);
540 ASSERT_EQ(5000U, ValueData[1].Value);
541 ASSERT_EQ(5U, ValueData[1].Count);
542 ASSERT_EQ(4000U, ValueData[2].Value);
543 ASSERT_EQ(4U, ValueData[2].Count);
544 ASSERT_EQ(3000U, ValueData[3].Value);
545 ASSERT_EQ(3U, ValueData[3].Count);
546 ASSERT_EQ(2000U, ValueData[4].Value);
547 ASSERT_EQ(2U, ValueData[4].Count);
548
549 // Remove the MD_prof metadata
550 Inst->setMetadata(LLVMContext::MD_prof, 0);
551 // Annotate with 4 records.
552 InstrProfValueData VD0Sorted[] = {{1000, 6}, {2000, 5}, {3000, 4}, {4000, 3},
553 {5000, 2}, {6000, 1}};
554 annotateValueSite(*M, *Inst, ArrayRef(VD0Sorted).slice(2), 10,
555 IPVK_IndirectCallTarget, 5);
556 Res = getValueProfDataFromInst(*Inst, IPVK_IndirectCallTarget, 5,
557 ValueData, N, T);
558 ASSERT_TRUE(Res);
559 ASSERT_EQ(4U, N);
560 ASSERT_EQ(10U, T);
561 ASSERT_EQ(3000U, ValueData[0].Value);
562 ASSERT_EQ(4U, ValueData[0].Count);
563 ASSERT_EQ(4000U, ValueData[1].Value);
564 ASSERT_EQ(3U, ValueData[1].Count);
565 ASSERT_EQ(5000U, ValueData[2].Value);
566 ASSERT_EQ(2U, ValueData[2].Count);
567 ASSERT_EQ(6000U, ValueData[3].Value);
568 ASSERT_EQ(1U, ValueData[3].Count);
569 }
570
571 TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write_with_weight) {
572 NamedInstrProfRecord Record1("caller", 0x1234, {1, 2});
573
574 // 4 value sites.
575 Record1.reserveSites(IPVK_IndirectCallTarget, 4);
576 InstrProfValueData VD0[] = {
577 {(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}, {(uint64_t)callee3, 3}};
578 Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr);
579 // No value profile data at the second site.
580 Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
581 InstrProfValueData VD2[] = {{(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}};
582 Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, nullptr);
583 InstrProfValueData VD3[] = {{(uint64_t)callee1, 1}};
584 Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr);
585
586 Writer.addRecord(std::move(Record1), 10, Err);
587 Writer.addRecord({"callee1", 0x1235, {3, 4}}, Err);
588 Writer.addRecord({"callee2", 0x1235, {3, 4}}, Err);
589 Writer.addRecord({"callee3", 0x1235, {3, 4}}, Err);
590 auto Profile = Writer.writeBuffer();
591 readProfile(std::move(Profile));
592
593 Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
594 EXPECT_THAT_ERROR(R.takeError(), Succeeded());
595 ASSERT_EQ(4U, R->getNumValueSites(IPVK_IndirectCallTarget));
596 ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
597 ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
598 ASSERT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
599 ASSERT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
600
601 uint64_t TotalC;
602 std::unique_ptr<InstrProfValueData[]> VD =
603 R->getValueForSite(IPVK_IndirectCallTarget, 0, &TotalC);
604 ASSERT_EQ(30U, VD[0].Count);
605 ASSERT_EQ(20U, VD[1].Count);
606 ASSERT_EQ(10U, VD[2].Count);
607 ASSERT_EQ(60U, TotalC);
608
609 ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3"));
610 ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2"));
611 ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1"));
612 }
613
614 TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write_big_endian) {
615 NamedInstrProfRecord Record1("caller", 0x1234, {1, 2});
616
617 // 4 value sites.
618 Record1.reserveSites(IPVK_IndirectCallTarget, 4);
619 InstrProfValueData VD0[] = {
620 {(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}, {(uint64_t)callee3, 3}};
621 Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr);
622 // No value profile data at the second site.
623 Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
624 InstrProfValueData VD2[] = {{(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}};
625 Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, nullptr);
626 InstrProfValueData VD3[] = {{(uint64_t)callee1, 1}};
627 Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr);
628
629 Writer.addRecord(std::move(Record1), Err);
630 Writer.addRecord({"callee1", 0x1235, {3, 4}}, Err);
631 Writer.addRecord({"callee2", 0x1235, {3, 4}}, Err);
632 Writer.addRecord({"callee3", 0x1235, {3, 4}}, Err);
633
634 // Set big endian output.
635 Writer.setValueProfDataEndianness(support::big);
636
637 auto Profile = Writer.writeBuffer();
638 readProfile(std::move(Profile));
639
640 // Set big endian input.
641 Reader->setValueProfDataEndianness(support::big);
642
643 Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
644 EXPECT_THAT_ERROR(R.takeError(), Succeeded());
645 ASSERT_EQ(4U, R->getNumValueSites(IPVK_IndirectCallTarget));
646 ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
647 ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
648 ASSERT_EQ(2U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
649 ASSERT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
650
651 std::unique_ptr<InstrProfValueData[]> VD =
652 R->getValueForSite(IPVK_IndirectCallTarget, 0);
653 ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3"));
654 ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2"));
655 ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1"));
656
657 // Restore little endian default:
658 Writer.setValueProfDataEndianness(support::little);
659 }
660
661 TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1) {
662 static const char caller[] = "caller";
663 NamedInstrProfRecord Record11(caller, 0x1234, {1, 2});
664 NamedInstrProfRecord Record12(caller, 0x1234, {1, 2});
665
666 // 5 value sites.
667 Record11.reserveSites(IPVK_IndirectCallTarget, 5);
668 InstrProfValueData VD0[] = {{uint64_t(callee1), 1},
669 {uint64_t(callee2), 2},
670 {uint64_t(callee3), 3},
671 {uint64_t(callee4), 4}};
672 Record11.addValueData(IPVK_IndirectCallTarget, 0, VD0, 4, nullptr);
673
674 // No value profile data at the second site.
675 Record11.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
676
677 InstrProfValueData VD2[] = {
678 {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}};
679 Record11.addValueData(IPVK_IndirectCallTarget, 2, VD2, 3, nullptr);
680
681 InstrProfValueData VD3[] = {{uint64_t(callee1), 1}};
682 Record11.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr);
683
684 InstrProfValueData VD4[] = {{uint64_t(callee1), 1},
685 {uint64_t(callee2), 2},
686 {uint64_t(callee3), 3}};
687 Record11.addValueData(IPVK_IndirectCallTarget, 4, VD4, 3, nullptr);
688
689 // A different record for the same caller.
690 Record12.reserveSites(IPVK_IndirectCallTarget, 5);
691 InstrProfValueData VD02[] = {{uint64_t(callee2), 5}, {uint64_t(callee3), 3}};
692 Record12.addValueData(IPVK_IndirectCallTarget, 0, VD02, 2, nullptr);
693
694 // No value profile data at the second site.
695 Record12.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
696
697 InstrProfValueData VD22[] = {
698 {uint64_t(callee2), 1}, {uint64_t(callee3), 3}, {uint64_t(callee4), 4}};
699 Record12.addValueData(IPVK_IndirectCallTarget, 2, VD22, 3, nullptr);
700
701 Record12.addValueData(IPVK_IndirectCallTarget, 3, nullptr, 0, nullptr);
702
703 InstrProfValueData VD42[] = {{uint64_t(callee1), 1},
704 {uint64_t(callee2), 2},
705 {uint64_t(callee3), 3}};
706 Record12.addValueData(IPVK_IndirectCallTarget, 4, VD42, 3, nullptr);
707
708 Writer.addRecord(std::move(Record11), Err);
709 // Merge profile data.
710 Writer.addRecord(std::move(Record12), Err);
711
712 Writer.addRecord({callee1, 0x1235, {3, 4}}, Err);
713 Writer.addRecord({callee2, 0x1235, {3, 4}}, Err);
714 Writer.addRecord({callee3, 0x1235, {3, 4}}, Err);
715 Writer.addRecord({callee3, 0x1235, {3, 4}}, Err);
716 Writer.addRecord({callee4, 0x1235, {3, 5}}, Err);
717 auto Profile = Writer.writeBuffer();
718 readProfile(std::move(Profile));
719
720 Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
721 EXPECT_THAT_ERROR(R.takeError(), Succeeded());
722 ASSERT_EQ(5U, R->getNumValueSites(IPVK_IndirectCallTarget));
723 ASSERT_EQ(4U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
724 ASSERT_EQ(0U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
725 ASSERT_EQ(4U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
726 ASSERT_EQ(1U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
727 ASSERT_EQ(3U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 4));
728
729 std::unique_ptr<InstrProfValueData[]> VD =
730 R->getValueForSite(IPVK_IndirectCallTarget, 0);
731 ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee2"));
732 ASSERT_EQ(7U, VD[0].Count);
733 ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee3"));
734 ASSERT_EQ(6U, VD[1].Count);
735 ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee4"));
736 ASSERT_EQ(4U, VD[2].Count);
737 ASSERT_EQ(StringRef((const char *)VD[3].Value, 7), StringRef("callee1"));
738 ASSERT_EQ(1U, VD[3].Count);
739
740 std::unique_ptr<InstrProfValueData[]> VD_2(
741 R->getValueForSite(IPVK_IndirectCallTarget, 2));
742 ASSERT_EQ(StringRef((const char *)VD_2[0].Value, 7), StringRef("callee3"));
743 ASSERT_EQ(6U, VD_2[0].Count);
744 ASSERT_EQ(StringRef((const char *)VD_2[1].Value, 7), StringRef("callee4"));
745 ASSERT_EQ(4U, VD_2[1].Count);
746 ASSERT_EQ(StringRef((const char *)VD_2[2].Value, 7), StringRef("callee2"));
747 ASSERT_EQ(3U, VD_2[2].Count);
748 ASSERT_EQ(StringRef((const char *)VD_2[3].Value, 7), StringRef("callee1"));
749 ASSERT_EQ(1U, VD_2[3].Count);
750
751 std::unique_ptr<InstrProfValueData[]> VD_3(
752 R->getValueForSite(IPVK_IndirectCallTarget, 3));
753 ASSERT_EQ(StringRef((const char *)VD_3[0].Value, 7), StringRef("callee1"));
754 ASSERT_EQ(1U, VD_3[0].Count);
755
756 std::unique_ptr<InstrProfValueData[]> VD_4(
757 R->getValueForSite(IPVK_IndirectCallTarget, 4));
758 ASSERT_EQ(StringRef((const char *)VD_4[0].Value, 7), StringRef("callee3"));
759 ASSERT_EQ(6U, VD_4[0].Count);
760 ASSERT_EQ(StringRef((const char *)VD_4[1].Value, 7), StringRef("callee2"));
761 ASSERT_EQ(4U, VD_4[1].Count);
762 ASSERT_EQ(StringRef((const char *)VD_4[2].Value, 7), StringRef("callee1"));
763 ASSERT_EQ(2U, VD_4[2].Count);
764 }
765
766 TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1_saturation) {
767 static const char bar[] = "bar";
768
769 const uint64_t MaxValCount = std::numeric_limits<uint64_t>::max();
770 const uint64_t MaxEdgeCount = getInstrMaxCountValue();
771
772 instrprof_error Result;
773 auto Err = [&](Error E) { Result = InstrProfError::take(std::move(E)); };
774 Result = instrprof_error::success;
775 Writer.addRecord({"foo", 0x1234, {1}}, Err);
776 ASSERT_EQ(Result, instrprof_error::success);
777
778 // Verify counter overflow.
779 Result = instrprof_error::success;
780 Writer.addRecord({"foo", 0x1234, {MaxEdgeCount}}, Err);
781 ASSERT_EQ(Result, instrprof_error::counter_overflow);
782
783 Result = instrprof_error::success;
784 Writer.addRecord({bar, 0x9012, {8}}, Err);
785 ASSERT_EQ(Result, instrprof_error::success);
786
787 NamedInstrProfRecord Record4("baz", 0x5678, {3, 4});
788 Record4.reserveSites(IPVK_IndirectCallTarget, 1);
789 InstrProfValueData VD4[] = {{uint64_t(bar), 1}};
790 Record4.addValueData(IPVK_IndirectCallTarget, 0, VD4, 1, nullptr);
791 Result = instrprof_error::success;
792 Writer.addRecord(std::move(Record4), Err);
793 ASSERT_EQ(Result, instrprof_error::success);
794
795 // Verify value data counter overflow.
796 NamedInstrProfRecord Record5("baz", 0x5678, {5, 6});
797 Record5.reserveSites(IPVK_IndirectCallTarget, 1);
798 InstrProfValueData VD5[] = {{uint64_t(bar), MaxValCount}};
799 Record5.addValueData(IPVK_IndirectCallTarget, 0, VD5, 1, nullptr);
800 Result = instrprof_error::success;
801 Writer.addRecord(std::move(Record5), Err);
802 ASSERT_EQ(Result, instrprof_error::counter_overflow);
803
804 auto Profile = Writer.writeBuffer();
805 readProfile(std::move(Profile));
806
807 // Verify saturation of counts.
808 Expected<InstrProfRecord> ReadRecord1 =
809 Reader->getInstrProfRecord("foo", 0x1234);
810 EXPECT_THAT_ERROR(ReadRecord1.takeError(), Succeeded());
811 ASSERT_EQ(MaxEdgeCount, ReadRecord1->Counts[0]);
812
813 Expected<InstrProfRecord> ReadRecord2 =
814 Reader->getInstrProfRecord("baz", 0x5678);
815 ASSERT_TRUE(bool(ReadRecord2));
816 ASSERT_EQ(1U, ReadRecord2->getNumValueSites(IPVK_IndirectCallTarget));
817 std::unique_ptr<InstrProfValueData[]> VD =
818 ReadRecord2->getValueForSite(IPVK_IndirectCallTarget, 0);
819 ASSERT_EQ(StringRef("bar"), StringRef((const char *)VD[0].Value, 3));
820 ASSERT_EQ(MaxValCount, VD[0].Count);
821 }
822
823 // This test tests that when there are too many values
824 // for a given site, the merged results are properly
825 // truncated.
826 TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge_site_trunc) {
827 static const char caller[] = "caller";
828
829 NamedInstrProfRecord Record11(caller, 0x1234, {1, 2});
830 NamedInstrProfRecord Record12(caller, 0x1234, {1, 2});
831
832 // 2 value sites.
833 Record11.reserveSites(IPVK_IndirectCallTarget, 2);
834 InstrProfValueData VD0[255];
835 for (int I = 0; I < 255; I++) {
836 VD0[I].Value = 2 * I;
837 VD0[I].Count = 2 * I + 1000;
838 }
839
840 Record11.addValueData(IPVK_IndirectCallTarget, 0, VD0, 255, nullptr);
841 Record11.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
842
843 Record12.reserveSites(IPVK_IndirectCallTarget, 2);
844 InstrProfValueData VD1[255];
845 for (int I = 0; I < 255; I++) {
846 VD1[I].Value = 2 * I + 1;
847 VD1[I].Count = 2 * I + 1001;
848 }
849
850 Record12.addValueData(IPVK_IndirectCallTarget, 0, VD1, 255, nullptr);
851 Record12.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
852
853 Writer.addRecord(std::move(Record11), Err);
854 // Merge profile data.
855 Writer.addRecord(std::move(Record12), Err);
856
857 auto Profile = Writer.writeBuffer();
858 readProfile(std::move(Profile));
859
860 Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
861 EXPECT_THAT_ERROR(R.takeError(), Succeeded());
862 std::unique_ptr<InstrProfValueData[]> VD(
863 R->getValueForSite(IPVK_IndirectCallTarget, 0));
864 ASSERT_EQ(2U, R->getNumValueSites(IPVK_IndirectCallTarget));
865 ASSERT_EQ(255U, R->getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
866 for (unsigned I = 0; I < 255; I++) {
867 ASSERT_EQ(VD[I].Value, 509 - I);
868 ASSERT_EQ(VD[I].Count, 1509 - I);
869 }
870 }
871
872 static void addValueProfData(InstrProfRecord &Record) {
873 Record.reserveSites(IPVK_IndirectCallTarget, 5);
874 InstrProfValueData VD0[] = {{uint64_t(callee1), 400},
875 {uint64_t(callee2), 1000},
876 {uint64_t(callee3), 500},
877 {uint64_t(callee4), 300},
878 {uint64_t(callee5), 100}};
879 Record.addValueData(IPVK_IndirectCallTarget, 0, VD0, 5, nullptr);
880 InstrProfValueData VD1[] = {{uint64_t(callee5), 800},
881 {uint64_t(callee3), 1000},
882 {uint64_t(callee2), 2500},
883 {uint64_t(callee1), 1300}};
884 Record.addValueData(IPVK_IndirectCallTarget, 1, VD1, 4, nullptr);
885 InstrProfValueData VD2[] = {{uint64_t(callee6), 800},
886 {uint64_t(callee3), 1000},
887 {uint64_t(callee4), 5500}};
888 Record.addValueData(IPVK_IndirectCallTarget, 2, VD2, 3, nullptr);
889 InstrProfValueData VD3[] = {{uint64_t(callee2), 1800},
890 {uint64_t(callee3), 2000}};
891 Record.addValueData(IPVK_IndirectCallTarget, 3, VD3, 2, nullptr);
892 Record.addValueData(IPVK_IndirectCallTarget, 4, nullptr, 0, nullptr);
893 }
894
895 TEST_P(MaybeSparseInstrProfTest, value_prof_data_read_write) {
896 InstrProfRecord SrcRecord({1ULL << 31, 2});
897 addValueProfData(SrcRecord);
898 std::unique_ptr<ValueProfData> VPData =
899 ValueProfData::serializeFrom(SrcRecord);
900
901 InstrProfRecord Record({1ULL << 31, 2});
902 VPData->deserializeTo(Record, nullptr);
903
904 // Now read data from Record and sanity check the data
905 ASSERT_EQ(5U, Record.getNumValueSites(IPVK_IndirectCallTarget));
906 ASSERT_EQ(5U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
907 ASSERT_EQ(4U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
908 ASSERT_EQ(3U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
909 ASSERT_EQ(2U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
910 ASSERT_EQ(0U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 4));
911
912 auto Cmp = [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) {
913 return VD1.Count > VD2.Count;
914 };
915 std::unique_ptr<InstrProfValueData[]> VD_0(
916 Record.getValueForSite(IPVK_IndirectCallTarget, 0));
917 llvm::sort(&VD_0[0], &VD_0[5], Cmp);
918 ASSERT_EQ(StringRef((const char *)VD_0[0].Value, 7), StringRef("callee2"));
919 ASSERT_EQ(1000U, VD_0[0].Count);
920 ASSERT_EQ(StringRef((const char *)VD_0[1].Value, 7), StringRef("callee3"));
921 ASSERT_EQ(500U, VD_0[1].Count);
922 ASSERT_EQ(StringRef((const char *)VD_0[2].Value, 7), StringRef("callee1"));
923 ASSERT_EQ(400U, VD_0[2].Count);
924 ASSERT_EQ(StringRef((const char *)VD_0[3].Value, 7), StringRef("callee4"));
925 ASSERT_EQ(300U, VD_0[3].Count);
926 ASSERT_EQ(StringRef((const char *)VD_0[4].Value, 7), StringRef("callee5"));
927 ASSERT_EQ(100U, VD_0[4].Count);
928
929 std::unique_ptr<InstrProfValueData[]> VD_1(
930 Record.getValueForSite(IPVK_IndirectCallTarget, 1));
931 llvm::sort(&VD_1[0], &VD_1[4], Cmp);
932 ASSERT_EQ(StringRef((const char *)VD_1[0].Value, 7), StringRef("callee2"));
933 ASSERT_EQ(2500U, VD_1[0].Count);
934 ASSERT_EQ(StringRef((const char *)VD_1[1].Value, 7), StringRef("callee1"));
935 ASSERT_EQ(1300U, VD_1[1].Count);
936 ASSERT_EQ(StringRef((const char *)VD_1[2].Value, 7), StringRef("callee3"));
937 ASSERT_EQ(1000U, VD_1[2].Count);
938 ASSERT_EQ(StringRef((const char *)VD_1[3].Value, 7), StringRef("callee5"));
939 ASSERT_EQ(800U, VD_1[3].Count);
940
941 std::unique_ptr<InstrProfValueData[]> VD_2(
942 Record.getValueForSite(IPVK_IndirectCallTarget, 2));
943 llvm::sort(&VD_2[0], &VD_2[3], Cmp);
944 ASSERT_EQ(StringRef((const char *)VD_2[0].Value, 7), StringRef("callee4"));
945 ASSERT_EQ(5500U, VD_2[0].Count);
946 ASSERT_EQ(StringRef((const char *)VD_2[1].Value, 7), StringRef("callee3"));
947 ASSERT_EQ(1000U, VD_2[1].Count);
948 ASSERT_EQ(StringRef((const char *)VD_2[2].Value, 7), StringRef("callee6"));
949 ASSERT_EQ(800U, VD_2[2].Count);
950
951 std::unique_ptr<InstrProfValueData[]> VD_3(
952 Record.getValueForSite(IPVK_IndirectCallTarget, 3));
953 llvm::sort(&VD_3[0], &VD_3[2], Cmp);
954 ASSERT_EQ(StringRef((const char *)VD_3[0].Value, 7), StringRef("callee3"));
955 ASSERT_EQ(2000U, VD_3[0].Count);
956 ASSERT_EQ(StringRef((const char *)VD_3[1].Value, 7), StringRef("callee2"));
957 ASSERT_EQ(1800U, VD_3[1].Count);
958 }
959
960 TEST_P(MaybeSparseInstrProfTest, value_prof_data_read_write_mapping) {
961
962 NamedInstrProfRecord SrcRecord("caller", 0x1234, {1ULL << 31, 2});
963 addValueProfData(SrcRecord);
964 std::unique_ptr<ValueProfData> VPData =
965 ValueProfData::serializeFrom(SrcRecord);
966
967 NamedInstrProfRecord Record("caller", 0x1234, {1ULL << 31, 2});
968 InstrProfSymtab Symtab;
969 Symtab.mapAddress(uint64_t(callee1), 0x1000ULL);
970 Symtab.mapAddress(uint64_t(callee2), 0x2000ULL);
971 Symtab.mapAddress(uint64_t(callee3), 0x3000ULL);
972 Symtab.mapAddress(uint64_t(callee4), 0x4000ULL);
973 // Missing mapping for callee5
974
975 VPData->deserializeTo(Record, &Symtab);
976
977 // Now read data from Record and sanity check the data
978 ASSERT_EQ(5U, Record.getNumValueSites(IPVK_IndirectCallTarget));
979 ASSERT_EQ(5U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
980
981 auto Cmp = [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) {
982 return VD1.Count > VD2.Count;
983 };
984 std::unique_ptr<InstrProfValueData[]> VD_0(
985 Record.getValueForSite(IPVK_IndirectCallTarget, 0));
986 llvm::sort(&VD_0[0], &VD_0[5], Cmp);
987 ASSERT_EQ(VD_0[0].Value, 0x2000ULL);
988 ASSERT_EQ(1000U, VD_0[0].Count);
989 ASSERT_EQ(VD_0[1].Value, 0x3000ULL);
990 ASSERT_EQ(500U, VD_0[1].Count);
991 ASSERT_EQ(VD_0[2].Value, 0x1000ULL);
992 ASSERT_EQ(400U, VD_0[2].Count);
993
994 // callee5 does not have a mapped value -- default to 0.
995 ASSERT_EQ(VD_0[4].Value, 0ULL);
996 }
997
998 TEST_P(MaybeSparseInstrProfTest, get_max_function_count) {
999 Writer.addRecord({"foo", 0x1234, {1ULL << 31, 2}}, Err);
1000 Writer.addRecord({"bar", 0, {1ULL << 63}}, Err);
1001 Writer.addRecord({"baz", 0x5678, {0, 0, 0, 0}}, Err);
1002 auto Profile = Writer.writeBuffer();
1003 readProfile(std::move(Profile));
1004
1005 ASSERT_EQ(1ULL << 63, Reader->getMaximumFunctionCount(/* IsCS */ false));
1006 }
1007
1008 TEST_P(MaybeSparseInstrProfTest, get_weighted_function_counts) {
1009 Writer.addRecord({"foo", 0x1234, {1, 2}}, 3, Err);
1010 Writer.addRecord({"foo", 0x1235, {3, 4}}, 5, Err);
1011 auto Profile = Writer.writeBuffer();
1012 readProfile(std::move(Profile));
1013
1014 std::vector<uint64_t> Counts;
1015 EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo", 0x1234, Counts),
1016 Succeeded());
1017 ASSERT_EQ(2U, Counts.size());
1018 ASSERT_EQ(3U, Counts[0]);
1019 ASSERT_EQ(6U, Counts[1]);
1020
1021 EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo", 0x1235, Counts),
1022 Succeeded());
1023 ASSERT_EQ(2U, Counts.size());
1024 ASSERT_EQ(15U, Counts[0]);
1025 ASSERT_EQ(20U, Counts[1]);
1026 }
1027
1028 // Testing symtab creator interface used by indexed profile reader.
1029 TEST_P(MaybeSparseInstrProfTest, instr_prof_symtab_test) {
1030 std::vector<StringRef> FuncNames;
1031 FuncNames.push_back("func1");
1032 FuncNames.push_back("func2");
1033 FuncNames.push_back("func3");
1034 FuncNames.push_back("bar1");
1035 FuncNames.push_back("bar2");
1036 FuncNames.push_back("bar3");
1037 InstrProfSymtab Symtab;
1038 EXPECT_THAT_ERROR(Symtab.create(FuncNames), Succeeded());
1039 StringRef R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("func1"));
1040 ASSERT_EQ(StringRef("func1"), R);
1041 R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("func2"));
1042 ASSERT_EQ(StringRef("func2"), R);
1043 R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("func3"));
1044 ASSERT_EQ(StringRef("func3"), R);
1045 R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("bar1"));
1046 ASSERT_EQ(StringRef("bar1"), R);
1047 R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("bar2"));
1048 ASSERT_EQ(StringRef("bar2"), R);
1049 R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("bar3"));
1050 ASSERT_EQ(StringRef("bar3"), R);
1051
1052 // negative tests
1053 R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("bar4"));
1054 ASSERT_EQ(StringRef(), R);
1055 R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("foo4"));
1056 ASSERT_EQ(StringRef(), R);
1057
1058 // Now incrementally update the symtab
1059 EXPECT_THAT_ERROR(Symtab.addFuncName("blah_1"), Succeeded());
1060 EXPECT_THAT_ERROR(Symtab.addFuncName("blah_2"), Succeeded());
1061 EXPECT_THAT_ERROR(Symtab.addFuncName("blah_3"), Succeeded());
1062
1063 // Check again
1064 R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("blah_1"));
1065 ASSERT_EQ(StringRef("blah_1"), R);
1066 R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("blah_2"));
1067 ASSERT_EQ(StringRef("blah_2"), R);
1068 R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("blah_3"));
1069 ASSERT_EQ(StringRef("blah_3"), R);
1070 R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("func1"));
1071 ASSERT_EQ(StringRef("func1"), R);
1072 R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("func2"));
1073 ASSERT_EQ(StringRef("func2"), R);
1074 R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("func3"));
1075 ASSERT_EQ(StringRef("func3"), R);
1076 R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("bar1"));
1077 ASSERT_EQ(StringRef("bar1"), R);
1078 R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("bar2"));
1079 ASSERT_EQ(StringRef("bar2"), R);
1080 R = Symtab.getFuncName(IndexedInstrProf::ComputeHash("bar3"));
1081 ASSERT_EQ(StringRef("bar3"), R);
1082 }
1083
1084 // Test that we get an error when creating a bogus symtab.
1085 TEST_P(MaybeSparseInstrProfTest, instr_prof_bogus_symtab_empty_func_name) {
1086 InstrProfSymtab Symtab;
1087 EXPECT_TRUE(ErrorEquals(instrprof_error::malformed, Symtab.addFuncName("")));
1088 }
1089
1090 // Testing symtab creator interface used by value profile transformer.
1091 TEST_P(MaybeSparseInstrProfTest, instr_prof_symtab_module_test) {
1092 LLVMContext Ctx;
1093 std::unique_ptr<Module> M = std::make_unique<Module>("MyModule.cpp", Ctx);
1094 FunctionType *FTy = FunctionType::get(Type::getVoidTy(Ctx),
1095 /*isVarArg=*/false);
1096 Function::Create(FTy, Function::ExternalLinkage, "Gfoo", M.get());
1097 Function::Create(FTy, Function::ExternalLinkage, "Gblah", M.get());
1098 Function::Create(FTy, Function::ExternalLinkage, "Gbar", M.get());
1099 Function::Create(FTy, Function::InternalLinkage, "Ifoo", M.get());
1100 Function::Create(FTy, Function::InternalLinkage, "Iblah", M.get());
1101 Function::Create(FTy, Function::InternalLinkage, "Ibar", M.get());
1102 Function::Create(FTy, Function::PrivateLinkage, "Pfoo", M.get());
1103 Function::Create(FTy, Function::PrivateLinkage, "Pblah", M.get());
1104 Function::Create(FTy, Function::PrivateLinkage, "Pbar", M.get());
1105 Function::Create(FTy, Function::WeakODRLinkage, "Wfoo", M.get());
1106 Function::Create(FTy, Function::WeakODRLinkage, "Wblah", M.get());
1107 Function::Create(FTy, Function::WeakODRLinkage, "Wbar", M.get());
1108
1109 InstrProfSymtab ProfSymtab;
1110 EXPECT_THAT_ERROR(ProfSymtab.create(*M), Succeeded());
1111
1112 StringRef Funcs[] = {"Gfoo", "Gblah", "Gbar", "Ifoo", "Iblah", "Ibar",
1113 "Pfoo", "Pblah", "Pbar", "Wfoo", "Wblah", "Wbar"};
1114
1115 for (unsigned I = 0; I < std::size(Funcs); I++) {
1116 Function *F = M->getFunction(Funcs[I]);
1117 ASSERT_TRUE(F != nullptr);
1118 std::string PGOName = getPGOFuncName(*F);
1119 uint64_t Key = IndexedInstrProf::ComputeHash(PGOName);
1120 ASSERT_EQ(StringRef(PGOName),
1121 ProfSymtab.getFuncName(Key));
1122 ASSERT_EQ(StringRef(Funcs[I]), ProfSymtab.getOrigFuncName(Key));
1123 }
1124 }
1125
1126 // Testing symtab serialization and creator/deserialization interface
1127 // used by coverage map reader, and raw profile reader.
1128 TEST_P(MaybeSparseInstrProfTest, instr_prof_symtab_compression_test) {
1129 std::vector<std::string> FuncNames1;
1130 std::vector<std::string> FuncNames2;
1131 for (int I = 0; I < 3; I++) {
1132 std::string str;
1133 raw_string_ostream OS(str);
1134 OS << "func_" << I;
1135 FuncNames1.push_back(OS.str());
1136 str.clear();
1137 OS << "f oooooooooooooo_" << I;
1138 FuncNames1.push_back(OS.str());
1139 str.clear();
1140 OS << "BAR_" << I;
1141 FuncNames2.push_back(OS.str());
1142 str.clear();
1143 OS << "BlahblahBlahblahBar_" << I;
1144 FuncNames2.push_back(OS.str());
1145 }
1146
1147 for (bool DoCompression : {false, true}) {
1148 // Compressing:
1149 std::string FuncNameStrings1;
1150 EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
1151 FuncNames1,
1152 (DoCompression && compression::zlib::isAvailable()),
1153 FuncNameStrings1),
1154 Succeeded());
1155
1156 // Compressing:
1157 std::string FuncNameStrings2;
1158 EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
1159 FuncNames2,
1160 (DoCompression && compression::zlib::isAvailable()),
1161 FuncNameStrings2),
1162 Succeeded());
1163
1164 for (int Padding = 0; Padding < 2; Padding++) {
1165 // Join with paddings :
1166 std::string FuncNameStrings = FuncNameStrings1;
1167 for (int P = 0; P < Padding; P++) {
1168 FuncNameStrings.push_back('\0');
1169 }
1170 FuncNameStrings += FuncNameStrings2;
1171
1172 // Now decompress:
1173 InstrProfSymtab Symtab;
1174 EXPECT_THAT_ERROR(Symtab.create(StringRef(FuncNameStrings)), Succeeded());
1175
1176 // Now do the checks:
1177 // First sampling some data points:
1178 StringRef R = Symtab.getFuncName(IndexedInstrProf::ComputeHash(FuncNames1[0]));
1179 ASSERT_EQ(StringRef("func_0"), R);
1180 R = Symtab.getFuncName(IndexedInstrProf::ComputeHash(FuncNames1[1]));
1181 ASSERT_EQ(StringRef("f oooooooooooooo_0"), R);
1182 for (int I = 0; I < 3; I++) {
1183 std::string N[4];
1184 N[0] = FuncNames1[2 * I];
1185 N[1] = FuncNames1[2 * I + 1];
1186 N[2] = FuncNames2[2 * I];
1187 N[3] = FuncNames2[2 * I + 1];
1188 for (int J = 0; J < 4; J++) {
1189 StringRef R = Symtab.getFuncName(IndexedInstrProf::ComputeHash(N[J]));
1190 ASSERT_EQ(StringRef(N[J]), R);
1191 }
1192 }
1193 }
1194 }
1195 }
1196
1197 TEST_P(MaybeSparseInstrProfTest, remapping_test) {
1198 Writer.addRecord({"_Z3fooi", 0x1234, {1, 2, 3, 4}}, Err);
1199 Writer.addRecord({"file:_Z3barf", 0x567, {5, 6, 7}}, Err);
1200 auto Profile = Writer.writeBuffer();
1201 readProfile(std::move(Profile), llvm::MemoryBuffer::getMemBuffer(R"(
1202 type i l
1203 name 3bar 4quux
1204 )"));
1205
1206 std::vector<uint64_t> Counts;
1207 for (StringRef FooName : {"_Z3fooi", "_Z3fool"}) {
1208 EXPECT_THAT_ERROR(Reader->getFunctionCounts(FooName, 0x1234, Counts),
1209 Succeeded());
1210 ASSERT_EQ(4u, Counts.size());
1211 EXPECT_EQ(1u, Counts[0]);
1212 EXPECT_EQ(2u, Counts[1]);
1213 EXPECT_EQ(3u, Counts[2]);
1214 EXPECT_EQ(4u, Counts[3]);
1215 }
1216
1217 for (StringRef BarName : {"file:_Z3barf", "file:_Z4quuxf"}) {
1218 EXPECT_THAT_ERROR(Reader->getFunctionCounts(BarName, 0x567, Counts),
1219 Succeeded());
1220 ASSERT_EQ(3u, Counts.size());
1221 EXPECT_EQ(5u, Counts[0]);
1222 EXPECT_EQ(6u, Counts[1]);
1223 EXPECT_EQ(7u, Counts[2]);
1224 }
1225
1226 for (StringRef BadName : {"_Z3foof", "_Z4quuxi", "_Z3barl", "", "_ZZZ",
1227 "_Z3barf", "otherfile:_Z4quuxf"}) {
1228 EXPECT_THAT_ERROR(Reader->getFunctionCounts(BadName, 0x1234, Counts),
1229 Failed());
1230 EXPECT_THAT_ERROR(Reader->getFunctionCounts(BadName, 0x567, Counts),
1231 Failed());
1232 }
1233 }
1234
1235 TEST_F(SparseInstrProfTest, preserve_no_records) {
1236 Writer.addRecord({"foo", 0x1234, {0}}, Err);
1237 Writer.addRecord({"bar", 0x4321, {0, 0}}, Err);
1238 Writer.addRecord({"baz", 0x4321, {0, 0, 0}}, Err);
1239
1240 auto Profile = Writer.writeBuffer();
1241 readProfile(std::move(Profile));
1242
1243 auto I = Reader->begin(), E = Reader->end();
1244 ASSERT_TRUE(I == E);
1245 }
1246
1247 INSTANTIATE_TEST_SUITE_P(MaybeSparse, MaybeSparseInstrProfTest,
1248 ::testing::Bool());
1249
1250 #if defined(_LP64) && defined(EXPENSIVE_CHECKS)
1251 TEST(ProfileReaderTest, ReadsLargeFiles) {
1252 const size_t LargeSize = 1ULL << 32; // 4GB
1253
1254 auto RawProfile = WritableMemoryBuffer::getNewUninitMemBuffer(LargeSize);
1255 if (!RawProfile)
1256 return;
1257 auto RawProfileReaderOrErr = InstrProfReader::create(std::move(RawProfile));
1258 ASSERT_TRUE(InstrProfError::take(RawProfileReaderOrErr.takeError()) ==
1259 instrprof_error::unrecognized_format);
1260
1261 auto IndexedProfile = WritableMemoryBuffer::getNewUninitMemBuffer(LargeSize);
1262 if (!IndexedProfile)
1263 return;
1264 auto IndexedReaderOrErr =
1265 IndexedInstrProfReader::create(std::move(IndexedProfile), nullptr);
1266 ASSERT_TRUE(InstrProfError::take(IndexedReaderOrErr.takeError()) ==
1267 instrprof_error::bad_magic);
1268 }
1269 #endif
1270
1271 } // end anonymous namespace