Debian Codegraph @

Source/Reasoner/Answerer/Composition/CQtConcurrentVariableMappingsCompositionJoinComputator.cpp

raw
1 /*
2 * Copyright (C) 2013-2015, 2019 by the Konclude Developer Team.
3 *
4 * This file is part of the reasoning system Konclude.
5 * For details and support, see <http://konclude.com/>.
6 *
7 * Konclude is free software: you can redistribute it and/or modify
8 * it under the terms of version 3 of the GNU Lesser General Public
9 * License (LGPLv3) as published by the Free Software Foundation.
10 *
11 * Konclude is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU (Lesser) General Public License for more details.
15 *
16 * You should have received a copy of the GNU (Lesser) General Public
17 * License along with Konclude. If not, see <http://www.gnu.org/licenses/>.
18 *
19 */
20
21 #include "CQtConcurrentVariableMappingsCompositionJoinComputator.h"
22
23 namespace Konclude {
24
25 namespace Reasoner {
26
27 namespace Answerer {
28
29 namespace Composition {
30
31 CQtConcurrentVariableMappingsCompositionJoinComputator::CQtConcurrentVariableMappingsCompositionJoinComputator() {
32 mComputerName = "Concurrent Qt join";
33 }
34
35
36 CAbstractVariableMappingsCompositionComputator* CQtConcurrentVariableMappingsCompositionJoinComputator::configureComputator(COptimizedComplexExpressionOntologyAnsweringItem* ontoAnsweringItem, CAbstractVariableMappingsCompositionItemRequirementProcessor* reqProcessor, CAnswererContext* answererContext) {
37 CAbstractVariableMappingsCompositionJoinComputator::configureComputator(ontoAnsweringItem, reqProcessor, answererContext);
38 mConcurrentJoinComputationTaskCount = CConfigDataReader::readConfigInteger(ontoAnsweringItem->getCalculationConfiguration(), "Konclude.Answering.ConcurrentJoinComputationTaskCount", 211);
39 mConfBatchMaxHandlingCount = CConfigDataReader::readConfigInteger(ontoAnsweringItem->getCalculationConfiguration(), "Konclude.Answering.ConcurrentJoinComputationMaxBatchHandlingCount", 10000);
40 mConfSplitTestingBatchMaxHandlingCount = CConfigDataReader::readConfigInteger(ontoAnsweringItem->getCalculationConfiguration(), "Konclude.Answering.ConcurrentJoinLastStepSplitTestingComputationMaxBatchHandlingCount", 1000);
41 mConfPartitionizedMemoryManagement = CConfigDataReader::readConfigBoolean(ontoAnsweringItem->getCalculationConfiguration(), "Konclude.Answering.ConcurrentJoinPartitionizedMemoryManagement", false);
42 mConfPerformanceLogging = CConfigDataReader::readConfigBoolean(ontoAnsweringItem->getCalculationConfiguration(), "Konclude.Answering.ConcurrentJoinPerformanceLogging", false);
43 mConfCheckingSideDirectJoining = CConfigDataReader::readConfigBoolean(ontoAnsweringItem->getCalculationConfiguration(), "Konclude.Answering.ConcurrentJoinDirectJoiningForCheckingSide", false);
44 mConfConcurrentJoiningHashCount = mConcurrentJoinComputationTaskCount;
45 mConfConcurrentVariableMappingPartCount = mConcurrentJoinComputationTaskCount;
46 mConfBatchSizeLimit = 200;
47 mConfBatchSizeCount = mConcurrentJoinComputationTaskCount;
48 return this;
49 }
50
51
52 bool CQtConcurrentVariableMappingsCompositionJoinComputator::computeVariableMappingsJoinComposition(COptimizedComplexVariableJoiningItem* joiningItem, COptimizedComplexBuildingVariableCompositionsItem* buildingVarItem, CAnswererContext* answererContext, bool& processing) {
53
54 COptimizedComplexVariableCompositionItem* leftItem = joiningItem->getLeftItem();
55 COptimizedComplexVariableCompositionItem* rightItem = joiningItem->getRightItem();
56
57 COptimizedComplexVariableIndividualMappings* leftVarMapping = leftItem->getVariableMapping();
58 COptimizedComplexVariableIndividualMappings* rightVarMapping = rightItem->getVariableMapping();
59 COptimizedComplexVariableIndividualMappingsMultiHash* joinedVarMapping = (COptimizedComplexVariableIndividualMappingsMultiHash*)joiningItem->getVariableMapping();
60
61 cint64 leftSize = leftVarMapping->getBindingSize();
62 cint64 rightSize = rightVarMapping->getBindingSize();
63 cint64 joinSize = joinedVarMapping->getBindingSize();
64
65
66 COptimizedComplexVariableJoiningBindingPositionMapping* variablePositionMapping = joiningItem->getPositionMapping();
67
68 bool itemSameVarReductionJoining = variablePositionMapping->isSelfSameVariableReduction();
69 cint64* rightPosMapArray = variablePositionMapping->getRightPosMapArray();
70 cint64* leftPosMapArray = variablePositionMapping->getLeftPosMapArray();
71
72 CXLinker<cint64>* leftKeyBindingLinker = variablePositionMapping->getLeftKeyBindingLinker();
73 CXLinker<cint64>* rightKeyBindingLinker = variablePositionMapping->getRightKeyBindingLinker();
74
75
76
77 QVector<COptimizedComplexVariableJoiningHashMemoryManaged*>& joiningHashVector = joiningItem->getJoiningHashVector();
78 if (joiningHashVector.size() < mConfConcurrentJoiningHashCount) {
79 joiningHashVector.resize(mConfConcurrentJoiningHashCount);
80 }
81
82
83
84 COptimizedComplexVariableCompositionItemDependence* leftItemDep = joiningItem->getLeftItemDependence();
85 COptimizedComplexVariableCompositionItemDependence* rightItemDep = joiningItem->getRightItemDependence();
86
87
88 cint64& leftSampleKeyCount = joiningItem->getLeftSampleKeyCount();
89 cint64& rightSampleKeyCount = joiningItem->getRightSampleKeyCount();
90
91 cint64& leftSampleInsertionCount = joiningItem->getLeftSampleInsertionCount();
92 cint64& rightSampleInsertionCount = joiningItem->getRightSampleInsertionCount();
93
94 QAtomicInt atomicLeftSampleKeyCount(leftSampleKeyCount);
95 QAtomicInt atomicRightSampleKeyCount(rightSampleKeyCount);
96
97
98
99 bool reduction = variablePositionMapping->isVariableReduction();
100 CSameRealization* sameRealization = mOntoAnsweringItem->getOntology()->getRealization()->getSameRealization();
101
102
103
104 auto joiningVariableMappingCreationFunc = [&]( COptimizedComplexVariableIndividualBindings* leftvarMapping, COptimizedComplexVariableIndividualBindingsCardinality* leftNewCardinalites, COptimizedComplexVariableIndividualBindingsCardinality* leftPrevCardinalites,
105 COptimizedComplexVariableIndividualBindings* rightvarMapping, COptimizedComplexVariableIndividualBindingsCardinality* rightNewCardinalites, COptimizedComplexVariableIndividualBindingsCardinality* righPrevCardinalites,
106 COptimizedComplexVariableIndividualBindings* joinedBindings, COptimizedComplexVariableIndividualBindingsCardinalityLinker* joinedLinker, COptimizedComplexVariableIndividualBindingsCardinality* providedJoinedCardinalites, CMemoryAllocationManager* memMan) -> COptimizedComplexVariableIndividualBindingsCardinalityLinker* {
107
108
109 COptimizedComplexVariableIndividualBindingsCardinality* joinedCardinalites = nullptr;
110
111 COptimizedComplexVariableIndividualBindingsCardinality tmpLeftCardinalites;
112 COptimizedComplexVariableIndividualBindingsCardinality tmpRightCardinalites;
113
114 COptimizedComplexVariableIndividualBindingsCardinality* leftCardinalites = nullptr;
115 COptimizedComplexVariableIndividualBindingsCardinality* rightCardinalites = nullptr;
116
117 bool cardCreated = false;
118
119 if (leftNewCardinalites && leftPrevCardinalites) {
120 tmpLeftCardinalites = *leftNewCardinalites;
121 tmpLeftCardinalites.substractCardinalities(leftPrevCardinalites);
122 leftCardinalites = &tmpLeftCardinalites;
123 } else if (leftNewCardinalites && !leftPrevCardinalites) {
124 tmpLeftCardinalites = *leftNewCardinalites;
125 leftCardinalites = &tmpLeftCardinalites;
126 }
127
128 if (rightNewCardinalites && righPrevCardinalites) {
129 tmpRightCardinalites = *rightNewCardinalites;
130 tmpRightCardinalites.substractCardinalities(righPrevCardinalites);
131 rightCardinalites = &tmpRightCardinalites;
132 } else if (rightNewCardinalites && !righPrevCardinalites) {
133 tmpRightCardinalites = *rightNewCardinalites;
134 rightCardinalites = &tmpRightCardinalites;
135 }
136
137
138 TIndividualInstanceItemDataBinding* leftBinding = leftvarMapping->getBindingArray();
139 TIndividualInstanceItemDataBinding* rightBinding = rightvarMapping->getBindingArray();
140
141 if (!joinedCardinalites && (leftCardinalites || rightCardinalites || reduction)) {
142 joinedCardinalites = providedJoinedCardinalites;
143 if (!joinedCardinalites) {
144 joinedCardinalites = createBindingsCardinalitesForVariableCompositionItems(joiningItem, buildingVarItem, memMan);
145 }
146 cardCreated = true;
147 }
148 if (leftCardinalites && rightCardinalites) {
149 *joinedCardinalites = *rightCardinalites;
150 joinedCardinalites->setSameIndividualsJointlyConsideredCardinality(joinedCardinalites->getSameIndividualsJointlyConsideredCardinality() * leftCardinalites->getSameIndividualsJointlyConsideredCardinality());
151 joinedCardinalites->setSameIndividualsSeparatlyConsideredCardinality(joinedCardinalites->getSameIndividualsSeparatlyConsideredCardinality() * leftCardinalites->getSameIndividualsSeparatlyConsideredCardinality());
152 } else if (rightCardinalites) {
153 *joinedCardinalites = *rightCardinalites;
154 } else if (leftCardinalites) {
155 *joinedCardinalites = *leftCardinalites;
156 } else if (joinedCardinalites) {
157 joinedCardinalites->initCardinalities(1, 1);
158 }
159
160 if (joinedCardinalites) {
161 joinedCardinalites->setCardinalityUpdateId(-1);
162 }
163
164
165 if (!joinedBindings) {
166 joinedBindings = createBindingsForVariableCompositionItems(joinSize, joiningItem, buildingVarItem, memMan);
167 }
168
169 for (cint64 i = 0; i < leftSize; ++i) {
170 if (leftPosMapArray[i] > -1) {
171 joinedBindings->setBinding(leftPosMapArray[i], leftBinding[i]);
172 } else if (leftPosMapArray[i] == -2) {
173 TIndividualInstanceItemDataBinding& reducedBinding = leftBinding[i];
174 CInstanceBindingIndividualCountingVisitor visitor;
175 sameRealization->visitSameIndividuals(reducedBinding.reference, &visitor);
176 if (visitor.individualCount > 1) {
177 joinedCardinalites->setSameIndividualsSeparatlyConsideredCardinality(visitor.individualCount * joinedCardinalites->getSameIndividualsSeparatlyConsideredCardinality());
178 }
179 }
180 }
181 for (cint64 i = 0; i < rightSize; ++i) {
182 if (rightPosMapArray[i] > -1) {
183 joinedBindings->setBinding(rightPosMapArray[i], rightBinding[i]);
184 } else if (rightPosMapArray[i] == -2) {
185 TIndividualInstanceItemDataBinding& reducedBinding = rightBinding[i];
186 CInstanceBindingIndividualCountingVisitor visitor;
187 sameRealization->visitSameIndividuals(reducedBinding.reference, &visitor);
188 if (visitor.individualCount > 1) {
189 joinedCardinalites->setSameIndividualsSeparatlyConsideredCardinality(visitor.individualCount * joinedCardinalites->getSameIndividualsSeparatlyConsideredCardinality());
190 }
191 }
192 }
193
194 //QStringList bindingsStringList;
195 //for (cint64 i = 0; i < joinSize; ++i) {
196 // bindingsStringList.append(CIRIName::getRecentIRIName(joinedBindings->getBinding(i).reference.getIndividual()->getIndividualNameLinker()));
197 //}
198 //QString bindingsString = bindingsStringList.join("\t");
199
200 if (!joinedLinker) {
201 joinedLinker = createBindingsLinkerForVariableCompositionItems(joinedBindings, joinedCardinalites, joiningItem, buildingVarItem, memMan);
202 } else {
203 joinedLinker->setInitialCardinality(joinedCardinalites);
204 }
205 return joinedLinker;
206 };
207
208 auto joiningVariableMappingCreationLeftRightFunc = [&]( COptimizedComplexVariableIndividualBindings* leftvarMapping, COptimizedComplexVariableIndividualBindingsCardinality* leftNewCardinalites, COptimizedComplexVariableIndividualBindingsCardinality* leftPrevCardinalites,
209 COptimizedComplexVariableIndividualBindings* rightvarMapping, COptimizedComplexVariableIndividualBindingsCardinality* rightNewCardinalites, COptimizedComplexVariableIndividualBindingsCardinality* righPrevCardinalites, bool swap,
210 COptimizedComplexVariableIndividualBindings* joinedBindings, COptimizedComplexVariableIndividualBindingsCardinalityLinker* joinedLinker, COptimizedComplexVariableIndividualBindingsCardinality* providedJoinedCardinalites, CMemoryAllocationManager* memMan) -> COptimizedComplexVariableIndividualBindingsCardinalityLinker* {
211 if (swap) {
212 return joiningVariableMappingCreationFunc(rightvarMapping, rightNewCardinalites, righPrevCardinalites, leftvarMapping, leftNewCardinalites, leftPrevCardinalites, joinedBindings, joinedLinker, providedJoinedCardinalites, memMan);
213 } else {
214 return joiningVariableMappingCreationFunc(leftvarMapping, leftNewCardinalites, leftPrevCardinalites, rightvarMapping, rightNewCardinalites, righPrevCardinalites, joinedBindings, joinedLinker, providedJoinedCardinalites, memMan);
215 }
216 };
217
218
219 auto joiningVariableMappingCreationLeftRightLinkerFunc = [&]( COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker* leftLinker, COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker* rightLinker, bool swap,
220 COptimizedComplexVariableIndividualBindings* joinedBindings, COptimizedComplexVariableIndividualBindingsCardinalityLinker* joinedLinker, COptimizedComplexVariableIndividualBindingsCardinality* providedJoinedCardinalites, CMemoryAllocationManager* memMan) -> COptimizedComplexVariableIndividualBindingsCardinalityLinker* {
221 return joiningVariableMappingCreationLeftRightFunc(leftLinker->getBindings(), leftLinker->getNewCardinalities(), leftLinker->getPreviousCardinalities(), rightLinker->getBindings(), rightLinker->getNewCardinalities(), rightLinker->getPreviousCardinalities(), swap, joinedBindings, joinedLinker, providedJoinedCardinalites, memMan);
222 };
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238 //QList<QString> images;
239 //std::function<QString(const QString &)> mapFunc = [](const QString &img) {
240 // return img + "x";
241 //};
242 //std::function<void(QString &, const QString &)> redFunc = [](QString &img, const QString &red) {
243 //};
244 //QFuture<QString> thumbnails = QtConcurrent::mapped(images, mapFunc);
245
246
247
248
249
250
251
252
253 cint64 currentBatchSizeCount = mConfBatchSizeCount;
254 cint64 currentBatchSizeLimit = mConfBatchSizeLimit;
255
256
257
258
259 function<void(bool left, COptimizedComplexVariableCompositionItemDependence* itemDep, cint64& sampleInsertionCount, CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker>& keyMappingBatchDataVector)> joiningHashSampleBatchingFunc =
260 [&](bool left, COptimizedComplexVariableCompositionItemDependence* itemDep, cint64& sampleInsertionCount, CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker>& keyMappingBatchDataVector) -> void {
261
262 cint64 batchSize = 1;
263 ++sampleInsertionCount;
264 COptimizedComplexVariableIndividualBindingsCardinalityLinker* firstBindingLinker = itemDep->getBatchCurrentBindingsCardinalityLinker(true);
265 COptimizedComplexVariableIndividualBindingsCardinalityLinker* lastBindingLinker = firstBindingLinker;
266 while (batchSize++ < currentBatchSizeLimit && itemDep->getBatchCurrentBindingsCardinalityLinker(false) && sampleInsertionCount < mConfSamplingBasedJoinMappingSize) {
267 ++sampleInsertionCount;
268 lastBindingLinker = itemDep->getBatchCurrentBindingsCardinalityLinker(true);
269 }
270
271 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker>* vecData = keyMappingBatchDataVector.getVectorLinkerData(keyMappingBatchDataVector.getNextIndex(true), true);
272 CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker* data = nullptr;
273 if (!mConfPartitionizedMemoryManagement) {
274 data = keyMappingBatchDataVector.createBatchLinker();
275 } else {
276 data = vecData->createBatchLinker();
277 }
278 data->setBatchBindingsCardinalityLinkers(firstBindingLinker, lastBindingLinker);
279 data->setInserting(true);
280 data->setLeftSide(left);
281 vecData->addBatchLinker(data);
282 };
283
284
285 cint64 batchMaxHandlingCount = mConfBatchMaxHandlingCount;
286 if (mConfAllowSplitModeActivation && joiningItem->isLastComputation() && !joiningItem->isSplitComputationMode()) {
287 batchMaxHandlingCount = mConfSplitTestingBatchMaxHandlingCount;
288 }
289
290
291 function<void(bool left, COptimizedComplexVariableCompositionItemDependence* itemDep, bool inserting, CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker>& keyMappingBatchDataVector)> joiningHashBindingsInsertingCheckingBatchingFunc =
292 [&](bool left, COptimizedComplexVariableCompositionItemDependence* itemDep, bool inserting, CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker>& keyMappingBatchDataVector) -> void {
293
294 cint64 batchHandlingCount = 0;
295 while (batchHandlingCount++ < batchMaxHandlingCount && itemDep->getBatchCurrentBindingsCardinalityLinker(false)) {
296 cint64 batchSize = 1;
297 COptimizedComplexVariableIndividualBindingsCardinalityLinker* firstBindingLinker = itemDep->getBatchCurrentBindingsCardinalityLinker(true);
298 COptimizedComplexVariableIndividualBindingsCardinalityLinker* lastBindingLinker = firstBindingLinker;
299 while (batchSize++ < currentBatchSizeLimit && itemDep->getBatchCurrentBindingsCardinalityLinker(false)) {
300 lastBindingLinker = itemDep->getBatchCurrentBindingsCardinalityLinker(true);
301 }
302
303 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker>* vecData = keyMappingBatchDataVector.getVectorLinkerData(keyMappingBatchDataVector.getNextIndex(true), true);
304 CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker* data = nullptr;
305 if (!mConfPartitionizedMemoryManagement) {
306 data = keyMappingBatchDataVector.createBatchLinker();
307 } else {
308 data = vecData->createBatchLinker();
309 }
310 data->setBatchBindingsCardinalityLinkers(firstBindingLinker, lastBindingLinker);
311 data->setInserting(inserting);
312 data->setLeftSide(left);
313 vecData->addBatchLinker(data);
314 }
315 };
316
317
318
319 function<void(bool left, COptimizedComplexVariableCompositionItemDependence* itemDep, bool inserting, CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker>& keyMappingBatchDataVector)> joiningHashCardinalitiesInsertingCheckingBatchingFunc =
320 [&](bool left, COptimizedComplexVariableCompositionItemDependence* itemDep, bool inserting, CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker>& keyMappingBatchDataVector) -> void {
321
322 cint64 batchHandlingCount = 0;
323 while (batchHandlingCount++ < batchMaxHandlingCount && itemDep->getBatchCurrentUpdatedCardinalityLinker(false)) {
324 cint64 batchSize = 1;
325 COptimizedComplexVariableIndividualUpdateCardinalityLinker* firstCardinalityLinker = itemDep->getBatchCurrentUpdatedCardinalityLinker(true);
326 COptimizedComplexVariableIndividualUpdateCardinalityLinker* lastCardinalityLinker = firstCardinalityLinker;
327 while (batchSize++ < currentBatchSizeLimit && itemDep->getBatchCurrentUpdatedCardinalityLinker(false)) {
328 lastCardinalityLinker = itemDep->getBatchCurrentUpdatedCardinalityLinker(true);
329 }
330 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker>* vecData = keyMappingBatchDataVector.getVectorLinkerData(keyMappingBatchDataVector.getNextIndex(true), true);
331 CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker* data = nullptr;
332 if (!mConfPartitionizedMemoryManagement) {
333 data = keyMappingBatchDataVector.createBatchLinker();
334 } else {
335 data = vecData->createBatchLinker();
336 }
337 data->setBatchCardinalityUpdateLinkers(firstCardinalityLinker, lastCardinalityLinker);
338 data->setInserting(inserting);
339 data->setLeftSide(left);
340 vecData->addBatchLinker(data);
341 }
342 };
343
344
345
346
347
348 bool splitComputationMode = joiningItem->isSplitComputationMode();
349 cint64 leftSplitCount = joiningItem->getLeftSplitCount();
350 cint64 rightSplitCount = joiningItem->getRightSplitCount();
351 cint64 leftSplitPos = joiningItem->getLeftSplitPosition();
352 cint64 rightSplitPos = joiningItem->getRightSplitPosition();
353
354 CXLinker<cint64>* leftRemainingBindingLinker = nullptr;
355 CXLinker<cint64>* rightRemainingBindingLinker = nullptr;
356
357 if (splitComputationMode) {
358 leftRemainingBindingLinker = variablePositionMapping->getLeftRemainingBindingLinker();
359 rightRemainingBindingLinker = variablePositionMapping->getRightRemainingBindingLinker();
360 }
361
362 cint64 splitComputationCount = 0;
363
364
365
366
367
368
369
370 // map & reduce of step 1
371 cint64 keyMappingSize = mConfConcurrentJoiningHashCount;
372
373 function<CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinMappedKeysBindingsCardinalityLinker>(const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker>* const & batchData)> bindingBatchesHashingKeyMapFunc =
374 [&](const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker>* const & batchData) ->CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinMappedKeysBindingsCardinalityLinker> {
375
376 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinMappedKeysBindingsCardinalityLinker> keyMappedVec(keyMappingSize);
377 if (batchData) {
378
379 CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker* batchLinker = batchData->getNextBatchLinker();
380 while (batchLinker) {
381
382 bool leftSide = batchLinker->isLeftSide();
383 bool inserting = batchLinker->isInserting();
384
385 COptimizedComplexVariableIndividualBindingsCardinalityLinker* bindingsCardinalityCurrentLinker = batchLinker->getNextBatchBindingsCardinalityLinker(true);
386 while (bindingsCardinalityCurrentLinker) {
387
388 CXLinker<cint64>* keyBindingLinker = leftKeyBindingLinker;
389 CXLinker<cint64>* remainingBindingLinker = leftRemainingBindingLinker;
390 cint64 splitCount = leftSplitCount;
391 cint64 splitPos = leftSplitPos;
392 if (!leftSide) {
393 keyBindingLinker = rightKeyBindingLinker;
394 remainingBindingLinker = rightRemainingBindingLinker;
395 splitPos = rightSplitPos;
396 splitCount = rightSplitCount;
397 }
398
399 COptimizedComplexVariableJoiningWithRemainingHasher bindingHasher(bindingsCardinalityCurrentLinker->getBindings()->getBindingArray(), keyBindingLinker, remainingBindingLinker);
400 cint64 keyHashValue = bindingHasher.getHashValue();
401 uint uintHashValue = uint((keyHashValue >> (8 * sizeof(uint) - 1)) ^ keyHashValue);
402 cint64 mappedKeyIndex = uintHashValue % keyMappingSize;
403
404 if (!splitComputationMode || bindingHasher.getRemainingHashValue() % splitCount == splitPos) {
405
406 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionJoinMappedKeysBindingsCardinalityLinker>* keyMappedVecData = keyMappedVec.getVectorLinkerData(mappedKeyIndex, true);
407 CQtConcurrentVariableMappingsCompositionJoinMappedKeysBindingsCardinalityLinker* mappedKeyDataLinker = nullptr;
408 if (!mConfPartitionizedMemoryManagement) {
409 mappedKeyDataLinker = keyMappedVec.createBatchLinker();
410 } else {
411 mappedKeyDataLinker = keyMappedVecData->createBatchLinker();
412 }
413 mappedKeyDataLinker->initMappedKeysBindingsCardinalityData(bindingHasher, mappedKeyIndex, bindingsCardinalityCurrentLinker->getBindings(), bindingsCardinalityCurrentLinker->getInitialCardinalities(), nullptr, leftSide, inserting);
414 keyMappedVecData->addBatchLinker(mappedKeyDataLinker);
415
416 }
417 bindingsCardinalityCurrentLinker = batchLinker->getNextBatchBindingsCardinalityLinker(true);
418 }
419
420
421
422 COptimizedComplexVariableIndividualUpdateCardinalityLinker* cardinalityUpdateLinker = batchLinker->getNextCardinalityUpdateLinker(true);
423 while (cardinalityUpdateLinker) {
424
425 CXLinker<cint64>* keyBindingLinker = leftKeyBindingLinker;
426 CXLinker<cint64>* remainingBindingLinker = leftRemainingBindingLinker;
427 cint64 splitCount = leftSplitCount;
428 cint64 splitPos = leftSplitPos;
429 if (!leftSide) {
430 keyBindingLinker = rightKeyBindingLinker;
431 remainingBindingLinker = rightRemainingBindingLinker;
432 splitPos = rightSplitPos;
433 splitCount = rightSplitCount;
434 }
435
436 COptimizedComplexVariableJoiningWithRemainingHasher bindingHasher(cardinalityUpdateLinker->getUpdatedBindingsCardinalityLinker()->getBindings()->getBindingArray(), keyBindingLinker, remainingBindingLinker);
437 cint64 keyHashValue = bindingHasher.getHashValue();
438 uint uintHashValue = uint((keyHashValue >> (8 * sizeof(uint) - 1)) ^ keyHashValue);
439 cint64 mappedKeyIndex = uintHashValue % keyMappingSize;
440
441 if (!splitComputationMode || bindingHasher.getRemainingHashValue() % splitCount == splitPos) {
442 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionJoinMappedKeysBindingsCardinalityLinker>* keyMappedVecData = keyMappedVec.getVectorLinkerData(mappedKeyIndex, true);
443 CQtConcurrentVariableMappingsCompositionJoinMappedKeysBindingsCardinalityLinker* mappedKeyDataLinker = nullptr;
444 if (!mConfPartitionizedMemoryManagement) {
445 mappedKeyDataLinker = keyMappedVec.createBatchLinker();
446 } else {
447 mappedKeyDataLinker = keyMappedVecData->createBatchLinker();
448 }
449 mappedKeyDataLinker->initMappedKeysBindingsCardinalityData(bindingHasher, mappedKeyIndex, cardinalityUpdateLinker->getUpdatedBindingsCardinalityLinker()->getBindings(), cardinalityUpdateLinker->getNewCardinality(), cardinalityUpdateLinker->getPreviousCardinality(), leftSide, inserting);
450 keyMappedVecData->addBatchLinker(mappedKeyDataLinker);
451
452 }
453 cardinalityUpdateLinker = batchLinker->getNextCardinalityUpdateLinker(true);
454 }
455
456
457 batchLinker = batchLinker->getNext();
458 }
459 }
460
461 return keyMappedVec;
462 };
463
464
465 function<void(CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinMappedKeysBindingsCardinalityLinker>& vec, const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinMappedKeysBindingsCardinalityLinker>& reduceVec)> bindingBatchesHashingKeyReduceFunc =
466 [&](CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinMappedKeysBindingsCardinalityLinker>& vec, const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinMappedKeysBindingsCardinalityLinker>& reduceVec) -> void {
467
468 if (vec.size() < reduceVec.size()) {
469 vec.resize(reduceVec.size());
470 }
471 for (cint64 i = 0; i < vec.count(); ++i) {
472 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionJoinMappedKeysBindingsCardinalityLinker>*& vecData = vec[i];
473 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionJoinMappedKeysBindingsCardinalityLinker>* const & reduceVecData = reduceVec[i];
474 if (reduceVecData) {
475 if (!vecData) {
476 vecData = reduceVecData;
477 } else {
478 vecData->mergeBatchLinkers(reduceVecData);
479 }
480 }
481
482 }
483 vec.appendMemoryPool(reduceVec.getMemoryPools());
484 };
485
486
487
488
489
490
491
492 // map & reduce of step 2
493 cint64 bindingsMappingSize = mConfConcurrentVariableMappingPartCount;
494
495
496 auto joinedVariableMappingCreationRightLeftCopyingVecInsertionFunc = [&](COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker* leftLinker, COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker* rightLinker, bool swap,
497 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinMappedJoinedBindingsCardinalityLinker>& bindingsMappedVec, CMemoryAllocationManager* memMan, COptimizedComplexVariableIndividualBindings*& tmpJoinedBindings) -> void {
498
499
500 CQtConcurrentVariableMappingsCompositionJoinMappedJoinedBindingsCardinalityLinker* mappedBindingsDataLinker = nullptr;
501 if (!mConfPartitionizedMemoryManagement) {
502 mappedBindingsDataLinker = bindingsMappedVec.createBatchLinker();
503
504 COptimizedComplexVariableIndividualBindingsCardinalityLinker* joinedLinker = joiningVariableMappingCreationLeftRightLinkerFunc(leftLinker, rightLinker, swap, nullptr, nullptr, nullptr, memMan);
505
506 COptimizedComplexVariableIndividualBindingsHasher bindingsHasher(joinedLinker);
507 cint64 keyHashValue = bindingsHasher.getHashValue();
508 cint64 mappedBindingsIndex = uint((keyHashValue >> (8 * sizeof(uint) - 1)) ^ keyHashValue) % bindingsMappingSize;
509
510 mappedBindingsDataLinker->initMappedJoinedBindingsCardinalityData(mappedBindingsIndex, bindingsHasher, joinedLinker);
511 bindingsMappedVec.addBatchLinker(mappedBindingsIndex, mappedBindingsDataLinker);
512
513 } else {
514
515 if (!tmpJoinedBindings) {
516 tmpJoinedBindings = createBindingsForVariableCompositionItems(joinSize, joiningItem, buildingVarItem, memMan);
517 }
518 COptimizedComplexVariableIndividualBindingsCardinalityLinker tmpJoinedLinker;
519 COptimizedComplexVariableIndividualBindingsCardinality tmpJoinedCardinalites;
520
521 joiningVariableMappingCreationLeftRightLinkerFunc(leftLinker, rightLinker, swap, tmpJoinedBindings, &tmpJoinedLinker, &tmpJoinedCardinalites, memMan);
522
523 COptimizedComplexVariableIndividualBindingsHasher tmpBindingsHasher(&tmpJoinedLinker);
524 cint64 keyHashValue = tmpBindingsHasher.getHashValue();
525 cint64 mappedBindingsIndex = uint((keyHashValue >> (8 * sizeof(uint) - 1)) ^ keyHashValue) % bindingsMappingSize;
526
527 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionJoinMappedJoinedBindingsCardinalityLinker>* bindingsMappedVecData = bindingsMappedVec.getVectorLinkerData(mappedBindingsIndex, true);
528
529 // copy data with corresponding memory manager
530 memMan = bindingsMappedVecData->getMemoryAllocationManager();
531 COptimizedComplexVariableIndividualBindings* joinedBindings = createBindingsForVariableCompositionItems(joinSize, joiningItem, buildingVarItem, memMan);
532 for (cint64 i = 0; i < joinSize; ++i) {
533 joinedBindings->setBinding(i, tmpJoinedBindings->getBinding(i));
534 }
535 COptimizedComplexVariableIndividualBindingsCardinality* joinedCardinalites = nullptr;
536 if (tmpJoinedLinker.getCurrentCardinalities()) {
537 joinedCardinalites = createBindingsCardinalitesForVariableCompositionItems(joiningItem, buildingVarItem, memMan);
538 *joinedCardinalites = tmpJoinedCardinalites;
539 }
540 COptimizedComplexVariableIndividualBindingsCardinalityLinker* joinedLinker = createBindingsLinkerForVariableCompositionItems(joinedBindings, joinedCardinalites, joiningItem, buildingVarItem, memMan);
541 COptimizedComplexVariableIndividualBindingsHasher bindingsHasher(joinedLinker, keyHashValue);
542
543 mappedBindingsDataLinker = bindingsMappedVecData->createBatchLinker();
544 mappedBindingsDataLinker->initMappedJoinedBindingsCardinalityData(mappedBindingsIndex, bindingsHasher, joinedLinker);
545 bindingsMappedVecData->addBatchLinker(mappedBindingsDataLinker);
546 }
547 };
548
549
550
551
552 function<CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinMappedJoinedBindingsCardinalityLinker>(const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionJoinMappedKeysBindingsCardinalityLinker>* const & keyMappedBindingsVecData)> bindingBatchesHashingJoinMapFunc =
553 [&](const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionJoinMappedKeysBindingsCardinalityLinker>* const & keyMappedBindingsVecData) -> CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinMappedJoinedBindingsCardinalityLinker> {
554
555 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinMappedJoinedBindingsCardinalityLinker> bindingsMappedVec(bindingsMappingSize);
556
557 COptimizedComplexVariableJoiningHashMemoryManaged* lastJoiningHash = nullptr;
558 QTime* time = nullptr;
559 if (mConfPerformanceLogging) {
560 time = new QTime();
561 time->start();
562 }
563 if (keyMappedBindingsVecData) {
564 CQtConcurrentVariableMappingsCompositionJoinMappedKeysBindingsCardinalityLinker* batchLinker = keyMappedBindingsVecData->getNextBatchLinker();
565 COptimizedComplexVariableIndividualBindings* tmpJoinedBindings = nullptr;
566
567 while (batchLinker) {
568
569 cint64 mappedKeyIndex = batchLinker->getMappedKeyIndex();
570 COptimizedComplexVariableJoiningHashMemoryManaged*& joiningHash = joiningHashVector[mappedKeyIndex];
571 if (!joiningHash) {
572 joiningHash = new COptimizedComplexVariableJoiningHashMemoryManaged(leftKeyBindingLinker, rightKeyBindingLinker);
573 }
574 lastJoiningHash = joiningHash;
575
576 const COptimizedComplexVariableJoiningHasher& bindingHasher = batchLinker->getBindingHasher();
577
578 bool sampling = !joiningItem->isSamplingCompleted();
579 bool leftSide = batchLinker->isLeftSide();
580 bool inserting = batchLinker->isInserting();
581
582
583
584 COptimizedComplexVariableIndividualBindings* varMapping = batchLinker->getVariableMapping();
585 COptimizedComplexVariableIndividualBindingsCardinality* newCardinalites = batchLinker->getNewCardinalites();
586 COptimizedComplexVariableIndividualBindingsCardinality* prevCardinalites = batchLinker->getPreviousCardinalites();
587
588 COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker* linker = joiningHash->createDataLinker();
589 if (inserting) {
590 COptimizedComplexVariableJoiningData& joiningData = (*joiningHash)[bindingHasher];
591
592 if (sampling && !joiningData.hasBindings(leftSide)) {
593 // TODO: make thread/concurrent safe
594 if (leftSide) {
595 atomicLeftSampleKeyCount.ref();
596 } else {
597 atomicRightSampleKeyCount.ref();
598 }
599 }
600
601 linker->initLinker(varMapping, newCardinalites, prevCardinalites);
602 #ifdef CONCURRENT_JOIN_COMPUTATION_PERFORMANCE_MEASUREMENT
603 joiningData.mInsertionCount++;
604 #endif //CONCURRENT_JOIN_COMPUTATION_PERFORMANCE_MEASUREMENT
605 joiningHash->mInsertionCount++;
606 for (COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker* existingOtherLinkerIt = joiningData.getBindingLinker(!leftSide); existingOtherLinkerIt; existingOtherLinkerIt = existingOtherLinkerIt->getNext()) {
607 joinedVariableMappingCreationRightLeftCopyingVecInsertionFunc(existingOtherLinkerIt, linker, leftSide, bindingsMappedVec, bindingsMappedVec.getMemoryAllocationManager(), tmpJoinedBindings);
608 #ifdef CONCURRENT_JOIN_COMPUTATION_PERFORMANCE_MEASUREMENT
609 joiningData.mJoinedCount++;
610 #endif //CONCURRENT_JOIN_COMPUTATION_PERFORMANCE_MEASUREMENT
611 joiningHash->mJoinedCount++;
612 }
613 joiningData.addBindingLinker(leftSide, linker);
614 if (leftSide) {
615 #ifdef CONCURRENT_JOIN_COMPUTATION_PERFORMANCE_MEASUREMENT
616 joiningData.mLeftAdded++;
617 #endif //CONCURRENT_JOIN_COMPUTATION_PERFORMANCE_MEASUREMENT
618 joiningHash->mLeftAdded++;
619 } else {
620 #ifdef CONCURRENT_JOIN_COMPUTATION_PERFORMANCE_MEASUREMENT
621 joiningData.mRightAdded++;
622 #endif //CONCURRENT_JOIN_COMPUTATION_PERFORMANCE_MEASUREMENT
623 joiningHash->mRightAdded++;
624 }
625 } else {
626 const COptimizedComplexVariableJoiningData& joiningData = joiningHash->value(bindingHasher);
627
628 #ifdef CONCURRENT_JOIN_COMPUTATION_PERFORMANCE_MEASUREMENT
629 joiningData.mCheckingCount++;
630 #endif //CONCURRENT_JOIN_COMPUTATION_PERFORMANCE_MEASUREMENT
631 joiningHash->mCheckingCount++;
632 COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker tmpLinker(varMapping, newCardinalites, prevCardinalites);
633 for (COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker* existingOtherLinkerIt = joiningData.getBindingLinker(!leftSide); existingOtherLinkerIt; existingOtherLinkerIt = existingOtherLinkerIt->getNext()) {
634 joinedVariableMappingCreationRightLeftCopyingVecInsertionFunc(existingOtherLinkerIt, &tmpLinker, leftSide, bindingsMappedVec, bindingsMappedVec.getMemoryAllocationManager(), tmpJoinedBindings);
635 #ifdef CONCURRENT_JOIN_COMPUTATION_PERFORMANCE_MEASUREMENT
636 joiningData.mJoinedCount++;
637 #endif //CONCURRENT_JOIN_COMPUTATION_PERFORMANCE_MEASUREMENT
638 joiningHash->mJoinedCount++;
639 }
640 }
641
642 batchLinker = batchLinker->getNext();
643 }
644 }
645
646 if (lastJoiningHash && time) {
647 lastJoiningHash->mProcessingTime += time->elapsed();
648 delete time;
649 }
650
651
652 return bindingsMappedVec;
653 };
654
655
656
657
658 function<void(CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinMappedJoinedBindingsCardinalityLinker>& vec, const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinMappedJoinedBindingsCardinalityLinker>& reduceVec)> bindingBatchesHashingJoinReduceFunc =
659 [&](CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinMappedJoinedBindingsCardinalityLinker>& vec, const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinMappedJoinedBindingsCardinalityLinker>& reduceVec) -> void {
660
661 if (vec.size() < reduceVec.size()) {
662 vec.resize(reduceVec.size());
663 }
664 for (cint64 i = 0; i < vec.count(); ++i) {
665 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionJoinMappedJoinedBindingsCardinalityLinker>*& vecData = vec[i];
666 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionJoinMappedJoinedBindingsCardinalityLinker>* const & reduceVecData = reduceVec[i];
667 if (reduceVecData) {
668 if (!vecData) {
669 vecData = reduceVecData;
670 } else {
671 vecData->mergeBatchLinkers(reduceVecData);
672 }
673 }
674
675 }
676 vec.appendMemoryPool(reduceVec.getMemoryPools());
677 };
678
679
680
681
682
683
684
685
686
687 // map (& reduce) of step 3
688
689 function<COptimizedComplexVariableIndividualMappingsMultiHashPart*(const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionJoinMappedJoinedBindingsCardinalityLinker>* const & mappedBindingsVecData)> bindingBatchesInsertingMapFunc =
690 [&](const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionJoinMappedJoinedBindingsCardinalityLinker>* const & mappedBindingsVecData) -> COptimizedComplexVariableIndividualMappingsMultiHashPart* {
691
692
693
694 QTime* time = nullptr;
695 if (mConfPerformanceLogging) {
696 time = new QTime();
697 time->start();
698 }
699
700 COptimizedComplexVariableIndividualMappingsMultiHashPart* joinedVarMappingMultiHashPart = nullptr;
701
702 if (mappedBindingsVecData) {
703
704 CQtConcurrentVariableMappingsCompositionJoinMappedJoinedBindingsCardinalityLinker* batchLinker = mappedBindingsVecData->getNextBatchLinker();
705
706 while (batchLinker) {
707
708 cint64 mappedBindingsIndex = batchLinker->getBindingsIndex();
709 if (!joinedVarMappingMultiHashPart) {
710 joinedVarMappingMultiHashPart = joinedVarMapping->getMultiHashPart(mappedBindingsIndex);
711 }
712
713
714 COptimizedComplexVariableIndividualBindingsCardinalityLinker* joinedLinker = batchLinker->getBindingsCardinalityLinker();
715 cint64 hashValue = batchLinker->getBindingHasher().getHashValue();
716
717 joinedVarMappingMultiHashPart->addInsertingBindingsCardinalityLinkerCopy(joinedLinker, hashValue, true);
718
719 batchLinker = batchLinker->getNext();
720 }
721 }
722
723 if (joinedVarMappingMultiHashPart && time) {
724 joinedVarMappingMultiHashPart->mStatsProcessingTime += time->elapsed();
725 delete time;
726 }
727
728
729 return joinedVarMappingMultiHashPart;
730 };
731
732
733
734
735 function<void(const QVector<COptimizedComplexVariableIndividualMappingsMultiHashPart*>& mappedVarMappingParts)> bindingBatchesInsertingReduceFunc =
736 [&](const QVector<COptimizedComplexVariableIndividualMappingsMultiHashPart*>& mappedVarMappingParts) -> void {
737
738 for (QVector<COptimizedComplexVariableIndividualMappingsMultiHashPart*>::const_iterator it = mappedVarMappingParts.constBegin(), itEnd = mappedVarMappingParts.constEnd(); it != itEnd; ++it) {
739 COptimizedComplexVariableIndividualMappingsMultiHashPart* reduceMappPart = (*it);
740 if (reduceMappPart) {
741 joinedVarMapping->updateLinkers(reduceMappPart);
742 joinedVarMapping->updateBindingCount(reduceMappPart);
743 }
744 }
745 };
746
747
748
749
750 function<void(CMemoryPool* memPools)> clearMemoryPoolsFunc = [&](CMemoryPool* memPools) -> void {
751 while (memPools) {
752 CMemoryPool* tmpMemPool = memPools;
753 memPools = memPools->getNextMemoryPool();
754 delete [] tmpMemPool->getMemoryBlockData();
755 delete tmpMemPool;
756 }
757 };
758
759 function<void(CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker>& keyMappingBatchDataVector)> mapReducedConcurrentJoinInsertionComputation = [&](CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker>& keyMappingBatchDataVector) -> void {
760 if (!keyMappingBatchDataVector.isEmpty()) {
761
762 QTime* stepTimer = nullptr;
763 if (mConfPerformanceLogging) {
764 stepTimer = new QTime();
765 stepTimer->start();
766 }
767 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinMappedKeysBindingsCardinalityLinker> mappedJoiningHashKeyVec = QtConcurrent::blockingMappedReduced<CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinMappedKeysBindingsCardinalityLinker>>(keyMappingBatchDataVector, bindingBatchesHashingKeyMapFunc, bindingBatchesHashingKeyReduceFunc);
768 if (stepTimer) {
769 cint64 partitionsUsed = 0;
770 for (cint64 i = 0; i < keyMappingBatchDataVector.size(); ++i) {
771 if (keyMappingBatchDataVector[i]) {
772 ++partitionsUsed;
773 }
774 }
775 QString x1 = QString("Insertion step 1 of concurrent join computation for item %3 with %4/%1 tasks took %2 ms.").arg(keyMappingBatchDataVector.size()).arg(stepTimer->elapsed()).arg(joiningItem->getComputationStepId()).arg(partitionsUsed);
776 if (mConfDirectPerformanceConsoleOutput) {
777 cout << x1.toLocal8Bit().constData() << endl;
778 } else {
779 LOG(INFO, getLogDomain(), x1, this);
780 }
781 delete stepTimer;
782 }
783
784
785
786 if (mConfPerformanceLogging) {
787 stepTimer = new QTime();
788 stepTimer->start();
789 }
790 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinMappedJoinedBindingsCardinalityLinker> mappedJoinedHashKeyVec = QtConcurrent::blockingMappedReduced<CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinMappedJoinedBindingsCardinalityLinker>>(mappedJoiningHashKeyVec, bindingBatchesHashingJoinMapFunc, bindingBatchesHashingJoinReduceFunc);
791 if (stepTimer) {
792 cint64 partitionsUsed = 0;
793 for (cint64 i = 0; i < mappedJoiningHashKeyVec.size(); ++i) {
794 if (mappedJoiningHashKeyVec[i]) {
795 ++partitionsUsed;
796 }
797 }
798 QString x2 = QString("Insertion step 2 of concurrent join computation for item %3 with %4/%1 tasks took %2 ms.").arg(mappedJoiningHashKeyVec.size()).arg(stepTimer->elapsed()).arg(joiningItem->getComputationStepId()).arg(partitionsUsed);
799 if (mConfDirectPerformanceConsoleOutput) {
800 cout << x2.toLocal8Bit().constData() << endl;
801 } else {
802 LOG(INFO, getLogDomain(), x2, this);
803 }
804 delete stepTimer;
805
806 cint64 statMaxJoiningCount = 0;
807 cint64 statMaxProcessingTime = 0;
808 cint64 statTotalJoiningCount = 0;
809 cint64 statTotalProcessingTime = 0;
810 cint64 statUsedCount = 0;
811 for (cint64 i = 0; i < mappedJoiningHashKeyVec.size(); ++i) {
812 COptimizedComplexVariableJoiningHashMemoryManaged* joiningHash = joiningHashVector[i];
813 if (joiningHash && joiningHash->mJoinedCount > 0) {
814 ++statUsedCount;
815 statMaxJoiningCount = qMax(statMaxJoiningCount, joiningHash->mJoinedCount);
816 statMaxProcessingTime = qMax(statMaxProcessingTime, joiningHash->mProcessingTime);
817 statTotalJoiningCount += joiningHash->mJoinedCount;
818 statTotalProcessingTime += joiningHash->mProcessingTime;
819 }
820 }
821 cint64 averageJoiningCount = 0;
822 if (statUsedCount > 0) {
823 averageJoiningCount = statTotalJoiningCount / statUsedCount;
824 }
825 cint64 averageProcessingTime = 0;
826 if (statUsedCount > 0) {
827 averageProcessingTime = statTotalProcessingTime / statUsedCount;
828 }
829 QString x3 = QString("Insertion step 2 of concurrent join computation for item %1 had in average %2 (max %3) created joins and took in average %4 (max %5) ms.").arg(joiningItem->getComputationStepId()).arg(averageJoiningCount).arg(statMaxJoiningCount).arg(averageProcessingTime).arg(statMaxProcessingTime);
830 if (mConfDirectPerformanceConsoleOutput) {
831 cout << x3.toLocal8Bit().constData() << endl;
832 } else {
833 LOG(INFO, getLogDomain(), x3, this);
834 }
835 }
836
837
838
839 if (mConfPerformanceLogging) {
840 stepTimer = new QTime();
841 stepTimer->start();
842 }
843 QVector<COptimizedComplexVariableIndividualMappingsMultiHashPart*> mappedVarMappingParts = QtConcurrent::blockingMapped<QVector<COptimizedComplexVariableIndividualMappingsMultiHashPart*>>(mappedJoinedHashKeyVec, bindingBatchesInsertingMapFunc);
844 if (stepTimer) {
845 cint64 partitionsUsed = 0;
846 for (cint64 i = 0; i < mappedJoinedHashKeyVec.size(); ++i) {
847 if (mappedJoinedHashKeyVec[i]) {
848 ++partitionsUsed;
849 }
850 }
851 QString x4 = QString("Insertion step 3 of concurrent join computation for item %3 with %4/%1 tasks took %2 ms.").arg(mappedJoinedHashKeyVec.size()).arg(stepTimer->elapsed()).arg(joiningItem->getComputationStepId()).arg(partitionsUsed);
852 if (mConfDirectPerformanceConsoleOutput) {
853 cout << x4.toLocal8Bit().constData() << endl;
854 } else {
855 LOG(INFO, getLogDomain(), x4, this);
856 }
857 delete stepTimer;
858
859 cint64 statMaxInsertionCount = 0;
860 cint64 statMaxProcessingTime = 0;
861 cint64 statTotalInsertionCount = 0;
862 cint64 statTotalProcessingTime = 0;
863 cint64 statUsedCount = 0;
864 for (cint64 i = 0; i < mappedJoinedHashKeyVec.size(); ++i) {
865 COptimizedComplexVariableIndividualMappingsMultiHashPart* joinedVarMappingMultiHashPart = joinedVarMapping->getMultiHashPart(i);
866 if (joinedVarMappingMultiHashPart && joinedVarMappingMultiHashPart->mStatInsertionCount > 0) {
867 ++statUsedCount;
868 statMaxInsertionCount = qMax(statMaxInsertionCount, joinedVarMappingMultiHashPart->mStatInsertionCount);
869 statMaxProcessingTime = qMax(statMaxProcessingTime, joinedVarMappingMultiHashPart->mStatsProcessingTime);
870 statTotalInsertionCount += joinedVarMappingMultiHashPart->mStatInsertionCount;
871 statTotalProcessingTime += joinedVarMappingMultiHashPart->mStatsProcessingTime;
872 }
873 }
874 cint64 averageInsertionCount = 0;
875 if (statUsedCount > 0) {
876 averageInsertionCount = statTotalInsertionCount / statUsedCount;
877 }
878 cint64 averageProcessingTime = 0;
879 if (statUsedCount > 0) {
880 averageProcessingTime = statTotalProcessingTime / statUsedCount;
881 }
882 QString x5 = QString("Insertion step 3 of concurrent join computation for item %1 had in average %2 (max %3) bindings insertions and took in average %4 (max %5) ms.").arg(joiningItem->getComputationStepId()).arg(averageInsertionCount).arg(statMaxInsertionCount).arg(averageProcessingTime).arg(statMaxProcessingTime);
883 if (mConfDirectPerformanceConsoleOutput) {
884 cout << x5.toLocal8Bit().constData() << endl;
885 } else {
886 LOG(INFO, getLogDomain(), x5, this);
887 }
888 }
889
890
891
892 if (mConfPerformanceLogging) {
893 stepTimer = new QTime();
894 stepTimer->start();
895 }
896 bindingBatchesInsertingReduceFunc(mappedVarMappingParts);
897 CMemoryPool* mappedKeyBatchHashVecMemPools = keyMappingBatchDataVector.takeMemoryPools();
898 CMemoryPool* mappedJoiningHashVecMemPools = mappedJoiningHashKeyVec.takeMemoryPools();
899 CMemoryPool* mappedJoinedHashVecMemPools = mappedJoinedHashKeyVec.takeMemoryPools();
900 CConcurrentTaskScheduler::run([=] {
901 clearMemoryPoolsFunc(mappedKeyBatchHashVecMemPools);
902 });
903 CConcurrentTaskScheduler::run([=] {
904 clearMemoryPoolsFunc(mappedJoiningHashVecMemPools);
905 });
906 CConcurrentTaskScheduler::run([=] {
907 clearMemoryPoolsFunc(mappedJoinedHashVecMemPools);
908 });
909 keyMappingBatchDataVector.clear();
910 if (stepTimer) {
911 cint64 partitionsUsed = 0;
912 for (cint64 i = 0; i < mappedVarMappingParts.size(); ++i) {
913 if (mappedVarMappingParts[i]) {
914 ++partitionsUsed;
915 }
916 }
917 QString x6 = QString("Insertion step 4 of concurrent join computation for item %3 with %4/%1 tasks took %2 ms.").arg(mappedVarMappingParts.size()).arg(stepTimer->elapsed()).arg(joiningItem->getComputationStepId()).arg(partitionsUsed);
918 if (mConfDirectPerformanceConsoleOutput) {
919 cout << x6.toLocal8Bit().constData() << endl;
920 } else {
921 LOG(INFO, getLogDomain(), x6, this);
922 }
923 delete stepTimer;
924 }
925
926 }
927 };
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960 function<CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinMappedJoinedBindingsCardinalityLinker>(const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker>* const & batchData)> bindingBatchesHashingDirectJoiningMapFunc =
961 [&](const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker>* const & batchData) ->CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinMappedJoinedBindingsCardinalityLinker> {
962
963 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinMappedJoinedBindingsCardinalityLinker> bindingsMappedVec(bindingsMappingSize);
964 if (batchData) {
965 COptimizedComplexVariableIndividualBindings* tmpJoinedBindings = nullptr;
966
967 CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker* batchLinker = batchData->getNextBatchLinker();
968 while (batchLinker) {
969
970 bool leftSide = batchLinker->isLeftSide();
971
972 COptimizedComplexVariableIndividualBindingsCardinalityLinker* bindingsCardinalityCurrentLinker = batchLinker->getNextBatchBindingsCardinalityLinker(true);
973 while (bindingsCardinalityCurrentLinker) {
974
975 CXLinker<cint64>* keyBindingLinker = leftKeyBindingLinker;
976 CXLinker<cint64>* remainingBindingLinker = leftRemainingBindingLinker;
977 cint64 splitCount = leftSplitCount;
978 cint64 splitPos = leftSplitPos;
979 if (!leftSide) {
980 keyBindingLinker = rightKeyBindingLinker;
981 remainingBindingLinker = rightRemainingBindingLinker;
982 splitPos = rightSplitPos;
983 splitCount = rightSplitCount;
984 }
985
986 COptimizedComplexVariableJoiningWithRemainingHasher bindingHasher(bindingsCardinalityCurrentLinker->getBindings()->getBindingArray(), keyBindingLinker, remainingBindingLinker);
987 cint64 keyHashValue = bindingHasher.getHashValue();
988 uint uintHashValue = uint((keyHashValue >> (8 * sizeof(uint) - 1)) ^ keyHashValue);
989 cint64 mappedKeyIndex = uintHashValue % keyMappingSize;
990
991 if (!splitComputationMode || bindingHasher.getRemainingHashValue() % splitCount == splitPos) {
992 COptimizedComplexVariableJoiningHashMemoryManaged* joiningHash = joiningHashVector[mappedKeyIndex];
993
994 if (joiningHash) {
995
996 const COptimizedComplexVariableJoiningData& joiningData = joiningHash->value(bindingHasher);
997
998 bool leftSide = batchLinker->isLeftSide();
999
1000 COptimizedComplexVariableIndividualBindings* varMapping = bindingsCardinalityCurrentLinker->getBindings();
1001 COptimizedComplexVariableIndividualBindingsCardinality* newCardinalites = bindingsCardinalityCurrentLinker->getInitialCardinalities();
1002 COptimizedComplexVariableIndividualBindingsCardinality* prevCardinalites = nullptr;
1003
1004 COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker* linker = joiningHash->createDataLinker();
1005
1006 COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker tmpLinker(varMapping, newCardinalites, prevCardinalites);
1007 for (COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker* existingOtherLinkerIt = joiningData.getBindingLinker(!leftSide); existingOtherLinkerIt; existingOtherLinkerIt = existingOtherLinkerIt->getNext()) {
1008 joinedVariableMappingCreationRightLeftCopyingVecInsertionFunc(existingOtherLinkerIt, &tmpLinker, leftSide, bindingsMappedVec, bindingsMappedVec.getMemoryAllocationManager(), tmpJoinedBindings);
1009 }
1010 }
1011
1012 }
1013
1014 bindingsCardinalityCurrentLinker = batchLinker->getNextBatchBindingsCardinalityLinker(true);
1015 }
1016
1017
1018
1019 COptimizedComplexVariableIndividualUpdateCardinalityLinker* cardinalityUpdateLinker = batchLinker->getNextCardinalityUpdateLinker(true);
1020 while (cardinalityUpdateLinker) {
1021
1022 CXLinker<cint64>* keyBindingLinker = leftKeyBindingLinker;
1023 CXLinker<cint64>* remainingBindingLinker = leftRemainingBindingLinker;
1024 cint64 splitCount = leftSplitCount;
1025 cint64 splitPos = leftSplitPos;
1026 if (!leftSide) {
1027 keyBindingLinker = rightKeyBindingLinker;
1028 remainingBindingLinker = rightRemainingBindingLinker;
1029 splitPos = rightSplitPos;
1030 splitCount = rightSplitCount;
1031 }
1032
1033 COptimizedComplexVariableJoiningWithRemainingHasher bindingHasher(cardinalityUpdateLinker->getUpdatedBindingsCardinalityLinker()->getBindings()->getBindingArray(), keyBindingLinker, remainingBindingLinker);
1034 cint64 keyHashValue = bindingHasher.getHashValue();
1035 uint uintHashValue = uint((keyHashValue >> (8 * sizeof(uint) - 1)) ^ keyHashValue);
1036 cint64 mappedKeyIndex = uintHashValue % keyMappingSize;
1037
1038 if (!splitComputationMode || bindingHasher.getRemainingHashValue() % splitCount == splitPos) {
1039 COptimizedComplexVariableJoiningHashMemoryManaged* joiningHash = joiningHashVector[mappedKeyIndex];
1040
1041 if (joiningHash) {
1042
1043 const COptimizedComplexVariableJoiningData& joiningData = joiningHash->value(bindingHasher);
1044
1045 bool leftSide = batchLinker->isLeftSide();
1046
1047 COptimizedComplexVariableIndividualBindings* varMapping = cardinalityUpdateLinker->getUpdatedBindingsCardinalityLinker()->getBindings();
1048 COptimizedComplexVariableIndividualBindingsCardinality* newCardinalites = cardinalityUpdateLinker->getNewCardinality();
1049 COptimizedComplexVariableIndividualBindingsCardinality* prevCardinalites = cardinalityUpdateLinker->getPreviousCardinality();
1050
1051 COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker* linker = joiningHash->createDataLinker();
1052
1053 COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker tmpLinker(varMapping, newCardinalites, prevCardinalites);
1054 for (COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker* existingOtherLinkerIt = joiningData.getBindingLinker(!leftSide); existingOtherLinkerIt; existingOtherLinkerIt = existingOtherLinkerIt->getNext()) {
1055 joinedVariableMappingCreationRightLeftCopyingVecInsertionFunc(existingOtherLinkerIt, &tmpLinker, leftSide, bindingsMappedVec, bindingsMappedVec.getMemoryAllocationManager(), tmpJoinedBindings);
1056 }
1057 }
1058 }
1059
1060
1061 cardinalityUpdateLinker = batchLinker->getNextCardinalityUpdateLinker(true);
1062 }
1063
1064
1065 batchLinker = batchLinker->getNext();
1066 }
1067 }
1068
1069 return bindingsMappedVec;
1070 };
1071
1072
1073
1074
1075
1076 function<void(CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker>& keyMappingBatchDataVector)> mapReducedConcurrentJoinCheckingComputation = [&](CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker>& keyMappingBatchDataVector) -> void {
1077 if (!keyMappingBatchDataVector.isEmpty()) {
1078
1079 QTime* stepTimer = nullptr;
1080 if (mConfPerformanceLogging) {
1081 stepTimer = new QTime();
1082 stepTimer->start();
1083 }
1084 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinMappedJoinedBindingsCardinalityLinker> mappedJoinedHashKeyVec = QtConcurrent::blockingMappedReduced<CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinMappedJoinedBindingsCardinalityLinker>>(keyMappingBatchDataVector, bindingBatchesHashingDirectJoiningMapFunc, bindingBatchesHashingJoinReduceFunc);
1085 if (stepTimer) {
1086 cint64 partitionsUsed = 0;
1087 for (cint64 i = 0; i < joiningHashVector.size(); ++i) {
1088 if (joiningHashVector[i]) {
1089 ++partitionsUsed;
1090 }
1091 }
1092 QString x1 = QString("Checking step 1 of concurrent join computation for item %3 with %4/%1 tasks took %2 ms.").arg(joiningHashVector.size()).arg(stepTimer->elapsed()).arg(joiningItem->getComputationStepId()).arg(partitionsUsed);
1093 if (mConfDirectPerformanceConsoleOutput) {
1094 cout << x1.toLocal8Bit().constData() << endl;
1095 } else {
1096 LOG(INFO, getLogDomain(), x1, this);
1097 }
1098 delete stepTimer;
1099
1100 cint64 statMaxJoiningCount = 0;
1101 cint64 statMaxProcessingTime = 0;
1102 cint64 statTotalJoiningCount = 0;
1103 cint64 statTotalProcessingTime = 0;
1104 cint64 statUsedCount = 0;
1105 for (cint64 i = 0; i < joiningHashVector.size(); ++i) {
1106 COptimizedComplexVariableJoiningHashMemoryManaged* joiningHash = joiningHashVector[i];
1107 if (joiningHash && joiningHash->mJoinedCount > 0) {
1108 ++statUsedCount;
1109 statMaxJoiningCount = qMax(statMaxJoiningCount, joiningHash->mJoinedCount);
1110 statMaxProcessingTime = qMax(statMaxProcessingTime, joiningHash->mProcessingTime);
1111 statTotalJoiningCount += joiningHash->mJoinedCount;
1112 statTotalProcessingTime += joiningHash->mProcessingTime;
1113 }
1114 }
1115 cint64 averageJoiningCount = 0;
1116 if (statUsedCount > 0) {
1117 averageJoiningCount = statTotalJoiningCount / statUsedCount;
1118 }
1119 cint64 averageProcessingTime = 0;
1120 if (statUsedCount > 0) {
1121 averageProcessingTime = statTotalProcessingTime / statUsedCount;
1122 }
1123
1124 QString x2 = QString("Checking step 1 of concurrent join computation for item %1 had in average %2 (max %3) created joins and took in average %4 (max %5) ms.").arg(joiningItem->getComputationStepId()).arg(averageJoiningCount).arg(statMaxJoiningCount).arg(averageProcessingTime).arg(statMaxProcessingTime);
1125 if (mConfDirectPerformanceConsoleOutput) {
1126 cout << x2.toLocal8Bit().constData() << endl;
1127 } else {
1128 LOG(INFO, getLogDomain(), x2, this);
1129 }
1130 }
1131
1132
1133
1134
1135
1136 if (mConfPerformanceLogging) {
1137 stepTimer = new QTime();
1138 stepTimer->start();
1139 }
1140 QVector<COptimizedComplexVariableIndividualMappingsMultiHashPart*> mappedVarMappingParts = QtConcurrent::blockingMapped<QVector<COptimizedComplexVariableIndividualMappingsMultiHashPart*>>(mappedJoinedHashKeyVec, bindingBatchesInsertingMapFunc);
1141 if (stepTimer) {
1142 cint64 partitionsUsed = 0;
1143 for (cint64 i = 0; i < mappedJoinedHashKeyVec.size(); ++i) {
1144 if (mappedJoinedHashKeyVec[i]) {
1145 ++partitionsUsed;
1146 }
1147 }
1148 QString x3 = QString("Checking step 2 of concurrent join computation for item %3 with %4/%1 tasks took %2 ms.").arg(mappedJoinedHashKeyVec.size()).arg(stepTimer->elapsed()).arg(joiningItem->getComputationStepId()).arg(partitionsUsed);
1149 if (mConfDirectPerformanceConsoleOutput) {
1150 cout << x3.toLocal8Bit().constData() << endl;
1151 } else {
1152 LOG(INFO, getLogDomain(), x3, this);
1153 }
1154 delete stepTimer;
1155
1156 cint64 statMaxInsertionCount = 0;
1157 cint64 statMaxProcessingTime = 0;
1158 cint64 statTotalInsertionCount = 0;
1159 cint64 statTotalProcessingTime = 0;
1160 cint64 statUsedCount = 0;
1161 for (cint64 i = 0; i < mappedJoinedHashKeyVec.size(); ++i) {
1162 COptimizedComplexVariableIndividualMappingsMultiHashPart* joinedVarMappingMultiHashPart = joinedVarMapping->getMultiHashPart(i);
1163 if (joinedVarMappingMultiHashPart && joinedVarMappingMultiHashPart->mStatInsertionCount > 0) {
1164 ++statUsedCount;
1165 statMaxInsertionCount = qMax(statMaxInsertionCount, joinedVarMappingMultiHashPart->mStatInsertionCount);
1166 statMaxProcessingTime = qMax(statMaxProcessingTime, joinedVarMappingMultiHashPart->mStatsProcessingTime);
1167 statTotalInsertionCount += joinedVarMappingMultiHashPart->mStatInsertionCount;
1168 statTotalProcessingTime += joinedVarMappingMultiHashPart->mStatsProcessingTime;
1169 }
1170 }
1171 cint64 averageTotalInsertionCount = 0;
1172 if (statUsedCount > 0) {
1173 averageTotalInsertionCount = statTotalInsertionCount / statUsedCount;
1174 }
1175 cint64 averageProcessingTime = 0;
1176 if (statUsedCount > 0) {
1177 averageProcessingTime = statTotalProcessingTime / statUsedCount;
1178 }
1179
1180 QString x4 = QString("Checking step 2 of concurrent join computation for item %1 had in average %2 (max %3) bindings insertions and took in average %4 (max %5) ms.").arg(joiningItem->getComputationStepId()).arg(averageTotalInsertionCount).arg(statMaxInsertionCount).arg(averageProcessingTime).arg(statMaxProcessingTime);
1181 if (mConfDirectPerformanceConsoleOutput) {
1182 cout << x4.toLocal8Bit().constData() << endl;
1183 } else {
1184 LOG(INFO, getLogDomain(), x4, this);
1185 }
1186 }
1187
1188
1189
1190 if (mConfPerformanceLogging) {
1191 stepTimer = new QTime();
1192 stepTimer->start();
1193 }
1194 bindingBatchesInsertingReduceFunc(mappedVarMappingParts);
1195 CMemoryPool* mappedKeyBatchHashVecMemPools = keyMappingBatchDataVector.takeMemoryPools();
1196 CMemoryPool* mappedJoinedHashVecMemPools = mappedJoinedHashKeyVec.takeMemoryPools();
1197 CConcurrentTaskScheduler::run([=] {
1198 clearMemoryPoolsFunc(mappedKeyBatchHashVecMemPools);
1199 });
1200 CConcurrentTaskScheduler::run([=] {
1201 clearMemoryPoolsFunc(mappedJoinedHashVecMemPools);
1202 });
1203 keyMappingBatchDataVector.clear();
1204 if (stepTimer) {
1205 cint64 partitionsUsed = 0;
1206 for (cint64 i = 0; i < mappedVarMappingParts.size(); ++i) {
1207 if (mappedVarMappingParts[i]) {
1208 ++partitionsUsed;
1209 }
1210 }
1211 QString x5 = QString("Checking step 3 of concurrent join computation for item %3 with %4/%1 tasks took %2 ms.").arg(mappedVarMappingParts.size()).arg(stepTimer->elapsed()).arg(joiningItem->getComputationStepId()).arg(partitionsUsed);
1212 if (mConfDirectPerformanceConsoleOutput) {
1213 cout << x5.toLocal8Bit().constData() << endl;
1214 } else {
1215 LOG(INFO, getLogDomain(), x5, this);
1216 }
1217 delete stepTimer;
1218 }
1219
1220 }
1221 };
1222
1223
1224 auto checkingHandlingFunc = mapReducedConcurrentJoinInsertionComputation;
1225 if (mConfCheckingSideDirectJoining) {
1226 checkingHandlingFunc = mapReducedConcurrentJoinCheckingComputation;
1227 }
1228
1229
1230
1231
1232
1233 //if (leftVarMapping->size() > mConfSamplingBasedJoinMappingSize || rightVarMapping->size() > mConfSamplingBasedJoinMappingSize) {
1234 if (leftItemDep->isBatchProcessed(false) && rightItemDep->isBatchProcessed(false)) {
1235 leftItemDep->loadNextBatch(false);
1236 rightItemDep->loadNextBatch(false);
1237 }
1238
1239 bool mappingProcessed = false;
1240 bool requiresScheduling = false;
1241 while (!joiningItem->isSamplingCompleted() && requiresMoreVariableMappingComputation(joiningItem, true) && (!leftItemDep->isBatchProcessed(false) || !rightItemDep->isBatchProcessed(false) || leftItemDep->loadNextBatch(false) || rightItemDep->loadNextBatch(false)) && !requiresScheduling) {
1242
1243 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker> keyMappingBatchDataVector(currentBatchSizeCount);
1244 while (requiresMoreVariableMappingComputation(joiningItem, true) && ((leftItemDep->getBatchCurrentBindingsCardinalityLinker(false) && leftSampleInsertionCount < mConfSamplingBasedJoinMappingSize) || (rightItemDep->getBatchCurrentBindingsCardinalityLinker(false) && rightSampleInsertionCount < mConfSamplingBasedJoinMappingSize))) {
1245
1246 if (requiresMoreVariableMappingComputation(joiningItem, true) && leftItemDep->getBatchCurrentBindingsCardinalityLinker(false) && leftSampleInsertionCount < mConfSamplingBasedJoinMappingSize) {
1247 mappingProcessed = true;
1248
1249 joiningHashSampleBatchingFunc(true, leftItemDep, leftSampleInsertionCount, keyMappingBatchDataVector);
1250 }
1251
1252 if (requiresMoreVariableMappingComputation(joiningItem, true) && rightItemDep->getBatchCurrentBindingsCardinalityLinker(false) && rightSampleInsertionCount < mConfSamplingBasedJoinMappingSize) {
1253 mappingProcessed = true;
1254
1255 joiningHashSampleBatchingFunc(false, rightItemDep, rightSampleInsertionCount, keyMappingBatchDataVector);
1256 }
1257 }
1258 mapReducedConcurrentJoinInsertionComputation(keyMappingBatchDataVector);
1259
1260 if ((leftSampleInsertionCount >= mConfSamplingBasedJoinMappingSize || leftItemDep->isProcessingFinished(false)) && (rightSampleInsertionCount >= mConfSamplingBasedJoinMappingSize || rightItemDep->isProcessingFinished(false))) {
1261 joiningItem->setSamplingCompleted(true);
1262 }
1263
1264 if (leftItemDep->isBatchProcessed(false)) {
1265 leftItemDep->loadNextBatch(false);
1266 }
1267
1268 if (rightItemDep->isBatchProcessed(false)) {
1269 rightItemDep->loadNextBatch(false);
1270 }
1271
1272 if (!joiningItem->isSamplingCompleted() && (rightSampleInsertionCount >= mConfSamplingBasedJoinMappingSize || rightItemDep->isProcessingFinished(false))) {
1273 if (leftItemDep->isBatchProcessed(false) && !leftItemDep->isProcessingFinished(false) && leftSampleInsertionCount < mConfSamplingBasedJoinMappingSize) {
1274 requiresScheduling = true;
1275 }
1276 }
1277 if (!joiningItem->isSamplingCompleted() && (leftSampleInsertionCount >= mConfSamplingBasedJoinMappingSize || leftItemDep->isProcessingFinished(false))) {
1278 if (rightItemDep->isBatchProcessed(false) && !rightItemDep->isProcessingFinished(false) && rightSampleInsertionCount < mConfSamplingBasedJoinMappingSize) {
1279 requiresScheduling = true;
1280 }
1281 }
1282 }
1283 if ((leftSampleInsertionCount >= mConfSamplingBasedJoinMappingSize || leftItemDep->isProcessingFinished(false)) && (rightSampleInsertionCount >= mConfSamplingBasedJoinMappingSize || rightItemDep->isProcessingFinished(false))) {
1284 joiningItem->setSamplingCompleted(true);
1285 }
1286
1287
1288 if (joiningItem->isSamplingCompleted()) {
1289 leftSampleKeyCount = atomicLeftSampleKeyCount;
1290 rightSampleKeyCount = atomicRightSampleKeyCount;
1291 }
1292 bool leftInsertion = isSamplingLeftInsertionDecision(joiningItem);
1293
1294
1295 if (joiningItem->isSamplingCompleted()) {
1296
1297 COptimizedComplexVariableCompositionItemDependence* insertingItemDep = leftItemDep;
1298 COptimizedComplexVariableCompositionItemDependence* checkingItemDep = rightItemDep;
1299
1300
1301 CXLinker<cint64>* insertingKeyBindingLinker = leftKeyBindingLinker;
1302 CXLinker<cint64>* checkingKeyBindingLinker = rightKeyBindingLinker;
1303
1304
1305 if (!leftInsertion) {
1306 insertingItemDep = rightItemDep;
1307 checkingItemDep = leftItemDep;
1308
1309 insertingKeyBindingLinker = rightKeyBindingLinker;
1310 checkingKeyBindingLinker = leftKeyBindingLinker;
1311 }
1312
1313
1314 while (requiresMoreVariableMappingComputation(joiningItem, false) && (!insertingItemDep->isBatchProcessed() || insertingItemDep->loadNextBatch())) {
1315
1316 while (requiresMoreVariableMappingComputation(joiningItem, false) && insertingItemDep->getBatchCurrentBindingsCardinalityLinker(false)) {
1317 mappingProcessed = true;
1318
1319 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker> keyMappingBindingsBatchDataVector(currentBatchSizeCount);
1320 joiningHashBindingsInsertingCheckingBatchingFunc(leftInsertion, insertingItemDep, true, keyMappingBindingsBatchDataVector);
1321 mapReducedConcurrentJoinInsertionComputation(keyMappingBindingsBatchDataVector);
1322 }
1323
1324
1325
1326 // inserting remaining cardinality updates
1327 while (requiresMoreVariableMappingComputation(joiningItem, false) && insertingItemDep->getBatchCurrentUpdatedCardinalityLinker(false)) {
1328 mappingProcessed = true;
1329
1330 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker> keyMappingCardinalitiesBatchDataVector(currentBatchSizeCount);
1331 joiningHashCardinalitiesInsertingCheckingBatchingFunc(leftInsertion, insertingItemDep, true, keyMappingCardinalitiesBatchDataVector);
1332 mapReducedConcurrentJoinInsertionComputation(keyMappingCardinalitiesBatchDataVector);
1333 }
1334 }
1335
1336
1337
1338 if (insertingItemDep->isProcessingFinished()) {
1339 while (requiresMoreVariableMappingComputation(joiningItem, false) && (!checkingItemDep->isBatchProcessed() || checkingItemDep->loadNextBatch())) {
1340
1341
1342 // checking remaining mappings
1343 while (requiresMoreVariableMappingComputation(joiningItem, false) && checkingItemDep->getBatchCurrentBindingsCardinalityLinker(false)) {
1344 mappingProcessed = true;
1345
1346 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker> keyMappingBindingsBatchDataVector(currentBatchSizeCount);
1347 joiningHashBindingsInsertingCheckingBatchingFunc(!leftInsertion, checkingItemDep, false, keyMappingBindingsBatchDataVector);
1348 checkingHandlingFunc(keyMappingBindingsBatchDataVector);
1349 }
1350
1351
1352 // checking remaining cardinality updates
1353 while (requiresMoreVariableMappingComputation(joiningItem, false) && checkingItemDep->getBatchCurrentUpdatedCardinalityLinker(false)) {
1354 mappingProcessed = true;
1355
1356 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker> keyMappingCardinalitiesBatchDataVector(currentBatchSizeCount);
1357 joiningHashCardinalitiesInsertingCheckingBatchingFunc(!leftInsertion, checkingItemDep, false, keyMappingCardinalitiesBatchDataVector);
1358 checkingHandlingFunc(keyMappingCardinalitiesBatchDataVector);
1359 }
1360 }
1361
1362 }
1363
1364 }
1365
1366 joinedVarMapping->incCurrentUpdateId();
1367
1368
1369 return mappingProcessed;
1370 }
1371
1372
1373
1374 }; // end namespace Composition
1375
1376 }; // end namespace Answerer
1377
1378 }; // end namespace Reasoner
1379
1380 }; // end namespace Konclude