Debian Codegraph @

Source/Reasoner/Answerer/Composition/CQtConcurrentVariableMappingsCompositionPropagationComputator.cpp

raw
1 /*
2 * Copyright (C) 2013-2015, 2019 by the Konclude Developer Team.
3 *
4 * This file is part of the reasoning system Konclude.
5 * For details and support, see <http://konclude.com/>.
6 *
7 * Konclude is free software: you can redistribute it and/or modify
8 * it under the terms of version 3 of the GNU Lesser General Public
9 * License (LGPLv3) as published by the Free Software Foundation.
10 *
11 * Konclude is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU (Lesser) General Public License for more details.
15 *
16 * You should have received a copy of the GNU (Lesser) General Public
17 * License along with Konclude. If not, see <http://www.gnu.org/licenses/>.
18 *
19 */
20
21 #include "CQtConcurrentVariableMappingsCompositionPropagationComputator.h"
22
23 namespace Konclude {
24
25 namespace Reasoner {
26
27 namespace Answerer {
28
29 namespace Composition {
30
31 CQtConcurrentVariableMappingsCompositionPropagationComputator::CQtConcurrentVariableMappingsCompositionPropagationComputator() {
32 mComputerName = "Concurrent Qt propagation";
33 }
34
35
36
37 CAbstractVariableMappingsCompositionComputator* CQtConcurrentVariableMappingsCompositionPropagationComputator::configureComputator(COptimizedComplexExpressionOntologyAnsweringItem* ontoAnsweringItem, CAbstractVariableMappingsCompositionItemRequirementProcessor* reqProcessor, CAnswererContext* answererContext) {
38 CAbstractVariableMappingsCompositionPropagationComputator::configureComputator(ontoAnsweringItem, reqProcessor, answererContext);
39
40 mConfConcurrentPropagationHashCount = CConfigDataReader::readConfigInteger(ontoAnsweringItem->getCalculationConfiguration(), "Konclude.Answering.ConcurrentPropagationComputationTaskCount", 211);
41 mConfPartitionizedMemoryManagement = CConfigDataReader::readConfigBoolean(ontoAnsweringItem->getCalculationConfiguration(), "Konclude.Answering.ConcurrentPropagationPartitionizedMemoryManagement", false);
42 mConfPerformanceLogging = CConfigDataReader::readConfigBoolean(ontoAnsweringItem->getCalculationConfiguration(), "Konclude.Answering.ConcurrentPropagationPerformanceLogging", false);
43 mConfConcurrentVariableMappingPartCount = mConfConcurrentPropagationHashCount;
44 mConfBatchSizeLimit = 200;
45 mConfBatchSizeCount = mConfConcurrentPropagationHashCount;
46
47 return this;
48 }
49
50
51
52 bool CQtConcurrentVariableMappingsCompositionPropagationComputator::computeVariableMappingsPropagationComposition(COptimizedComplexVariableRolePropagationAbstractItem* compVarItem, COptimizedComplexBuildingIndividualVariableCompositionsItem* buildingVarItem, CAnswererContext* answererContext, bool& processing) {
53
54 COptimizedComplexVariableRolePropagationAbstractItem* rolePropItem = (COptimizedComplexVariableRolePropagationAbstractItem*)compVarItem;
55 COptimizedComplexVariableCompositionItem* baseVarItem = rolePropItem->getBaseItem();
56 CRole* role = rolePropItem->getRole();
57 bool inversed = rolePropItem->getInversed();
58 COptimizedComplexVariableIndividualMappings* baseVariableMapping = baseVarItem->getVariableMapping();
59 COptimizedComplexVariableCompositionItemDependence* baseItemDep = rolePropItem->getBaseItemDependence();
60 //COptimizedComplexVariableIndividualMappings* variableMapping = rolePropItem->getVariableMapping();
61 COptimizedComplexVariableIndividualMappingsMultiHash* propagatedVarMapping = (COptimizedComplexVariableIndividualMappingsMultiHash*)rolePropItem->getVariableMapping();
62
63 CRoleRealization* roleReal = mOntoAnsweringItem->getOntology()->getRealization()->getRoleRealization();
64
65 QList<COptimizedComplexVariableRolePropagationProcessingRealizationIteratorDataBase*>* waitingProcessedRealizationIteratorDataList = rolePropItem->getWaitingProcessedRealizationIteratorDataList();
66 QList<COntologyProcessingRequirement*>* schedulingReqList = rolePropItem->geSchedulingRealizationRequirementIteratorDataList();
67
68 QHash<CRealizationIndividualInstanceItemReference, COptimizedComplexVariableRolePropagationProcessingRealizationIteratorData*>* instItemRolePropInstIterationDataHash = rolePropItem->getInstanceItemRolePropagationInstanceIterationDataHash();
69 QVector<COptimizedComplexVariableRolePropagationProcessingRealizationIteratorDataHashMemoryManaged*>* instItemRolePropInstIterationDataHashVector = rolePropItem->getInstanceItemRolePropagationInstanceIterationDataHashVector();
70 if (instItemRolePropInstIterationDataHashVector->size() < mConfConcurrentPropagationHashCount) {
71 instItemRolePropInstIterationDataHashVector->resize(mConfConcurrentPropagationHashCount);
72 }
73
74
75 // get variable index
76 cint64 varIdx = rolePropItem->getPropagationVariableIndex();
77 cint64 baseBindingSize = baseVariableMapping->getBindingSize();
78 cint64 newBindingSize = baseVariableMapping->getBindingSize();
79 if (compVarItem->getCompositionType() == COptimizedComplexVariableCompositionItem::ROLE_PROPAGATION) {
80 newBindingSize = baseVariableMapping->getBindingSize() + 1;
81 }
82 cint64 newVarIdx = baseBindingSize;
83 bool joining = false;
84 bool replacing = false;
85 if (compVarItem->getCompositionType() == COptimizedComplexVariableRolePropagationJoiningItem::ROLE_PROPAGATION_JOIN) {
86 COptimizedComplexVariableRolePropagationJoiningItem* joinRolePropItem = (COptimizedComplexVariableRolePropagationJoiningItem*)compVarItem;
87 joining = true;
88 newVarIdx = joinRolePropItem->getJoiningVariableIndex();
89 }
90 if (compVarItem->getCompositionType() == COptimizedComplexVariableRolePropagationReplacementItem::ROLE_PROPAGATION_REPLACEMENT) {
91 COptimizedComplexVariableRolePropagationReplacementItem* replRolePropItem = (COptimizedComplexVariableRolePropagationReplacementItem*)compVarItem;
92 replacing = true;
93 newVarIdx = replRolePropItem->getReplacementVariableIndex();
94 }
95
96 cint64* baseBindingMapping = new cint64[newBindingSize];
97 for (cint64 i = 0; i < baseBindingSize; ++i) {
98 baseBindingMapping[i] = i;
99 }
100
101 QList<COptimizedComplexVariableRolePropagationProcessingRealizationIteratorDataBase*>* schedulingRealizationIteratorDataList = rolePropItem->getSchedulingRealizationIteratorDataList();
102
103 CRoleInstantiatedItem* roleItem = roleReal->getRoleInstantiatedItem(role);
104
105 COptimizedComplexVariableIndividualBindings* propagedBindings = nullptr;
106 COptimizedComplexVariableIndividualBindingsCardinality* propagedCardinalites = nullptr;
107 COptimizedComplexVariableIndividualBindingsCardinalityLinker* propagedLinker = nullptr;
108 CSameRealization* sameRealization = mOntoAnsweringItem->getOntology()->getRealization()->getSameRealization();
109
110
111
112 CRoleRealization* roleRealization = mOntoAnsweringItem->getOntology()->getRealization()->getRoleRealization();
113 CRoleInstantiatedItem* roleInstancesItem = roleRealization->getRoleInstantiatedItem(role);
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151 cint64 currentBatchSizeCount = mConfBatchSizeCount;
152 cint64 currentBatchSizeLimit = mConfBatchSizeLimit;
153
154
155
156
157 function<void(COptimizedComplexVariableCompositionItemDependence* itemDep, CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationBindingsCardinalityLinkerBatchLinker>& keyMappingBatchDataVector)> propagationMappingsBatchingFunc =
158 [&](COptimizedComplexVariableCompositionItemDependence* itemDep, CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationBindingsCardinalityLinkerBatchLinker>& keyMappingBatchDataVector) -> void {
159
160 cint64 batchSize = 1;
161 COptimizedComplexVariableIndividualBindingsCardinalityLinker* firstBindingLinker = itemDep->getBatchCurrentBindingsCardinalityLinker(true);
162 COptimizedComplexVariableIndividualBindingsCardinalityLinker* lastBindingLinker = firstBindingLinker;
163 while (batchSize++ < currentBatchSizeLimit && itemDep->getBatchCurrentBindingsCardinalityLinker(false)) {
164 lastBindingLinker = itemDep->getBatchCurrentBindingsCardinalityLinker(true);
165 }
166
167 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionPropagationBindingsCardinalityLinkerBatchLinker>* vecData = keyMappingBatchDataVector.getVectorLinkerData(keyMappingBatchDataVector.getNextIndex(true), true);
168 CQtConcurrentVariableMappingsCompositionPropagationBindingsCardinalityLinkerBatchLinker* data = nullptr;
169 if (!mConfPartitionizedMemoryManagement) {
170 data = keyMappingBatchDataVector.createBatchLinker();
171 } else {
172 data = vecData->createBatchLinker();
173 }
174 data->setBatchBindingsCardinalityLinkers(firstBindingLinker, lastBindingLinker);
175 vecData->addBatchLinker(data);
176 };
177
178
179
180
181 function<void(COptimizedComplexVariableCompositionItemDependence* itemDep, CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationBindingsCardinalityLinkerBatchLinker>& keyMappingBatchDataVector)> propagationCardinalitesBatchingFunc =
182 [&](COptimizedComplexVariableCompositionItemDependence* itemDep, CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationBindingsCardinalityLinkerBatchLinker>& keyMappingBatchDataVector) -> void {
183
184 cint64 batchSize = 1;
185 COptimizedComplexVariableIndividualUpdateCardinalityLinker* firstCardinalityLinker = itemDep->getBatchCurrentUpdatedCardinalityLinker(true);
186 COptimizedComplexVariableIndividualUpdateCardinalityLinker* lastCardinalityLinker = firstCardinalityLinker;
187 while (batchSize++ < currentBatchSizeLimit && itemDep->getBatchCurrentUpdatedCardinalityLinker(false)) {
188 lastCardinalityLinker = itemDep->getBatchCurrentUpdatedCardinalityLinker(true);
189 }
190
191 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionPropagationBindingsCardinalityLinkerBatchLinker>* vecData = keyMappingBatchDataVector.getVectorLinkerData(keyMappingBatchDataVector.getNextIndex(true), true);
192 CQtConcurrentVariableMappingsCompositionPropagationBindingsCardinalityLinkerBatchLinker* data = nullptr;
193 if (!mConfPartitionizedMemoryManagement) {
194 data = keyMappingBatchDataVector.createBatchLinker();
195 } else {
196 data = vecData->createBatchLinker();
197 }
198 data->setBatchCardinalityUpdateLinkers(firstCardinalityLinker, lastCardinalityLinker);
199 vecData->addBatchLinker(data);
200 };
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227 // map & reduce of step 1
228 cint64 keyMappingSize = mConfConcurrentPropagationHashCount;
229
230 function<CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationMappedKeyBindingsCardinalityLinker>(const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionPropagationBindingsCardinalityLinkerBatchLinker>* const & batchData)> bindingBatchesHashingKeyMapFunc =
231 [&](const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionPropagationBindingsCardinalityLinkerBatchLinker>* const & batchData) ->CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationMappedKeyBindingsCardinalityLinker> {
232
233 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationMappedKeyBindingsCardinalityLinker> keyMappedVec(keyMappingSize);
234 if (batchData) {
235 CQtConcurrentVariableMappingsCompositionPropagationBindingsCardinalityLinkerBatchLinker* batchLinker = batchData->getNextBatchLinker();
236
237 while (batchLinker) {
238
239
240
241
242
243 COptimizedComplexVariableRolePropagationProcessingRealizationIteratorDataMemoryManaged* contIterData = batchLinker->getNextContinueIterationData(true);
244 while (contIterData) {
245
246 CRealizationIndividualInstanceItemReference instItemRef = contIterData->getPropagationInstanceItemReference();
247 cint64 propIndiId = instItemRef.getIndividualID();
248
249 cint64 keyHashValue = propIndiId;
250 cint64 mappedKeyIndex = uint((keyHashValue >> (8 * sizeof(uint) - 1)) ^ keyHashValue) % keyMappingSize;
251
252 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionPropagationMappedKeyBindingsCardinalityLinker>* keyMappedVecData = keyMappedVec.getVectorLinkerData(mappedKeyIndex, true);
253 CQtConcurrentVariableMappingsCompositionPropagationMappedKeyBindingsCardinalityLinker* mappedKeyDataLinker = nullptr;
254 if (!mConfPartitionizedMemoryManagement) {
255 mappedKeyDataLinker = keyMappedVec.createBatchLinker();
256 } else {
257 mappedKeyDataLinker = keyMappedVecData->createBatchLinker();
258 }
259 mappedKeyDataLinker->initMappedKeysBindingsCardinalityData(instItemRef, mappedKeyIndex, nullptr, nullptr, nullptr);
260 keyMappedVecData->addBatchLinker(mappedKeyDataLinker);
261
262 contIterData = batchLinker->getNextContinueIterationData(true);
263 }
264
265
266
267 COptimizedComplexVariableIndividualBindingsCardinalityLinker* bindingsCardinalityCurrentLinker = batchLinker->getNextBatchBindingsCardinalityLinker(true);
268 while (bindingsCardinalityCurrentLinker) {
269
270
271 COptimizedComplexVariableIndividualBindings* varBinding = bindingsCardinalityCurrentLinker->getBindings();
272 CRealizationIndividualInstanceItemReference instItemRef = varBinding->getBinding(varIdx).reference;
273 cint64 propIndiId = instItemRef.getIndividualID();
274
275 cint64 keyHashValue = propIndiId;
276 cint64 mappedKeyIndex = uint((keyHashValue >> (8 * sizeof(uint) - 1)) ^ keyHashValue) % keyMappingSize;
277
278 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionPropagationMappedKeyBindingsCardinalityLinker>* keyMappedVecData = keyMappedVec.getVectorLinkerData(mappedKeyIndex, true);
279 CQtConcurrentVariableMappingsCompositionPropagationMappedKeyBindingsCardinalityLinker* mappedKeyDataLinker = nullptr;
280 if (!mConfPartitionizedMemoryManagement) {
281 mappedKeyDataLinker = keyMappedVec.createBatchLinker();
282 } else {
283 mappedKeyDataLinker = keyMappedVecData->createBatchLinker();
284 }
285 mappedKeyDataLinker->initMappedKeysBindingsCardinalityData(instItemRef, mappedKeyIndex, bindingsCardinalityCurrentLinker->getBindings(), bindingsCardinalityCurrentLinker->getInitialCardinalities(), nullptr);
286 keyMappedVecData->addBatchLinker(mappedKeyDataLinker);
287
288 bindingsCardinalityCurrentLinker = batchLinker->getNextBatchBindingsCardinalityLinker(true);
289 }
290
291
292
293 COptimizedComplexVariableIndividualUpdateCardinalityLinker* cardinalityUpdateLinker = batchLinker->getNextCardinalityUpdateLinker(true);
294 while (cardinalityUpdateLinker) {
295
296 COptimizedComplexVariableIndividualBindings* varBinding = cardinalityUpdateLinker->getUpdatedBindingsCardinalityLinker()->getBindings();
297 CRealizationIndividualInstanceItemReference instItemRef = varBinding->getBinding(varIdx).reference;
298 cint64 propIndiId = instItemRef.getIndividualID();
299
300 cint64 keyHashValue = propIndiId;
301 cint64 mappedKeyIndex = uint((keyHashValue >> (8 * sizeof(uint) - 1)) ^ keyHashValue) % keyMappingSize;
302
303 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionPropagationMappedKeyBindingsCardinalityLinker>* keyMappedVecData = keyMappedVec.getVectorLinkerData(mappedKeyIndex, true);
304 CQtConcurrentVariableMappingsCompositionPropagationMappedKeyBindingsCardinalityLinker* mappedKeyDataLinker = nullptr;
305 if (!mConfPartitionizedMemoryManagement) {
306 mappedKeyDataLinker = keyMappedVec.createBatchLinker();
307 } else {
308 mappedKeyDataLinker = keyMappedVecData->createBatchLinker();
309 }
310 mappedKeyDataLinker->initMappedKeysBindingsCardinalityData(instItemRef, mappedKeyIndex, cardinalityUpdateLinker->getUpdatedBindingsCardinalityLinker()->getBindings(), cardinalityUpdateLinker->getNewCardinality(), cardinalityUpdateLinker->getPreviousCardinality());
311 keyMappedVecData->addBatchLinker(mappedKeyDataLinker);
312
313 cardinalityUpdateLinker = batchLinker->getNextCardinalityUpdateLinker(true);
314 }
315
316
317
318 batchLinker = batchLinker->getNext();
319 }
320 }
321
322 return keyMappedVec;
323 };
324
325
326 function<void(CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationMappedKeyBindingsCardinalityLinker>& vec, const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationMappedKeyBindingsCardinalityLinker>& reduceVec)> bindingBatchesHashingKeyReduceFunc =
327 [&](CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationMappedKeyBindingsCardinalityLinker>& vec, const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationMappedKeyBindingsCardinalityLinker>& reduceVec) -> void {
328
329 if (vec.size() < reduceVec.size()) {
330 vec.resize(reduceVec.size());
331 }
332 for (cint64 i = 0; i < vec.count(); ++i) {
333 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionPropagationMappedKeyBindingsCardinalityLinker>*& vecData = vec[i];
334 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionPropagationMappedKeyBindingsCardinalityLinker>* const & reduceVecData = reduceVec[i];
335 if (reduceVecData) {
336 if (!vecData) {
337 vecData = reduceVecData;
338 } else {
339 vecData->mergeBatchLinkers(reduceVecData);
340 }
341 }
342
343 }
344 vec.appendMemoryPool(reduceVec.getMemoryPools());
345 };
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386 // map & reduce of step 2
387 cint64 bindingsMappingSize = mConfConcurrentVariableMappingPartCount;
388
389
390
391 auto variableMappingCreationFunc = [&](COptimizedComplexVariableIndividualBindingsCardinalityLinker* varBindingCardLinker, COptimizedComplexVariableIndividualBindingsCardinality* varCardinalities, const CRealizationIndividualInstanceItemReference& propInstItem,
392 COptimizedComplexVariableIndividualBindings* propagedBindings, COptimizedComplexVariableIndividualBindingsCardinalityLinker* propagedLinker, COptimizedComplexVariableIndividualBindingsCardinality* propagedCardinalites, CMemoryAllocationManager* memMan) {
393
394 COptimizedComplexVariableIndividualBindings* varBinding = varBindingCardLinker->getBindings();
395 if (!propagedBindings) {
396 propagedBindings = createBindingsForVariableCompositionItems(newBindingSize, compVarItem, buildingVarItem, memMan);
397 }
398 if (!propagedCardinalites && varCardinalities) {
399 propagedCardinalites = createBindingsCardinalitesForVariableCompositionItems(compVarItem, buildingVarItem, memMan);
400 }
401 if (varCardinalities) {
402 *propagedCardinalites = *varCardinalities;
403 }
404
405
406 if (replacing) {
407 if (!propagedCardinalites) {
408 propagedCardinalites = createBindingsCardinalitesForVariableCompositionItems(compVarItem, buildingVarItem, memMan);
409 propagedCardinalites->initCardinalities(1, 1);
410 }
411 propagedCardinalites->setCardinalityUpdateId(-1);
412
413 TIndividualInstanceItemDataBinding& reducedBinding = varBinding->getBinding(newVarIdx);
414 CInstanceBindingIndividualCountingVisitor visitor;
415 sameRealization->visitSameIndividuals(reducedBinding.reference, &visitor);
416 if (visitor.individualCount > 1) {
417 propagedCardinalites->setSameIndividualsSeparatlyConsideredCardinality(visitor.individualCount * propagedCardinalites->getSameIndividualsSeparatlyConsideredCardinality());
418 }
419
420 }
421
422 COptimizedComplexVariableIndividualBindings nextNewBinding(newBindingSize);
423 for (cint64 i = 0; i < baseBindingSize; ++i) {
424 propagedBindings->setBinding(baseBindingMapping[i], varBinding->getBinding(i));
425 }
426 bool inserting = true;
427 if (joining) {
428 if (propInstItem == varBinding->getBinding(newVarIdx).reference) {
429 inserting = true;
430 } else {
431 inserting = false;
432 }
433 }
434 propagedBindings->setBinding(newVarIdx, propInstItem);
435
436 if (!propagedLinker) {
437 propagedLinker = createBindingsLinkerForVariableCompositionItems(propagedBindings, propagedCardinalites, compVarItem, buildingVarItem, memMan);
438 } else {
439 propagedLinker->setInitialCardinality(propagedCardinalites);
440 }
441
442 if (!inserting) {
443 //if (variableMapping->addInsertingBindingsCardinalityLinker(propagedLinker, true)) {
444 // propagedBindings = nullptr;
445 // propagedCardinalites = nullptr;
446 // propagedLinker = nullptr;
447 //} else {
448 // propagedCardinalites = propagedLinker->getCurrentCardinalities();
449 //}
450 propagedLinker = nullptr;
451 }
452
453 return propagedLinker;
454 };
455
456
457
458
459
460
461 auto propagatedVariableMappingCreationCopyingVecFunc = [&](COptimizedComplexVariableIndividualBindingsCardinalityLinker* varBindingCardLinker, COptimizedComplexVariableIndividualBindingsCardinality* varCardinalities, const CRealizationIndividualInstanceItemReference& propInstItem,
462 CQtConcurrentVariableMappingsCompositionPropagationRealizationSchedulingBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationMappedPropagatedBindingsCardinalityLinker>& bindingsMappedVec, CMemoryAllocationManager* memMan, COptimizedComplexVariableIndividualBindings*& tmpJoinedBindings) -> void {
463
464
465 CQtConcurrentVariableMappingsCompositionPropagationMappedPropagatedBindingsCardinalityLinker* mappedBindingsDataLinker = nullptr;
466 if (!mConfPartitionizedMemoryManagement) {
467 mappedBindingsDataLinker = bindingsMappedVec.createBatchLinker();
468
469 COptimizedComplexVariableIndividualBindingsCardinalityLinker* propagatedLinker = variableMappingCreationFunc(varBindingCardLinker, varCardinalities, propInstItem, nullptr, nullptr, nullptr, memMan);
470
471 if (propagatedLinker) {
472 COptimizedComplexVariableIndividualBindingsHasher bindingsHasher(propagatedLinker);
473 cint64 keyHashValue = bindingsHasher.getHashValue();
474 cint64 mappedBindingsIndex = uint((keyHashValue >> (8 * sizeof(uint) - 1)) ^ keyHashValue) % bindingsMappingSize;
475
476 mappedBindingsDataLinker->initMappedJoinedBindingsCardinalityData(mappedBindingsIndex, bindingsHasher, propagatedLinker);
477 bindingsMappedVec.addBatchLinker(mappedBindingsIndex, mappedBindingsDataLinker);
478 }
479
480 } else {
481
482 if (!tmpJoinedBindings) {
483 tmpJoinedBindings = createBindingsForVariableCompositionItems(newBindingSize, compVarItem, buildingVarItem, memMan);
484 }
485 COptimizedComplexVariableIndividualBindingsCardinalityLinker tmpPropagatedLinker;
486 COptimizedComplexVariableIndividualBindingsCardinality tmpJoinedCardinalites;
487
488 COptimizedComplexVariableIndividualBindingsCardinalityLinker* propagatedLinker = variableMappingCreationFunc(varBindingCardLinker, varCardinalities, propInstItem, tmpJoinedBindings, &tmpPropagatedLinker, &tmpJoinedCardinalites, memMan);
489 if (propagatedLinker) {
490
491 COptimizedComplexVariableIndividualBindingsHasher tmpBindingsHasher(&tmpPropagatedLinker);
492 cint64 keyHashValue = tmpBindingsHasher.getHashValue();
493 cint64 mappedBindingsIndex = uint((keyHashValue >> (8 * sizeof(uint) - 1)) ^ keyHashValue) % bindingsMappingSize;
494
495 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionPropagationMappedPropagatedBindingsCardinalityLinker>* bindingsMappedVecData = bindingsMappedVec.getVectorLinkerData(mappedBindingsIndex, true);
496
497 // copy data with corresponding memory manager
498 memMan = bindingsMappedVecData->getMemoryAllocationManager();
499 COptimizedComplexVariableIndividualBindings* joinedBindings = createBindingsForVariableCompositionItems(newBindingSize, compVarItem, buildingVarItem, memMan);
500 for (cint64 i = 0; i < newBindingSize; ++i) {
501 joinedBindings->setBinding(i, tmpJoinedBindings->getBinding(i));
502 }
503 COptimizedComplexVariableIndividualBindingsCardinality* joinedCardinalites = nullptr;
504 if (tmpPropagatedLinker.getCurrentCardinalities()) {
505 joinedCardinalites = createBindingsCardinalitesForVariableCompositionItems(compVarItem, buildingVarItem, memMan);
506 *joinedCardinalites = tmpJoinedCardinalites;
507 }
508 COptimizedComplexVariableIndividualBindingsCardinalityLinker* joinedLinker = createBindingsLinkerForVariableCompositionItems(joinedBindings, joinedCardinalites, compVarItem, buildingVarItem, memMan);
509 COptimizedComplexVariableIndividualBindingsHasher bindingsHasher(joinedLinker, keyHashValue);
510
511 mappedBindingsDataLinker = bindingsMappedVecData->createBatchLinker();
512 mappedBindingsDataLinker->initMappedJoinedBindingsCardinalityData(mappedBindingsIndex, bindingsHasher, joinedLinker);
513 bindingsMappedVecData->addBatchLinker(mappedBindingsDataLinker);
514 }
515 }
516 };
517
518
519
520
521 auto iterationDataHandleFunc = [&](COptimizedComplexVariableRolePropagationProcessingRealizationIteratorDataMemoryManaged* iteratorData, COptimizedComplexVariableRolePropagationProcessingRealizationIteratorDataHashMemoryManaged* instItemRolePropInstIterationDataHash,
522 CQtConcurrentVariableMappingsCompositionPropagationRealizationSchedulingBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationMappedPropagatedBindingsCardinalityLinker>& bindingsMappedVec, CMemoryAllocationManager* memMan, COptimizedComplexVariableIndividualBindings*& tmpJoinedBindings) {
523
524
525 //QString indiString = CIRIName::getRecentIRIName(iteratorData->debugPropInstItemRef.getIndividual()->getIndividualNameLinker());
526 //COptimizedKPSetRoleInstancesRedirectionItem* initialRedirectionItem = (COptimizedKPSetRoleInstancesRedirectionItem*)roleInstancesItem;
527 //COptimizedKPSetRoleInstancesItem* roleItem = (COptimizedKPSetRoleInstancesItem*)initialRedirectionItem->getRedirectedItem();
528 //QString roleString = CIRIName::getRecentIRIName(roleItem->getRole()->getPropertyNameLinker());
529 //QString inverseRoleString = CIRIName::getRecentIRIName(roleItem->getInverseRole()->getPropertyNameLinker());
530 //if (indiString == "http://ontology.dumontierlab.com/eswc-example-graph-3#datapoint1" && (roleString == "http://ontology.dumontierlab.com/isPartOf" || inverseRoleString == "http://ontology.dumontierlab.com/isPartOf")) {
531 // bool debug = true;
532 //}
533
534 CRealizationIndividualInstanceItemReference instItemRef = iteratorData->getPropagationInstanceItemReference();
535 if (!iteratorData->isRealizationIterationInitialized()) {
536 iteratorData->setRealizationIterationInitialized(true);
537 iteratorData->setRealizationIterator(roleRealization->getRoleFillerInstancesIterator(instItemRef, roleInstancesItem, inversed, mRoleFillerRealizationSorting));
538 }
539
540 if (!iteratorData->isRealizationScheduled()) {
541 CRealizationIndividualInstanceItemReferenceIterator* fillerIterator = iteratorData->getRealizationIterator();
542
543 while (fillerIterator && !fillerIterator->atEnd() && !fillerIterator->requiresInitialization()) {
544 CRealizationIndividualInstanceItemReference fillerInstanceItemRef = fillerIterator->currentIndividualInstanceItemReference();
545
546 iteratorData->addPropagationInstanceItemLinker(fillerInstanceItemRef);
547
548 instItemRolePropInstIterationDataHash->incFillerInstanceItemCount(iteratorData->getVariableMappingIteratorCount());
549
550 fillerIterator->moveNext();
551 }
552
553 if (fillerIterator) {
554 double remainingFillerEstimationCount = fillerIterator->remainingInstancesEstimation().getEstimatedCount();
555 double prevFillerEstimation = iteratorData->getExpectedFillerCount();
556 double newFillerEstimation = iteratorData->getPropagationInstanceItemCount() + remainingFillerEstimationCount;
557 iteratorData->setExpectedFillerCount(newFillerEstimation);
558 double diffFillerEstimation = newFillerEstimation - prevFillerEstimation;
559 instItemRolePropInstIterationDataHash->incFillerInstanceItemCount(diffFillerEstimation);
560 }
561
562 if (fillerIterator && fillerIterator->requiresInitialization()) {
563 bindingsMappedVec.addRealizationSchedulingIterationData(iteratorData);
564 //COntologyProcessingIteratorRealizationRequirement* roleRealReq = new COntologyProcessingIteratorRealizationRequirement(fillerIterator);
565 //schedulingRealizationIteratorDataList->append(iteratorData);
566 //schedulingReqList->append(roleRealReq);
567 iteratorData->setRealizationScheduled(true);
568 } else if (!fillerIterator || fillerIterator->atEnd()) {
569 if (fillerIterator) {
570 instItemRolePropInstIterationDataHash->incPropagationHandledInstanceItemCount();
571 delete fillerIterator;
572 iteratorData->setRealizationIterator(nullptr);
573 }
574 CXLinker<COptimizedComplexVariableIndividualBindingsCardinalityLinker*>* variableMappingIteratorLinker = iteratorData->getVariableMappingPropagationIteratorLinker();
575 for (CXLinker<COptimizedComplexVariableIndividualBindingsCardinalityLinker*>* variableMappingIteratorLinkerIt = variableMappingIteratorLinker; variableMappingIteratorLinkerIt; variableMappingIteratorLinkerIt = variableMappingIteratorLinkerIt->getNext()) {
576 COptimizedComplexVariableIndividualBindingsCardinalityLinker* varBindingCardLinker = variableMappingIteratorLinkerIt->getData();
577 COptimizedComplexVariableIndividualBindingsCardinality* cardinalites = varBindingCardLinker->getInitialCardinalities();
578 instItemRolePropInstIterationDataHash->incPropagatedInstanceItemCount();
579
580 CXLinker<CRealizationIndividualInstanceItemReference>* rolePropInstItemLinker = iteratorData->getPropagationInstanceItemLinker();
581 for (CXLinker<CRealizationIndividualInstanceItemReference>* rolePropInstItemLinkerIt = rolePropInstItemLinker; rolePropInstItemLinkerIt; rolePropInstItemLinkerIt = rolePropInstItemLinkerIt->getNext()) {
582 const CRealizationIndividualInstanceItemReference& propInstItem = rolePropInstItemLinkerIt->getData();
583 instItemRolePropInstIterationDataHash->incRealizationFinishedFillerInstanceItemCount(iteratorData->getPropagationInstanceItemCount());
584 propagatedVariableMappingCreationCopyingVecFunc(varBindingCardLinker, cardinalites, propInstItem, bindingsMappedVec, memMan, tmpJoinedBindings);
585 }
586 }
587 iteratorData->clearVariableMappingPropagationIterator();
588
589
590 CXLinker<COptimizedComplexVariableIndividualUpdateCardinalityLinker*>* cardinalityUpdateIteratorLinker = iteratorData->getCardinalityUpdatePropagationIteratorLinker();
591 for (CXLinker<COptimizedComplexVariableIndividualUpdateCardinalityLinker*>* cardinalityUpdateIteratorLinkerIt = cardinalityUpdateIteratorLinker; cardinalityUpdateIteratorLinkerIt; cardinalityUpdateIteratorLinkerIt = cardinalityUpdateIteratorLinkerIt->getNext()) {
592 COptimizedComplexVariableIndividualUpdateCardinalityLinker* updatedCardinalityLinker = cardinalityUpdateIteratorLinkerIt->getData();
593
594 COptimizedComplexVariableIndividualBindingsCardinality* prevCardinalites = updatedCardinalityLinker->getPreviousCardinality();
595 COptimizedComplexVariableIndividualBindingsCardinalityLinker* bindingLinker = updatedCardinalityLinker->getUpdatedBindingsCardinalityLinker();
596 COptimizedComplexVariableIndividualBindingsCardinality* newCardinalites = updatedCardinalityLinker->getNewCardinality();
597 COptimizedComplexVariableIndividualBindings* bindings = bindingLinker->getBindings();
598
599 COptimizedComplexVariableIndividualBindingsCardinality tmpDiffCardinalites = *newCardinalites;
600 tmpDiffCardinalites.substractCardinalities(prevCardinalites);
601
602 CXLinker<CRealizationIndividualInstanceItemReference>* rolePropInstItemLinker = iteratorData->getPropagationInstanceItemLinker();
603 for (CXLinker<CRealizationIndividualInstanceItemReference>* rolePropInstItemLinkerIt = rolePropInstItemLinker; rolePropInstItemLinkerIt; rolePropInstItemLinkerIt = rolePropInstItemLinkerIt->getNext()) {
604 const CRealizationIndividualInstanceItemReference& propInstItem = rolePropInstItemLinkerIt->getData();
605 propagatedVariableMappingCreationCopyingVecFunc(bindingLinker, &tmpDiffCardinalites, propInstItem, bindingsMappedVec, memMan, tmpJoinedBindings);
606 }
607 }
608 iteratorData->clearCardinalityUpdatePropagationIterator();
609
610 }
611 }
612 };
613
614
615
616
617 function<CQtConcurrentVariableMappingsCompositionPropagationRealizationSchedulingBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationMappedPropagatedBindingsCardinalityLinker>(const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionPropagationMappedKeyBindingsCardinalityLinker>* const & keyMappedBindingsVecData)> bindingBatchesHashingPropagationMapFunc =
618 [&](const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionPropagationMappedKeyBindingsCardinalityLinker>* const & keyMappedBindingsVecData) -> CQtConcurrentVariableMappingsCompositionPropagationRealizationSchedulingBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationMappedPropagatedBindingsCardinalityLinker> {
619
620 CQtConcurrentVariableMappingsCompositionPropagationRealizationSchedulingBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationMappedPropagatedBindingsCardinalityLinker> bindingsMappedVec(bindingsMappingSize);
621
622 COptimizedComplexVariableRolePropagationProcessingRealizationIteratorDataHashMemoryManaged* lastPropHash = nullptr;
623 QTime* time = nullptr;
624 if (mConfPerformanceLogging) {
625 time = new QTime();
626 time->start();
627 }
628 if (keyMappedBindingsVecData) {
629 CQtConcurrentVariableMappingsCompositionPropagationMappedKeyBindingsCardinalityLinker* batchLinker = keyMappedBindingsVecData->getNextBatchLinker();
630 COptimizedComplexVariableIndividualBindings* tmpJoinedBindings = nullptr;
631
632 while (batchLinker) {
633
634 cint64 mappedKeyIndex = batchLinker->getMappedKeyIndex();
635 COptimizedComplexVariableRolePropagationProcessingRealizationIteratorDataHashMemoryManaged*& instItemRolePropInstIterationDataHash = (*instItemRolePropInstIterationDataHashVector)[mappedKeyIndex];
636 if (!instItemRolePropInstIterationDataHash) {
637 instItemRolePropInstIterationDataHash = new COptimizedComplexVariableRolePropagationProcessingRealizationIteratorDataHashMemoryManaged();
638 }
639 lastPropHash = instItemRolePropInstIterationDataHash;
640
641
642 CRealizationIndividualInstanceItemReference instItemRef = batchLinker->getInstanceItemReference();
643
644 COptimizedComplexVariableRolePropagationProcessingRealizationIteratorDataMemoryManaged* iterationData = instItemRolePropInstIterationDataHash->getRolePropagationProcessingRealizationIteratorData(instItemRef);
645
646 COptimizedComplexVariableIndividualBindings* varMapping = batchLinker->getVariableMapping();
647 if (varMapping) {
648 COptimizedComplexVariableIndividualBindingsCardinality* newCardinalites = batchLinker->getNewCardinalites();
649 COptimizedComplexVariableIndividualBindingsCardinality* prevCardinalites = batchLinker->getPreviousCardinalites();
650
651 COptimizedComplexVariableIndividualBindingsCardinalityLinker* linker = CObjectAllocator< COptimizedComplexVariableIndividualBindingsCardinalityLinker >::allocateAndConstruct(instItemRolePropInstIterationDataHash->getMemoryAllocationManager());
652 linker->initBindingsCardinalityLinker(varMapping, newCardinalites);
653 iterationData->addVariableMappingPropagationIterator(linker);
654 instItemRolePropInstIterationDataHash->mPropagatedCount++;
655 }
656
657 iterationDataHandleFunc(iterationData, instItemRolePropInstIterationDataHash, bindingsMappedVec, bindingsMappedVec.getMemoryAllocationManager(), tmpJoinedBindings);
658
659 batchLinker = batchLinker->getNext();
660 }
661 }
662
663 if (lastPropHash && time) {
664 lastPropHash->mProcessingTime += time->elapsed();
665 delete time;
666 }
667
668
669 return bindingsMappedVec;
670 };
671
672
673
674
675 function<void(CQtConcurrentVariableMappingsCompositionPropagationRealizationSchedulingBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationMappedPropagatedBindingsCardinalityLinker>& vec, const CQtConcurrentVariableMappingsCompositionPropagationRealizationSchedulingBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationMappedPropagatedBindingsCardinalityLinker>& reduceVec)> bindingBatchesHashingPropagationReduceFunc =
676 [&](CQtConcurrentVariableMappingsCompositionPropagationRealizationSchedulingBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationMappedPropagatedBindingsCardinalityLinker>& vec, const CQtConcurrentVariableMappingsCompositionPropagationRealizationSchedulingBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationMappedPropagatedBindingsCardinalityLinker>& reduceVec) -> void {
677
678 if (vec.size() < reduceVec.size()) {
679 vec.resize(reduceVec.size());
680 }
681 for (cint64 i = 0; i < vec.count(); ++i) {
682 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionPropagationMappedPropagatedBindingsCardinalityLinker>*& vecData = vec[i];
683 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionPropagationMappedPropagatedBindingsCardinalityLinker>* const & reduceVecData = reduceVec[i];
684 if (reduceVecData) {
685 if (!vecData) {
686 vecData = reduceVecData;
687 } else {
688 vecData->mergeBatchLinkers(reduceVecData);
689 }
690 }
691
692 }
693 vec.mergeRealizationSchedulingIterationDataLinkers(reduceVec);
694 vec.appendMemoryPool(reduceVec.getMemoryPools());
695 };
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735 // map (& reduce) of step 3
736
737 function<COptimizedComplexVariableIndividualMappingsMultiHashPart*(const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionPropagationMappedPropagatedBindingsCardinalityLinker>* const & mappedBindingsVecData)> bindingBatchesInsertingMapFunc =
738 [&](const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionPropagationMappedPropagatedBindingsCardinalityLinker>* const & mappedBindingsVecData) -> COptimizedComplexVariableIndividualMappingsMultiHashPart* {
739
740
741
742 QTime* time = nullptr;
743 if (mConfPerformanceLogging) {
744 time = new QTime();
745 time->start();
746 }
747
748 COptimizedComplexVariableIndividualMappingsMultiHashPart* propagatedVarMappingMultiHashPart = nullptr;
749
750 if (mappedBindingsVecData) {
751 CQtConcurrentVariableMappingsCompositionPropagationMappedPropagatedBindingsCardinalityLinker* batchLinker = mappedBindingsVecData->getNextBatchLinker();
752
753 while (batchLinker) {
754
755 cint64 mappedBindingsIndex = batchLinker->getBindingsIndex();
756 if (!propagatedVarMappingMultiHashPart) {
757 propagatedVarMappingMultiHashPart = propagatedVarMapping->getMultiHashPart(mappedBindingsIndex);
758 }
759
760
761 COptimizedComplexVariableIndividualBindingsCardinalityLinker* joinedLinker = batchLinker->getBindingsCardinalityLinker();
762 cint64 hashValue = batchLinker->getBindingHasher().getHashValue();
763
764 propagatedVarMappingMultiHashPart->addInsertingBindingsCardinalityLinkerCopy(joinedLinker, hashValue, true);
765
766 batchLinker = batchLinker->getNext();
767 }
768 }
769
770 if (propagatedVarMappingMultiHashPart && time) {
771 propagatedVarMappingMultiHashPart->mStatsProcessingTime += time->elapsed();
772 delete time;
773 }
774
775
776 return propagatedVarMappingMultiHashPart;
777 };
778
779
780
781
782 function<void(const QVector<COptimizedComplexVariableIndividualMappingsMultiHashPart*>& mappedVarMappingParts)> bindingBatchesInsertingReduceFunc =
783 [&](const QVector<COptimizedComplexVariableIndividualMappingsMultiHashPart*>& mappedVarMappingParts) -> void {
784
785 for (QVector<COptimizedComplexVariableIndividualMappingsMultiHashPart*>::const_iterator it = mappedVarMappingParts.constBegin(), itEnd = mappedVarMappingParts.constEnd(); it != itEnd; ++it) {
786 COptimizedComplexVariableIndividualMappingsMultiHashPart* reduceMappPart = (*it);
787 if (reduceMappPart) {
788 propagatedVarMapping->updateLinkers(reduceMappPart);
789 propagatedVarMapping->updateBindingCount(reduceMappPart);
790 }
791 }
792 };
793
794
795
796
797 function<void(CMemoryPool* memPools)> clearMemoryPoolsFunc = [&](CMemoryPool* memPools) -> void {
798 while (memPools) {
799 CMemoryPool* tmpMemPool = memPools;
800 memPools = memPools->getNextMemoryPool();
801 delete[] tmpMemPool->getMemoryBlockData();
802 delete tmpMemPool;
803 }
804 };
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834 function<void(CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationBindingsCardinalityLinkerBatchLinker>& keyMappingBatchDataVector)> mapReducedConcurrentPropagationComputation = [&](CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationBindingsCardinalityLinkerBatchLinker>& keyMappingBatchDataVector) -> void {
835 if (!keyMappingBatchDataVector.isEmpty()) {
836
837 QTime* stepTimer = nullptr;
838 if (mConfPerformanceLogging) {
839 stepTimer = new QTime();
840 stepTimer->start();
841 }
842 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationMappedKeyBindingsCardinalityLinker> mappedPropagationHashKeyVec = QtConcurrent::blockingMappedReduced<CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationMappedKeyBindingsCardinalityLinker>>(keyMappingBatchDataVector, bindingBatchesHashingKeyMapFunc, bindingBatchesHashingKeyReduceFunc);
843 if (stepTimer) {
844 cint64 partitionsUsed = 0;
845 for (cint64 i = 0; i < keyMappingBatchDataVector.size(); ++i) {
846 if (keyMappingBatchDataVector[i]) {
847 ++partitionsUsed;
848 }
849 }
850 LOG(INFO, getLogDomain(), logTr("Propagation step 1 of concurrent propagation computation for item %3 with %4/%1 tasks took %2 ms.").arg(keyMappingBatchDataVector.size()).arg(stepTimer->elapsed()).arg(rolePropItem->getComputationStepId()).arg(partitionsUsed), this);
851 delete stepTimer;
852 }
853
854
855
856 if (mConfPerformanceLogging) {
857 stepTimer = new QTime();
858 stepTimer->start();
859 }
860 CQtConcurrentVariableMappingsCompositionPropagationRealizationSchedulingBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationMappedPropagatedBindingsCardinalityLinker> mappedPropagatedHashKeyVec = QtConcurrent::blockingMappedReduced<CQtConcurrentVariableMappingsCompositionPropagationRealizationSchedulingBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationMappedPropagatedBindingsCardinalityLinker>>(mappedPropagationHashKeyVec, bindingBatchesHashingPropagationMapFunc, bindingBatchesHashingPropagationReduceFunc);
861 if (stepTimer) {
862 cint64 partitionsUsed = 0;
863 for (cint64 i = 0; i < mappedPropagationHashKeyVec.size(); ++i) {
864 if (mappedPropagationHashKeyVec[i]) {
865 ++partitionsUsed;
866 }
867 }
868 LOG(INFO, getLogDomain(), logTr("Propagation step 2 of concurrent propagation computation for item %3 with %4/%1 tasks took %2 ms.").arg(mappedPropagationHashKeyVec.size()).arg(stepTimer->elapsed()).arg(rolePropItem->getComputationStepId()).arg(partitionsUsed), this);
869 delete stepTimer;
870
871 cint64 statMaxJoiningCount = 0;
872 cint64 statMaxProcessingTime = 0;
873 cint64 statTotalJoiningCount = 0;
874 cint64 statTotalProcessingTime = 0;
875 cint64 statUsedCount = 0;
876 for (cint64 i = 0; i < mappedPropagationHashKeyVec.size(); ++i) {
877 COptimizedComplexVariableRolePropagationProcessingRealizationIteratorDataHashMemoryManaged* instItemRolePropInstIterationDataHash = (*instItemRolePropInstIterationDataHashVector)[i];
878 if (instItemRolePropInstIterationDataHash && instItemRolePropInstIterationDataHash->mPropagatedCount > 0) {
879 ++statUsedCount;
880 statMaxJoiningCount = qMax(statMaxJoiningCount, instItemRolePropInstIterationDataHash->mPropagatedCount);
881 statMaxProcessingTime = qMax(statMaxProcessingTime, instItemRolePropInstIterationDataHash->mProcessingTime);
882 statTotalJoiningCount += instItemRolePropInstIterationDataHash->mPropagatedCount;
883 statTotalProcessingTime += instItemRolePropInstIterationDataHash->mProcessingTime;
884 }
885 }
886 cint64 averageJoiningCount = 0;
887 if (statUsedCount > 0) {
888 averageJoiningCount = statTotalJoiningCount / statUsedCount;
889 }
890 cint64 averageProcessingTime = 0;
891 if (statUsedCount > 0) {
892 averageProcessingTime = statTotalProcessingTime / statUsedCount;
893 }
894 LOG(INFO, getLogDomain(), logTr("Propagation step 2 of concurrent propagation computation for item %1 had in average %2 (max %3) propagated mappings and took in average %4 (max %5) ms.").arg(rolePropItem->getComputationStepId()).arg(averageJoiningCount).arg(statMaxJoiningCount).arg(averageProcessingTime).arg(statMaxProcessingTime), this);
895 }
896 CXLinker<COptimizedComplexVariableRolePropagationProcessingRealizationIteratorDataMemoryManaged*>* realSchedItDataLinker = mappedPropagatedHashKeyVec.getFirstRealizationSchedulingIterationDataLinker();
897 for (CXLinker<COptimizedComplexVariableRolePropagationProcessingRealizationIteratorDataMemoryManaged*>* realSchedItDataLinkerIt = realSchedItDataLinker; realSchedItDataLinkerIt; realSchedItDataLinkerIt = realSchedItDataLinkerIt->getNext()) {
898 COptimizedComplexVariableRolePropagationProcessingRealizationIteratorDataMemoryManaged* iteratorData = realSchedItDataLinkerIt->getData();
899 COntologyProcessingIteratorRealizationRequirement* roleRealReq = new COntologyProcessingIteratorRealizationRequirement(iteratorData->getRealizationIterator());
900 schedulingRealizationIteratorDataList->append(iteratorData);
901 schedulingReqList->append(roleRealReq);
902 }
903
904
905
906 if (mConfPerformanceLogging) {
907 stepTimer = new QTime();
908 stepTimer->start();
909 }
910 QVector<COptimizedComplexVariableIndividualMappingsMultiHashPart*> mappedVarMappingParts = QtConcurrent::blockingMapped<QVector<COptimizedComplexVariableIndividualMappingsMultiHashPart*>>(mappedPropagatedHashKeyVec, bindingBatchesInsertingMapFunc);
911 if (stepTimer) {
912 cint64 partitionsUsed = 0;
913 for (cint64 i = 0; i < mappedPropagatedHashKeyVec.size(); ++i) {
914 if (mappedPropagatedHashKeyVec[i]) {
915 ++partitionsUsed;
916 }
917 }
918 LOG(INFO, getLogDomain(), logTr("Propagation step 3 of concurrent propagation computation for item %3 with %4/%1 tasks took %2 ms.").arg(mappedPropagatedHashKeyVec.size()).arg(stepTimer->elapsed()).arg(rolePropItem->getComputationStepId()).arg(partitionsUsed), this);
919 delete stepTimer;
920
921 cint64 statMaxInsertionCount = 0;
922 cint64 statMaxProcessingTime = 0;
923 cint64 statTotalInsertionCount = 0;
924 cint64 statTotalProcessingTime = 0;
925 cint64 statUsedCount = 0;
926 for (cint64 i = 0; i < mappedPropagatedHashKeyVec.size(); ++i) {
927 COptimizedComplexVariableIndividualMappingsMultiHashPart* joinedVarMappingMultiHashPart = propagatedVarMapping->getMultiHashPart(i);
928 if (joinedVarMappingMultiHashPart && joinedVarMappingMultiHashPart->mStatInsertionCount > 0) {
929 ++statUsedCount;
930 statMaxInsertionCount = qMax(statMaxInsertionCount, joinedVarMappingMultiHashPart->mStatInsertionCount);
931 statMaxProcessingTime = qMax(statMaxProcessingTime, joinedVarMappingMultiHashPart->mStatsProcessingTime);
932 statTotalInsertionCount += joinedVarMappingMultiHashPart->mStatInsertionCount;
933 statTotalProcessingTime += joinedVarMappingMultiHashPart->mStatsProcessingTime;
934 }
935 }
936 cint64 averageInsertionCount = 0;
937 if (statUsedCount > 0) {
938 averageInsertionCount = statTotalInsertionCount / statUsedCount;
939 }
940 cint64 averageProcessingTime = 0;
941 if (statUsedCount > 0) {
942 averageProcessingTime = statTotalProcessingTime / statUsedCount;
943 }
944 LOG(INFO, getLogDomain(), logTr("Propagation step 3 of concurrent propagation computation for item %1 had in average %2 (max %3) bindings insertions and took in average %4 (max %5) ms.").arg(rolePropItem->getComputationStepId()).arg(averageInsertionCount).arg(statMaxInsertionCount).arg(averageProcessingTime).arg(statMaxProcessingTime), this);
945 }
946
947
948
949 if (mConfPerformanceLogging) {
950 stepTimer = new QTime();
951 stepTimer->start();
952 }
953 bindingBatchesInsertingReduceFunc(mappedVarMappingParts);
954 CMemoryPool* mappedKeyBatchHashVecMemPools = keyMappingBatchDataVector.takeMemoryPools();
955 CMemoryPool* mappedJoiningHashVecMemPools = mappedPropagationHashKeyVec.takeMemoryPools();
956 CMemoryPool* mappedJoinedHashVecMemPools = mappedPropagatedHashKeyVec.takeMemoryPools();
957 CConcurrentTaskScheduler::run([=] {
958 clearMemoryPoolsFunc(mappedKeyBatchHashVecMemPools);
959 });
960 CConcurrentTaskScheduler::run([=] {
961 clearMemoryPoolsFunc(mappedJoiningHashVecMemPools);
962 });
963 CConcurrentTaskScheduler::run([=] {
964 clearMemoryPoolsFunc(mappedJoinedHashVecMemPools);
965 });
966 keyMappingBatchDataVector.clear();
967 keyMappingBatchDataVector.resize(keyMappingBatchDataVector.getIndexSize());
968 if (stepTimer) {
969 cint64 partitionsUsed = 0;
970 for (cint64 i = 0; i < mappedVarMappingParts.size(); ++i) {
971 if (mappedVarMappingParts[i]) {
972 ++partitionsUsed;
973 }
974 }
975 LOG(INFO, getLogDomain(), logTr("Propagation step 4 of concurrent propagation computation for item %3 with %4/%1 tasks took %2 ms.").arg(mappedVarMappingParts.size()).arg(stepTimer->elapsed()).arg(rolePropItem->getComputationStepId()).arg(partitionsUsed), this);
976 delete stepTimer;
977 }
978
979 }
980 };
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020 bool successorCollectionContinued = false;
1021
1022 if (!waitingProcessedRealizationIteratorDataList->isEmpty() && rolePropItem->requiresMoreVariableMappingsComputation()) {
1023 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationBindingsCardinalityLinkerBatchLinker> keyMappingBatchDataVector(currentBatchSizeCount);
1024 CXLinker<COptimizedComplexVariableRolePropagationProcessingRealizationIteratorDataMemoryManaged*>* contLinker = nullptr;
1025 cint64 currentContLinkerCount = 0;
1026 while (!waitingProcessedRealizationIteratorDataList->isEmpty() && rolePropItem->requiresMoreVariableMappingsComputation()) {
1027 COptimizedComplexVariableRolePropagationProcessingRealizationIteratorDataMemoryManaged* iteratorData = (COptimizedComplexVariableRolePropagationProcessingRealizationIteratorDataMemoryManaged*)waitingProcessedRealizationIteratorDataList->takeFirst();
1028 iteratorData->getRealizationIterator()->begin();
1029 iteratorData->setRealizationScheduled(false);
1030 rolePropItem->decScheduledRealizationCount();
1031
1032 CXLinker<COptimizedComplexVariableRolePropagationProcessingRealizationIteratorDataMemoryManaged*>* linker = CObjectAllocator< CXLinker<COptimizedComplexVariableRolePropagationProcessingRealizationIteratorDataMemoryManaged*> >::allocateAndConstruct(keyMappingBatchDataVector.getMemoryAllocationManager());
1033 linker->initLinker(iteratorData);
1034
1035 currentContLinkerCount++;
1036 contLinker = linker->append(contLinker);
1037
1038 if (currentContLinkerCount >= currentBatchSizeLimit) {
1039 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionPropagationBindingsCardinalityLinkerBatchLinker>* vecData = keyMappingBatchDataVector.getVectorLinkerData(keyMappingBatchDataVector.getNextIndex(true), true);
1040 CQtConcurrentVariableMappingsCompositionPropagationBindingsCardinalityLinkerBatchLinker* data = nullptr;
1041 if (!mConfPartitionizedMemoryManagement) {
1042 data = keyMappingBatchDataVector.createBatchLinker();
1043 } else {
1044 data = vecData->createBatchLinker();
1045 }
1046 data->addContinueIterationDataLinker(contLinker);
1047 vecData->addBatchLinker(data);
1048 contLinker = nullptr;
1049 currentContLinkerCount = 0;
1050 }
1051
1052 successorCollectionContinued = true;
1053 }
1054
1055 if (contLinker) {
1056 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData<CQtConcurrentVariableMappingsCompositionPropagationBindingsCardinalityLinkerBatchLinker>* vecData = keyMappingBatchDataVector.getVectorLinkerData(keyMappingBatchDataVector.getNextIndex(true), true);
1057 CQtConcurrentVariableMappingsCompositionPropagationBindingsCardinalityLinkerBatchLinker* data = nullptr;
1058 if (!mConfPartitionizedMemoryManagement) {
1059 data = keyMappingBatchDataVector.createBatchLinker();
1060 } else {
1061 data = vecData->createBatchLinker();
1062 }
1063 data->addContinueIterationDataLinker(contLinker);
1064 vecData->addBatchLinker(data);
1065 }
1066
1067 mapReducedConcurrentPropagationComputation(keyMappingBatchDataVector);
1068 }
1069
1070
1071
1072 bool mappingProcessed = false;
1073 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationBindingsCardinalityLinkerBatchLinker> keyMappingBatchDataVector(currentBatchSizeCount);
1074 while (rolePropItem->requiresMoreVariableMappingsComputation() && (!baseItemDep->isBatchProcessed() || baseItemDep->loadNextBatch())) {
1075
1076
1077
1078 CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector<CQtConcurrentVariableMappingsCompositionPropagationBindingsCardinalityLinkerBatchLinker> keyMappingBatchDataVector(currentBatchSizeCount);
1079 while (rolePropItem->requiresMoreVariableMappingsComputation() && baseItemDep->getBatchCurrentBindingsCardinalityLinker(false)) {
1080 mappingProcessed = true;
1081 propagationMappingsBatchingFunc(baseItemDep, keyMappingBatchDataVector);
1082 }
1083 mapReducedConcurrentPropagationComputation(keyMappingBatchDataVector);
1084
1085
1086 while (rolePropItem->requiresMoreVariableMappingsComputation() && baseItemDep->getBatchCurrentUpdatedCardinalityLinker(false)) {
1087 mappingProcessed = true;
1088 propagationCardinalitesBatchingFunc(baseItemDep, keyMappingBatchDataVector);
1089 }
1090 mapReducedConcurrentPropagationComputation(keyMappingBatchDataVector);
1091
1092
1093 }
1094
1095
1096
1097 rolePropItem->setFillerInstanceItemCount(0);
1098 rolePropItem->setRealizationFinishedFillerInstanceItemCount(0);
1099 rolePropItem->setPropagationHandledInstanceItemCount(0);
1100 rolePropItem->setPropagatedInstanceItemCount(0);
1101 for (cint64 i = 0; i < mConfConcurrentPropagationHashCount; ++i) {
1102 COptimizedComplexVariableRolePropagationProcessingRealizationIteratorDataHashMemoryManaged* instItemRolePropInstIterationDataHash = (*instItemRolePropInstIterationDataHashVector)[i];
1103 if (instItemRolePropInstIterationDataHash) {
1104 rolePropItem->incFillerInstanceItemCount(instItemRolePropInstIterationDataHash->getFillerInstanceItemCount());
1105 rolePropItem->incRealizationFinishedFillerInstanceItemCount(instItemRolePropInstIterationDataHash->getRealizationFinishedFillerInstanceItemCount());
1106 rolePropItem->incPropagationHandledInstanceItemCount(instItemRolePropInstIterationDataHash->getPropagationHandledInstanceItemCount());
1107 rolePropItem->incPropagatedInstanceItemCount(instItemRolePropInstIterationDataHash->getPropagatedInstanceItemCount());
1108 }
1109 }
1110
1111
1112
1113
1114 propagatedVarMapping->incCurrentUpdateId();
1115
1116
1117
1118 return mappingProcessed;
1119
1120 }
1121
1122
1123
1124
1125
1126
1127 }; // end namespace Composition
1128
1129 }; // end namespace Answerer
1130
1131 }; // end namespace Reasoner
1132
1133 }; // end namespace Konclude