/*
* Copyright (C) 2013-2015, 2019 by the Konclude Developer Team.
*
* This file is part of the reasoning system Konclude.
* For details and support, see .
*
* Konclude is free software: you can redistribute it and/or modify
* it under the terms of version 3 of the GNU Lesser General Public
* License (LGPLv3) as published by the Free Software Foundation.
*
* Konclude is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU (Lesser) General Public License for more details.
*
* You should have received a copy of the GNU (Lesser) General Public
* License along with Konclude. If not, see .
*
*/
#include "CQtConcurrentVariableMappingsCompositionJoinComputator.h"
namespace Konclude {
namespace Reasoner {
namespace Answerer {
namespace Composition {
CQtConcurrentVariableMappingsCompositionJoinComputator::CQtConcurrentVariableMappingsCompositionJoinComputator() {
mComputerName = "Concurrent Qt join";
}
CAbstractVariableMappingsCompositionComputator* CQtConcurrentVariableMappingsCompositionJoinComputator::configureComputator(COptimizedComplexExpressionOntologyAnsweringItem* ontoAnsweringItem, CAbstractVariableMappingsCompositionItemRequirementProcessor* reqProcessor, CAnswererContext* answererContext) {
CAbstractVariableMappingsCompositionJoinComputator::configureComputator(ontoAnsweringItem, reqProcessor, answererContext);
mConcurrentJoinComputationTaskCount = CConfigDataReader::readConfigInteger(ontoAnsweringItem->getCalculationConfiguration(), "Konclude.Answering.ConcurrentJoinComputationTaskCount", 211);
mConfBatchMaxHandlingCount = CConfigDataReader::readConfigInteger(ontoAnsweringItem->getCalculationConfiguration(), "Konclude.Answering.ConcurrentJoinComputationMaxBatchHandlingCount", 10000);
mConfSplitTestingBatchMaxHandlingCount = CConfigDataReader::readConfigInteger(ontoAnsweringItem->getCalculationConfiguration(), "Konclude.Answering.ConcurrentJoinLastStepSplitTestingComputationMaxBatchHandlingCount", 1000);
mConfPartitionizedMemoryManagement = CConfigDataReader::readConfigBoolean(ontoAnsweringItem->getCalculationConfiguration(), "Konclude.Answering.ConcurrentJoinPartitionizedMemoryManagement", false);
mConfPerformanceLogging = CConfigDataReader::readConfigBoolean(ontoAnsweringItem->getCalculationConfiguration(), "Konclude.Answering.ConcurrentJoinPerformanceLogging", false);
mConfCheckingSideDirectJoining = CConfigDataReader::readConfigBoolean(ontoAnsweringItem->getCalculationConfiguration(), "Konclude.Answering.ConcurrentJoinDirectJoiningForCheckingSide", false);
mConfConcurrentJoiningHashCount = mConcurrentJoinComputationTaskCount;
mConfConcurrentVariableMappingPartCount = mConcurrentJoinComputationTaskCount;
mConfBatchSizeLimit = 200;
mConfBatchSizeCount = mConcurrentJoinComputationTaskCount;
return this;
}
bool CQtConcurrentVariableMappingsCompositionJoinComputator::computeVariableMappingsJoinComposition(COptimizedComplexVariableJoiningItem* joiningItem, COptimizedComplexBuildingVariableCompositionsItem* buildingVarItem, CAnswererContext* answererContext, bool& processing) {
COptimizedComplexVariableCompositionItem* leftItem = joiningItem->getLeftItem();
COptimizedComplexVariableCompositionItem* rightItem = joiningItem->getRightItem();
COptimizedComplexVariableIndividualMappings* leftVarMapping = leftItem->getVariableMapping();
COptimizedComplexVariableIndividualMappings* rightVarMapping = rightItem->getVariableMapping();
COptimizedComplexVariableIndividualMappingsMultiHash* joinedVarMapping = (COptimizedComplexVariableIndividualMappingsMultiHash*)joiningItem->getVariableMapping();
cint64 leftSize = leftVarMapping->getBindingSize();
cint64 rightSize = rightVarMapping->getBindingSize();
cint64 joinSize = joinedVarMapping->getBindingSize();
COptimizedComplexVariableJoiningBindingPositionMapping* variablePositionMapping = joiningItem->getPositionMapping();
bool itemSameVarReductionJoining = variablePositionMapping->isSelfSameVariableReduction();
cint64* rightPosMapArray = variablePositionMapping->getRightPosMapArray();
cint64* leftPosMapArray = variablePositionMapping->getLeftPosMapArray();
CXLinker* leftKeyBindingLinker = variablePositionMapping->getLeftKeyBindingLinker();
CXLinker* rightKeyBindingLinker = variablePositionMapping->getRightKeyBindingLinker();
QVector& joiningHashVector = joiningItem->getJoiningHashVector();
if (joiningHashVector.size() < mConfConcurrentJoiningHashCount) {
joiningHashVector.resize(mConfConcurrentJoiningHashCount);
}
COptimizedComplexVariableCompositionItemDependence* leftItemDep = joiningItem->getLeftItemDependence();
COptimizedComplexVariableCompositionItemDependence* rightItemDep = joiningItem->getRightItemDependence();
cint64& leftSampleKeyCount = joiningItem->getLeftSampleKeyCount();
cint64& rightSampleKeyCount = joiningItem->getRightSampleKeyCount();
cint64& leftSampleInsertionCount = joiningItem->getLeftSampleInsertionCount();
cint64& rightSampleInsertionCount = joiningItem->getRightSampleInsertionCount();
QAtomicInt atomicLeftSampleKeyCount(leftSampleKeyCount);
QAtomicInt atomicRightSampleKeyCount(rightSampleKeyCount);
bool reduction = variablePositionMapping->isVariableReduction();
CSameRealization* sameRealization = mOntoAnsweringItem->getOntology()->getRealization()->getSameRealization();
auto joiningVariableMappingCreationFunc = [&]( COptimizedComplexVariableIndividualBindings* leftvarMapping, COptimizedComplexVariableIndividualBindingsCardinality* leftNewCardinalites, COptimizedComplexVariableIndividualBindingsCardinality* leftPrevCardinalites,
COptimizedComplexVariableIndividualBindings* rightvarMapping, COptimizedComplexVariableIndividualBindingsCardinality* rightNewCardinalites, COptimizedComplexVariableIndividualBindingsCardinality* righPrevCardinalites,
COptimizedComplexVariableIndividualBindings* joinedBindings, COptimizedComplexVariableIndividualBindingsCardinalityLinker* joinedLinker, COptimizedComplexVariableIndividualBindingsCardinality* providedJoinedCardinalites, CMemoryAllocationManager* memMan) -> COptimizedComplexVariableIndividualBindingsCardinalityLinker* {
COptimizedComplexVariableIndividualBindingsCardinality* joinedCardinalites = nullptr;
COptimizedComplexVariableIndividualBindingsCardinality tmpLeftCardinalites;
COptimizedComplexVariableIndividualBindingsCardinality tmpRightCardinalites;
COptimizedComplexVariableIndividualBindingsCardinality* leftCardinalites = nullptr;
COptimizedComplexVariableIndividualBindingsCardinality* rightCardinalites = nullptr;
bool cardCreated = false;
if (leftNewCardinalites && leftPrevCardinalites) {
tmpLeftCardinalites = *leftNewCardinalites;
tmpLeftCardinalites.substractCardinalities(leftPrevCardinalites);
leftCardinalites = &tmpLeftCardinalites;
} else if (leftNewCardinalites && !leftPrevCardinalites) {
tmpLeftCardinalites = *leftNewCardinalites;
leftCardinalites = &tmpLeftCardinalites;
}
if (rightNewCardinalites && righPrevCardinalites) {
tmpRightCardinalites = *rightNewCardinalites;
tmpRightCardinalites.substractCardinalities(righPrevCardinalites);
rightCardinalites = &tmpRightCardinalites;
} else if (rightNewCardinalites && !righPrevCardinalites) {
tmpRightCardinalites = *rightNewCardinalites;
rightCardinalites = &tmpRightCardinalites;
}
TIndividualInstanceItemDataBinding* leftBinding = leftvarMapping->getBindingArray();
TIndividualInstanceItemDataBinding* rightBinding = rightvarMapping->getBindingArray();
if (!joinedCardinalites && (leftCardinalites || rightCardinalites || reduction)) {
joinedCardinalites = providedJoinedCardinalites;
if (!joinedCardinalites) {
joinedCardinalites = createBindingsCardinalitesForVariableCompositionItems(joiningItem, buildingVarItem, memMan);
}
cardCreated = true;
}
if (leftCardinalites && rightCardinalites) {
*joinedCardinalites = *rightCardinalites;
joinedCardinalites->setSameIndividualsJointlyConsideredCardinality(joinedCardinalites->getSameIndividualsJointlyConsideredCardinality() * leftCardinalites->getSameIndividualsJointlyConsideredCardinality());
joinedCardinalites->setSameIndividualsSeparatlyConsideredCardinality(joinedCardinalites->getSameIndividualsSeparatlyConsideredCardinality() * leftCardinalites->getSameIndividualsSeparatlyConsideredCardinality());
} else if (rightCardinalites) {
*joinedCardinalites = *rightCardinalites;
} else if (leftCardinalites) {
*joinedCardinalites = *leftCardinalites;
} else if (joinedCardinalites) {
joinedCardinalites->initCardinalities(1, 1);
}
if (joinedCardinalites) {
joinedCardinalites->setCardinalityUpdateId(-1);
}
if (!joinedBindings) {
joinedBindings = createBindingsForVariableCompositionItems(joinSize, joiningItem, buildingVarItem, memMan);
}
for (cint64 i = 0; i < leftSize; ++i) {
if (leftPosMapArray[i] > -1) {
joinedBindings->setBinding(leftPosMapArray[i], leftBinding[i]);
} else if (leftPosMapArray[i] == -2) {
TIndividualInstanceItemDataBinding& reducedBinding = leftBinding[i];
CInstanceBindingIndividualCountingVisitor visitor;
sameRealization->visitSameIndividuals(reducedBinding.reference, &visitor);
if (visitor.individualCount > 1) {
joinedCardinalites->setSameIndividualsSeparatlyConsideredCardinality(visitor.individualCount * joinedCardinalites->getSameIndividualsSeparatlyConsideredCardinality());
}
}
}
for (cint64 i = 0; i < rightSize; ++i) {
if (rightPosMapArray[i] > -1) {
joinedBindings->setBinding(rightPosMapArray[i], rightBinding[i]);
} else if (rightPosMapArray[i] == -2) {
TIndividualInstanceItemDataBinding& reducedBinding = rightBinding[i];
CInstanceBindingIndividualCountingVisitor visitor;
sameRealization->visitSameIndividuals(reducedBinding.reference, &visitor);
if (visitor.individualCount > 1) {
joinedCardinalites->setSameIndividualsSeparatlyConsideredCardinality(visitor.individualCount * joinedCardinalites->getSameIndividualsSeparatlyConsideredCardinality());
}
}
}
//QStringList bindingsStringList;
//for (cint64 i = 0; i < joinSize; ++i) {
// bindingsStringList.append(CIRIName::getRecentIRIName(joinedBindings->getBinding(i).reference.getIndividual()->getIndividualNameLinker()));
//}
//QString bindingsString = bindingsStringList.join("\t");
if (!joinedLinker) {
joinedLinker = createBindingsLinkerForVariableCompositionItems(joinedBindings, joinedCardinalites, joiningItem, buildingVarItem, memMan);
} else {
joinedLinker->setInitialCardinality(joinedCardinalites);
}
return joinedLinker;
};
auto joiningVariableMappingCreationLeftRightFunc = [&]( COptimizedComplexVariableIndividualBindings* leftvarMapping, COptimizedComplexVariableIndividualBindingsCardinality* leftNewCardinalites, COptimizedComplexVariableIndividualBindingsCardinality* leftPrevCardinalites,
COptimizedComplexVariableIndividualBindings* rightvarMapping, COptimizedComplexVariableIndividualBindingsCardinality* rightNewCardinalites, COptimizedComplexVariableIndividualBindingsCardinality* righPrevCardinalites, bool swap,
COptimizedComplexVariableIndividualBindings* joinedBindings, COptimizedComplexVariableIndividualBindingsCardinalityLinker* joinedLinker, COptimizedComplexVariableIndividualBindingsCardinality* providedJoinedCardinalites, CMemoryAllocationManager* memMan) -> COptimizedComplexVariableIndividualBindingsCardinalityLinker* {
if (swap) {
return joiningVariableMappingCreationFunc(rightvarMapping, rightNewCardinalites, righPrevCardinalites, leftvarMapping, leftNewCardinalites, leftPrevCardinalites, joinedBindings, joinedLinker, providedJoinedCardinalites, memMan);
} else {
return joiningVariableMappingCreationFunc(leftvarMapping, leftNewCardinalites, leftPrevCardinalites, rightvarMapping, rightNewCardinalites, righPrevCardinalites, joinedBindings, joinedLinker, providedJoinedCardinalites, memMan);
}
};
auto joiningVariableMappingCreationLeftRightLinkerFunc = [&]( COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker* leftLinker, COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker* rightLinker, bool swap,
COptimizedComplexVariableIndividualBindings* joinedBindings, COptimizedComplexVariableIndividualBindingsCardinalityLinker* joinedLinker, COptimizedComplexVariableIndividualBindingsCardinality* providedJoinedCardinalites, CMemoryAllocationManager* memMan) -> COptimizedComplexVariableIndividualBindingsCardinalityLinker* {
return joiningVariableMappingCreationLeftRightFunc(leftLinker->getBindings(), leftLinker->getNewCardinalities(), leftLinker->getPreviousCardinalities(), rightLinker->getBindings(), rightLinker->getNewCardinalities(), rightLinker->getPreviousCardinalities(), swap, joinedBindings, joinedLinker, providedJoinedCardinalites, memMan);
};
//QList images;
//std::function mapFunc = [](const QString &img) {
// return img + "x";
//};
//std::function redFunc = [](QString &img, const QString &red) {
//};
//QFuture thumbnails = QtConcurrent::mapped(images, mapFunc);
cint64 currentBatchSizeCount = mConfBatchSizeCount;
cint64 currentBatchSizeLimit = mConfBatchSizeLimit;
function& keyMappingBatchDataVector)> joiningHashSampleBatchingFunc =
[&](bool left, COptimizedComplexVariableCompositionItemDependence* itemDep, cint64& sampleInsertionCount, CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector& keyMappingBatchDataVector) -> void {
cint64 batchSize = 1;
++sampleInsertionCount;
COptimizedComplexVariableIndividualBindingsCardinalityLinker* firstBindingLinker = itemDep->getBatchCurrentBindingsCardinalityLinker(true);
COptimizedComplexVariableIndividualBindingsCardinalityLinker* lastBindingLinker = firstBindingLinker;
while (batchSize++ < currentBatchSizeLimit && itemDep->getBatchCurrentBindingsCardinalityLinker(false) && sampleInsertionCount < mConfSamplingBasedJoinMappingSize) {
++sampleInsertionCount;
lastBindingLinker = itemDep->getBatchCurrentBindingsCardinalityLinker(true);
}
CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData* vecData = keyMappingBatchDataVector.getVectorLinkerData(keyMappingBatchDataVector.getNextIndex(true), true);
CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker* data = nullptr;
if (!mConfPartitionizedMemoryManagement) {
data = keyMappingBatchDataVector.createBatchLinker();
} else {
data = vecData->createBatchLinker();
}
data->setBatchBindingsCardinalityLinkers(firstBindingLinker, lastBindingLinker);
data->setInserting(true);
data->setLeftSide(left);
vecData->addBatchLinker(data);
};
cint64 batchMaxHandlingCount = mConfBatchMaxHandlingCount;
if (mConfAllowSplitModeActivation && joiningItem->isLastComputation() && !joiningItem->isSplitComputationMode()) {
batchMaxHandlingCount = mConfSplitTestingBatchMaxHandlingCount;
}
function& keyMappingBatchDataVector)> joiningHashBindingsInsertingCheckingBatchingFunc =
[&](bool left, COptimizedComplexVariableCompositionItemDependence* itemDep, bool inserting, CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector& keyMappingBatchDataVector) -> void {
cint64 batchHandlingCount = 0;
while (batchHandlingCount++ < batchMaxHandlingCount && itemDep->getBatchCurrentBindingsCardinalityLinker(false)) {
cint64 batchSize = 1;
COptimizedComplexVariableIndividualBindingsCardinalityLinker* firstBindingLinker = itemDep->getBatchCurrentBindingsCardinalityLinker(true);
COptimizedComplexVariableIndividualBindingsCardinalityLinker* lastBindingLinker = firstBindingLinker;
while (batchSize++ < currentBatchSizeLimit && itemDep->getBatchCurrentBindingsCardinalityLinker(false)) {
lastBindingLinker = itemDep->getBatchCurrentBindingsCardinalityLinker(true);
}
CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData* vecData = keyMappingBatchDataVector.getVectorLinkerData(keyMappingBatchDataVector.getNextIndex(true), true);
CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker* data = nullptr;
if (!mConfPartitionizedMemoryManagement) {
data = keyMappingBatchDataVector.createBatchLinker();
} else {
data = vecData->createBatchLinker();
}
data->setBatchBindingsCardinalityLinkers(firstBindingLinker, lastBindingLinker);
data->setInserting(inserting);
data->setLeftSide(left);
vecData->addBatchLinker(data);
}
};
function& keyMappingBatchDataVector)> joiningHashCardinalitiesInsertingCheckingBatchingFunc =
[&](bool left, COptimizedComplexVariableCompositionItemDependence* itemDep, bool inserting, CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector& keyMappingBatchDataVector) -> void {
cint64 batchHandlingCount = 0;
while (batchHandlingCount++ < batchMaxHandlingCount && itemDep->getBatchCurrentUpdatedCardinalityLinker(false)) {
cint64 batchSize = 1;
COptimizedComplexVariableIndividualUpdateCardinalityLinker* firstCardinalityLinker = itemDep->getBatchCurrentUpdatedCardinalityLinker(true);
COptimizedComplexVariableIndividualUpdateCardinalityLinker* lastCardinalityLinker = firstCardinalityLinker;
while (batchSize++ < currentBatchSizeLimit && itemDep->getBatchCurrentUpdatedCardinalityLinker(false)) {
lastCardinalityLinker = itemDep->getBatchCurrentUpdatedCardinalityLinker(true);
}
CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData* vecData = keyMappingBatchDataVector.getVectorLinkerData(keyMappingBatchDataVector.getNextIndex(true), true);
CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker* data = nullptr;
if (!mConfPartitionizedMemoryManagement) {
data = keyMappingBatchDataVector.createBatchLinker();
} else {
data = vecData->createBatchLinker();
}
data->setBatchCardinalityUpdateLinkers(firstCardinalityLinker, lastCardinalityLinker);
data->setInserting(inserting);
data->setLeftSide(left);
vecData->addBatchLinker(data);
}
};
bool splitComputationMode = joiningItem->isSplitComputationMode();
cint64 leftSplitCount = joiningItem->getLeftSplitCount();
cint64 rightSplitCount = joiningItem->getRightSplitCount();
cint64 leftSplitPos = joiningItem->getLeftSplitPosition();
cint64 rightSplitPos = joiningItem->getRightSplitPosition();
CXLinker* leftRemainingBindingLinker = nullptr;
CXLinker* rightRemainingBindingLinker = nullptr;
if (splitComputationMode) {
leftRemainingBindingLinker = variablePositionMapping->getLeftRemainingBindingLinker();
rightRemainingBindingLinker = variablePositionMapping->getRightRemainingBindingLinker();
}
cint64 splitComputationCount = 0;
// map & reduce of step 1
cint64 keyMappingSize = mConfConcurrentJoiningHashCount;
function(const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData* const & batchData)> bindingBatchesHashingKeyMapFunc =
[&](const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData* const & batchData) ->CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector {
CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector keyMappedVec(keyMappingSize);
if (batchData) {
CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker* batchLinker = batchData->getNextBatchLinker();
while (batchLinker) {
bool leftSide = batchLinker->isLeftSide();
bool inserting = batchLinker->isInserting();
COptimizedComplexVariableIndividualBindingsCardinalityLinker* bindingsCardinalityCurrentLinker = batchLinker->getNextBatchBindingsCardinalityLinker(true);
while (bindingsCardinalityCurrentLinker) {
CXLinker* keyBindingLinker = leftKeyBindingLinker;
CXLinker* remainingBindingLinker = leftRemainingBindingLinker;
cint64 splitCount = leftSplitCount;
cint64 splitPos = leftSplitPos;
if (!leftSide) {
keyBindingLinker = rightKeyBindingLinker;
remainingBindingLinker = rightRemainingBindingLinker;
splitPos = rightSplitPos;
splitCount = rightSplitCount;
}
COptimizedComplexVariableJoiningWithRemainingHasher bindingHasher(bindingsCardinalityCurrentLinker->getBindings()->getBindingArray(), keyBindingLinker, remainingBindingLinker);
cint64 keyHashValue = bindingHasher.getHashValue();
uint uintHashValue = uint((keyHashValue >> (8 * sizeof(uint) - 1)) ^ keyHashValue);
cint64 mappedKeyIndex = uintHashValue % keyMappingSize;
if (!splitComputationMode || bindingHasher.getRemainingHashValue() % splitCount == splitPos) {
CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData* keyMappedVecData = keyMappedVec.getVectorLinkerData(mappedKeyIndex, true);
CQtConcurrentVariableMappingsCompositionJoinMappedKeysBindingsCardinalityLinker* mappedKeyDataLinker = nullptr;
if (!mConfPartitionizedMemoryManagement) {
mappedKeyDataLinker = keyMappedVec.createBatchLinker();
} else {
mappedKeyDataLinker = keyMappedVecData->createBatchLinker();
}
mappedKeyDataLinker->initMappedKeysBindingsCardinalityData(bindingHasher, mappedKeyIndex, bindingsCardinalityCurrentLinker->getBindings(), bindingsCardinalityCurrentLinker->getInitialCardinalities(), nullptr, leftSide, inserting);
keyMappedVecData->addBatchLinker(mappedKeyDataLinker);
}
bindingsCardinalityCurrentLinker = batchLinker->getNextBatchBindingsCardinalityLinker(true);
}
COptimizedComplexVariableIndividualUpdateCardinalityLinker* cardinalityUpdateLinker = batchLinker->getNextCardinalityUpdateLinker(true);
while (cardinalityUpdateLinker) {
CXLinker* keyBindingLinker = leftKeyBindingLinker;
CXLinker* remainingBindingLinker = leftRemainingBindingLinker;
cint64 splitCount = leftSplitCount;
cint64 splitPos = leftSplitPos;
if (!leftSide) {
keyBindingLinker = rightKeyBindingLinker;
remainingBindingLinker = rightRemainingBindingLinker;
splitPos = rightSplitPos;
splitCount = rightSplitCount;
}
COptimizedComplexVariableJoiningWithRemainingHasher bindingHasher(cardinalityUpdateLinker->getUpdatedBindingsCardinalityLinker()->getBindings()->getBindingArray(), keyBindingLinker, remainingBindingLinker);
cint64 keyHashValue = bindingHasher.getHashValue();
uint uintHashValue = uint((keyHashValue >> (8 * sizeof(uint) - 1)) ^ keyHashValue);
cint64 mappedKeyIndex = uintHashValue % keyMappingSize;
if (!splitComputationMode || bindingHasher.getRemainingHashValue() % splitCount == splitPos) {
CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData* keyMappedVecData = keyMappedVec.getVectorLinkerData(mappedKeyIndex, true);
CQtConcurrentVariableMappingsCompositionJoinMappedKeysBindingsCardinalityLinker* mappedKeyDataLinker = nullptr;
if (!mConfPartitionizedMemoryManagement) {
mappedKeyDataLinker = keyMappedVec.createBatchLinker();
} else {
mappedKeyDataLinker = keyMappedVecData->createBatchLinker();
}
mappedKeyDataLinker->initMappedKeysBindingsCardinalityData(bindingHasher, mappedKeyIndex, cardinalityUpdateLinker->getUpdatedBindingsCardinalityLinker()->getBindings(), cardinalityUpdateLinker->getNewCardinality(), cardinalityUpdateLinker->getPreviousCardinality(), leftSide, inserting);
keyMappedVecData->addBatchLinker(mappedKeyDataLinker);
}
cardinalityUpdateLinker = batchLinker->getNextCardinalityUpdateLinker(true);
}
batchLinker = batchLinker->getNext();
}
}
return keyMappedVec;
};
function& vec, const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector& reduceVec)> bindingBatchesHashingKeyReduceFunc =
[&](CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector& vec, const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector& reduceVec) -> void {
if (vec.size() < reduceVec.size()) {
vec.resize(reduceVec.size());
}
for (cint64 i = 0; i < vec.count(); ++i) {
CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData*& vecData = vec[i];
CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData* const & reduceVecData = reduceVec[i];
if (reduceVecData) {
if (!vecData) {
vecData = reduceVecData;
} else {
vecData->mergeBatchLinkers(reduceVecData);
}
}
}
vec.appendMemoryPool(reduceVec.getMemoryPools());
};
// map & reduce of step 2
cint64 bindingsMappingSize = mConfConcurrentVariableMappingPartCount;
auto joinedVariableMappingCreationRightLeftCopyingVecInsertionFunc = [&](COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker* leftLinker, COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker* rightLinker, bool swap,
CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector& bindingsMappedVec, CMemoryAllocationManager* memMan, COptimizedComplexVariableIndividualBindings*& tmpJoinedBindings) -> void {
CQtConcurrentVariableMappingsCompositionJoinMappedJoinedBindingsCardinalityLinker* mappedBindingsDataLinker = nullptr;
if (!mConfPartitionizedMemoryManagement) {
mappedBindingsDataLinker = bindingsMappedVec.createBatchLinker();
COptimizedComplexVariableIndividualBindingsCardinalityLinker* joinedLinker = joiningVariableMappingCreationLeftRightLinkerFunc(leftLinker, rightLinker, swap, nullptr, nullptr, nullptr, memMan);
COptimizedComplexVariableIndividualBindingsHasher bindingsHasher(joinedLinker);
cint64 keyHashValue = bindingsHasher.getHashValue();
cint64 mappedBindingsIndex = uint((keyHashValue >> (8 * sizeof(uint) - 1)) ^ keyHashValue) % bindingsMappingSize;
mappedBindingsDataLinker->initMappedJoinedBindingsCardinalityData(mappedBindingsIndex, bindingsHasher, joinedLinker);
bindingsMappedVec.addBatchLinker(mappedBindingsIndex, mappedBindingsDataLinker);
} else {
if (!tmpJoinedBindings) {
tmpJoinedBindings = createBindingsForVariableCompositionItems(joinSize, joiningItem, buildingVarItem, memMan);
}
COptimizedComplexVariableIndividualBindingsCardinalityLinker tmpJoinedLinker;
COptimizedComplexVariableIndividualBindingsCardinality tmpJoinedCardinalites;
joiningVariableMappingCreationLeftRightLinkerFunc(leftLinker, rightLinker, swap, tmpJoinedBindings, &tmpJoinedLinker, &tmpJoinedCardinalites, memMan);
COptimizedComplexVariableIndividualBindingsHasher tmpBindingsHasher(&tmpJoinedLinker);
cint64 keyHashValue = tmpBindingsHasher.getHashValue();
cint64 mappedBindingsIndex = uint((keyHashValue >> (8 * sizeof(uint) - 1)) ^ keyHashValue) % bindingsMappingSize;
CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData* bindingsMappedVecData = bindingsMappedVec.getVectorLinkerData(mappedBindingsIndex, true);
// copy data with corresponding memory manager
memMan = bindingsMappedVecData->getMemoryAllocationManager();
COptimizedComplexVariableIndividualBindings* joinedBindings = createBindingsForVariableCompositionItems(joinSize, joiningItem, buildingVarItem, memMan);
for (cint64 i = 0; i < joinSize; ++i) {
joinedBindings->setBinding(i, tmpJoinedBindings->getBinding(i));
}
COptimizedComplexVariableIndividualBindingsCardinality* joinedCardinalites = nullptr;
if (tmpJoinedLinker.getCurrentCardinalities()) {
joinedCardinalites = createBindingsCardinalitesForVariableCompositionItems(joiningItem, buildingVarItem, memMan);
*joinedCardinalites = tmpJoinedCardinalites;
}
COptimizedComplexVariableIndividualBindingsCardinalityLinker* joinedLinker = createBindingsLinkerForVariableCompositionItems(joinedBindings, joinedCardinalites, joiningItem, buildingVarItem, memMan);
COptimizedComplexVariableIndividualBindingsHasher bindingsHasher(joinedLinker, keyHashValue);
mappedBindingsDataLinker = bindingsMappedVecData->createBatchLinker();
mappedBindingsDataLinker->initMappedJoinedBindingsCardinalityData(mappedBindingsIndex, bindingsHasher, joinedLinker);
bindingsMappedVecData->addBatchLinker(mappedBindingsDataLinker);
}
};
function(const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData* const & keyMappedBindingsVecData)> bindingBatchesHashingJoinMapFunc =
[&](const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData* const & keyMappedBindingsVecData) -> CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector {
CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector bindingsMappedVec(bindingsMappingSize);
COptimizedComplexVariableJoiningHashMemoryManaged* lastJoiningHash = nullptr;
QTime* time = nullptr;
if (mConfPerformanceLogging) {
time = new QTime();
time->start();
}
if (keyMappedBindingsVecData) {
CQtConcurrentVariableMappingsCompositionJoinMappedKeysBindingsCardinalityLinker* batchLinker = keyMappedBindingsVecData->getNextBatchLinker();
COptimizedComplexVariableIndividualBindings* tmpJoinedBindings = nullptr;
while (batchLinker) {
cint64 mappedKeyIndex = batchLinker->getMappedKeyIndex();
COptimizedComplexVariableJoiningHashMemoryManaged*& joiningHash = joiningHashVector[mappedKeyIndex];
if (!joiningHash) {
joiningHash = new COptimizedComplexVariableJoiningHashMemoryManaged(leftKeyBindingLinker, rightKeyBindingLinker);
}
lastJoiningHash = joiningHash;
const COptimizedComplexVariableJoiningHasher& bindingHasher = batchLinker->getBindingHasher();
bool sampling = !joiningItem->isSamplingCompleted();
bool leftSide = batchLinker->isLeftSide();
bool inserting = batchLinker->isInserting();
COptimizedComplexVariableIndividualBindings* varMapping = batchLinker->getVariableMapping();
COptimizedComplexVariableIndividualBindingsCardinality* newCardinalites = batchLinker->getNewCardinalites();
COptimizedComplexVariableIndividualBindingsCardinality* prevCardinalites = batchLinker->getPreviousCardinalites();
COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker* linker = joiningHash->createDataLinker();
if (inserting) {
COptimizedComplexVariableJoiningData& joiningData = (*joiningHash)[bindingHasher];
if (sampling && !joiningData.hasBindings(leftSide)) {
// TODO: make thread/concurrent safe
if (leftSide) {
atomicLeftSampleKeyCount.ref();
} else {
atomicRightSampleKeyCount.ref();
}
}
linker->initLinker(varMapping, newCardinalites, prevCardinalites);
#ifdef CONCURRENT_JOIN_COMPUTATION_PERFORMANCE_MEASUREMENT
joiningData.mInsertionCount++;
#endif //CONCURRENT_JOIN_COMPUTATION_PERFORMANCE_MEASUREMENT
joiningHash->mInsertionCount++;
for (COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker* existingOtherLinkerIt = joiningData.getBindingLinker(!leftSide); existingOtherLinkerIt; existingOtherLinkerIt = existingOtherLinkerIt->getNext()) {
joinedVariableMappingCreationRightLeftCopyingVecInsertionFunc(existingOtherLinkerIt, linker, leftSide, bindingsMappedVec, bindingsMappedVec.getMemoryAllocationManager(), tmpJoinedBindings);
#ifdef CONCURRENT_JOIN_COMPUTATION_PERFORMANCE_MEASUREMENT
joiningData.mJoinedCount++;
#endif //CONCURRENT_JOIN_COMPUTATION_PERFORMANCE_MEASUREMENT
joiningHash->mJoinedCount++;
}
joiningData.addBindingLinker(leftSide, linker);
if (leftSide) {
#ifdef CONCURRENT_JOIN_COMPUTATION_PERFORMANCE_MEASUREMENT
joiningData.mLeftAdded++;
#endif //CONCURRENT_JOIN_COMPUTATION_PERFORMANCE_MEASUREMENT
joiningHash->mLeftAdded++;
} else {
#ifdef CONCURRENT_JOIN_COMPUTATION_PERFORMANCE_MEASUREMENT
joiningData.mRightAdded++;
#endif //CONCURRENT_JOIN_COMPUTATION_PERFORMANCE_MEASUREMENT
joiningHash->mRightAdded++;
}
} else {
const COptimizedComplexVariableJoiningData& joiningData = joiningHash->value(bindingHasher);
#ifdef CONCURRENT_JOIN_COMPUTATION_PERFORMANCE_MEASUREMENT
joiningData.mCheckingCount++;
#endif //CONCURRENT_JOIN_COMPUTATION_PERFORMANCE_MEASUREMENT
joiningHash->mCheckingCount++;
COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker tmpLinker(varMapping, newCardinalites, prevCardinalites);
for (COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker* existingOtherLinkerIt = joiningData.getBindingLinker(!leftSide); existingOtherLinkerIt; existingOtherLinkerIt = existingOtherLinkerIt->getNext()) {
joinedVariableMappingCreationRightLeftCopyingVecInsertionFunc(existingOtherLinkerIt, &tmpLinker, leftSide, bindingsMappedVec, bindingsMappedVec.getMemoryAllocationManager(), tmpJoinedBindings);
#ifdef CONCURRENT_JOIN_COMPUTATION_PERFORMANCE_MEASUREMENT
joiningData.mJoinedCount++;
#endif //CONCURRENT_JOIN_COMPUTATION_PERFORMANCE_MEASUREMENT
joiningHash->mJoinedCount++;
}
}
batchLinker = batchLinker->getNext();
}
}
if (lastJoiningHash && time) {
lastJoiningHash->mProcessingTime += time->elapsed();
delete time;
}
return bindingsMappedVec;
};
function& vec, const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector& reduceVec)> bindingBatchesHashingJoinReduceFunc =
[&](CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector& vec, const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector& reduceVec) -> void {
if (vec.size() < reduceVec.size()) {
vec.resize(reduceVec.size());
}
for (cint64 i = 0; i < vec.count(); ++i) {
CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData*& vecData = vec[i];
CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData* const & reduceVecData = reduceVec[i];
if (reduceVecData) {
if (!vecData) {
vecData = reduceVecData;
} else {
vecData->mergeBatchLinkers(reduceVecData);
}
}
}
vec.appendMemoryPool(reduceVec.getMemoryPools());
};
// map (& reduce) of step 3
function* const & mappedBindingsVecData)> bindingBatchesInsertingMapFunc =
[&](const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData* const & mappedBindingsVecData) -> COptimizedComplexVariableIndividualMappingsMultiHashPart* {
QTime* time = nullptr;
if (mConfPerformanceLogging) {
time = new QTime();
time->start();
}
COptimizedComplexVariableIndividualMappingsMultiHashPart* joinedVarMappingMultiHashPart = nullptr;
if (mappedBindingsVecData) {
CQtConcurrentVariableMappingsCompositionJoinMappedJoinedBindingsCardinalityLinker* batchLinker = mappedBindingsVecData->getNextBatchLinker();
while (batchLinker) {
cint64 mappedBindingsIndex = batchLinker->getBindingsIndex();
if (!joinedVarMappingMultiHashPart) {
joinedVarMappingMultiHashPart = joinedVarMapping->getMultiHashPart(mappedBindingsIndex);
}
COptimizedComplexVariableIndividualBindingsCardinalityLinker* joinedLinker = batchLinker->getBindingsCardinalityLinker();
cint64 hashValue = batchLinker->getBindingHasher().getHashValue();
joinedVarMappingMultiHashPart->addInsertingBindingsCardinalityLinkerCopy(joinedLinker, hashValue, true);
batchLinker = batchLinker->getNext();
}
}
if (joinedVarMappingMultiHashPart && time) {
joinedVarMappingMultiHashPart->mStatsProcessingTime += time->elapsed();
delete time;
}
return joinedVarMappingMultiHashPart;
};
function& mappedVarMappingParts)> bindingBatchesInsertingReduceFunc =
[&](const QVector& mappedVarMappingParts) -> void {
for (QVector::const_iterator it = mappedVarMappingParts.constBegin(), itEnd = mappedVarMappingParts.constEnd(); it != itEnd; ++it) {
COptimizedComplexVariableIndividualMappingsMultiHashPart* reduceMappPart = (*it);
if (reduceMappPart) {
joinedVarMapping->updateLinkers(reduceMappPart);
joinedVarMapping->updateBindingCount(reduceMappPart);
}
}
};
function clearMemoryPoolsFunc = [&](CMemoryPool* memPools) -> void {
while (memPools) {
CMemoryPool* tmpMemPool = memPools;
memPools = memPools->getNextMemoryPool();
delete [] tmpMemPool->getMemoryBlockData();
delete tmpMemPool;
}
};
function& keyMappingBatchDataVector)> mapReducedConcurrentJoinInsertionComputation = [&](CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector& keyMappingBatchDataVector) -> void {
if (!keyMappingBatchDataVector.isEmpty()) {
QTime* stepTimer = nullptr;
if (mConfPerformanceLogging) {
stepTimer = new QTime();
stepTimer->start();
}
CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector mappedJoiningHashKeyVec = QtConcurrent::blockingMappedReduced>(keyMappingBatchDataVector, bindingBatchesHashingKeyMapFunc, bindingBatchesHashingKeyReduceFunc);
if (stepTimer) {
cint64 partitionsUsed = 0;
for (cint64 i = 0; i < keyMappingBatchDataVector.size(); ++i) {
if (keyMappingBatchDataVector[i]) {
++partitionsUsed;
}
}
QString x1 = QString("Insertion step 1 of concurrent join computation for item %3 with %4/%1 tasks took %2 ms.").arg(keyMappingBatchDataVector.size()).arg(stepTimer->elapsed()).arg(joiningItem->getComputationStepId()).arg(partitionsUsed);
if (mConfDirectPerformanceConsoleOutput) {
cout << x1.toLocal8Bit().constData() << endl;
} else {
LOG(INFO, getLogDomain(), x1, this);
}
delete stepTimer;
}
if (mConfPerformanceLogging) {
stepTimer = new QTime();
stepTimer->start();
}
CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector mappedJoinedHashKeyVec = QtConcurrent::blockingMappedReduced>(mappedJoiningHashKeyVec, bindingBatchesHashingJoinMapFunc, bindingBatchesHashingJoinReduceFunc);
if (stepTimer) {
cint64 partitionsUsed = 0;
for (cint64 i = 0; i < mappedJoiningHashKeyVec.size(); ++i) {
if (mappedJoiningHashKeyVec[i]) {
++partitionsUsed;
}
}
QString x2 = QString("Insertion step 2 of concurrent join computation for item %3 with %4/%1 tasks took %2 ms.").arg(mappedJoiningHashKeyVec.size()).arg(stepTimer->elapsed()).arg(joiningItem->getComputationStepId()).arg(partitionsUsed);
if (mConfDirectPerformanceConsoleOutput) {
cout << x2.toLocal8Bit().constData() << endl;
} else {
LOG(INFO, getLogDomain(), x2, this);
}
delete stepTimer;
cint64 statMaxJoiningCount = 0;
cint64 statMaxProcessingTime = 0;
cint64 statTotalJoiningCount = 0;
cint64 statTotalProcessingTime = 0;
cint64 statUsedCount = 0;
for (cint64 i = 0; i < mappedJoiningHashKeyVec.size(); ++i) {
COptimizedComplexVariableJoiningHashMemoryManaged* joiningHash = joiningHashVector[i];
if (joiningHash && joiningHash->mJoinedCount > 0) {
++statUsedCount;
statMaxJoiningCount = qMax(statMaxJoiningCount, joiningHash->mJoinedCount);
statMaxProcessingTime = qMax(statMaxProcessingTime, joiningHash->mProcessingTime);
statTotalJoiningCount += joiningHash->mJoinedCount;
statTotalProcessingTime += joiningHash->mProcessingTime;
}
}
cint64 averageJoiningCount = 0;
if (statUsedCount > 0) {
averageJoiningCount = statTotalJoiningCount / statUsedCount;
}
cint64 averageProcessingTime = 0;
if (statUsedCount > 0) {
averageProcessingTime = statTotalProcessingTime / statUsedCount;
}
QString x3 = QString("Insertion step 2 of concurrent join computation for item %1 had in average %2 (max %3) created joins and took in average %4 (max %5) ms.").arg(joiningItem->getComputationStepId()).arg(averageJoiningCount).arg(statMaxJoiningCount).arg(averageProcessingTime).arg(statMaxProcessingTime);
if (mConfDirectPerformanceConsoleOutput) {
cout << x3.toLocal8Bit().constData() << endl;
} else {
LOG(INFO, getLogDomain(), x3, this);
}
}
if (mConfPerformanceLogging) {
stepTimer = new QTime();
stepTimer->start();
}
QVector mappedVarMappingParts = QtConcurrent::blockingMapped>(mappedJoinedHashKeyVec, bindingBatchesInsertingMapFunc);
if (stepTimer) {
cint64 partitionsUsed = 0;
for (cint64 i = 0; i < mappedJoinedHashKeyVec.size(); ++i) {
if (mappedJoinedHashKeyVec[i]) {
++partitionsUsed;
}
}
QString x4 = QString("Insertion step 3 of concurrent join computation for item %3 with %4/%1 tasks took %2 ms.").arg(mappedJoinedHashKeyVec.size()).arg(stepTimer->elapsed()).arg(joiningItem->getComputationStepId()).arg(partitionsUsed);
if (mConfDirectPerformanceConsoleOutput) {
cout << x4.toLocal8Bit().constData() << endl;
} else {
LOG(INFO, getLogDomain(), x4, this);
}
delete stepTimer;
cint64 statMaxInsertionCount = 0;
cint64 statMaxProcessingTime = 0;
cint64 statTotalInsertionCount = 0;
cint64 statTotalProcessingTime = 0;
cint64 statUsedCount = 0;
for (cint64 i = 0; i < mappedJoinedHashKeyVec.size(); ++i) {
COptimizedComplexVariableIndividualMappingsMultiHashPart* joinedVarMappingMultiHashPart = joinedVarMapping->getMultiHashPart(i);
if (joinedVarMappingMultiHashPart && joinedVarMappingMultiHashPart->mStatInsertionCount > 0) {
++statUsedCount;
statMaxInsertionCount = qMax(statMaxInsertionCount, joinedVarMappingMultiHashPart->mStatInsertionCount);
statMaxProcessingTime = qMax(statMaxProcessingTime, joinedVarMappingMultiHashPart->mStatsProcessingTime);
statTotalInsertionCount += joinedVarMappingMultiHashPart->mStatInsertionCount;
statTotalProcessingTime += joinedVarMappingMultiHashPart->mStatsProcessingTime;
}
}
cint64 averageInsertionCount = 0;
if (statUsedCount > 0) {
averageInsertionCount = statTotalInsertionCount / statUsedCount;
}
cint64 averageProcessingTime = 0;
if (statUsedCount > 0) {
averageProcessingTime = statTotalProcessingTime / statUsedCount;
}
QString x5 = QString("Insertion step 3 of concurrent join computation for item %1 had in average %2 (max %3) bindings insertions and took in average %4 (max %5) ms.").arg(joiningItem->getComputationStepId()).arg(averageInsertionCount).arg(statMaxInsertionCount).arg(averageProcessingTime).arg(statMaxProcessingTime);
if (mConfDirectPerformanceConsoleOutput) {
cout << x5.toLocal8Bit().constData() << endl;
} else {
LOG(INFO, getLogDomain(), x5, this);
}
}
if (mConfPerformanceLogging) {
stepTimer = new QTime();
stepTimer->start();
}
bindingBatchesInsertingReduceFunc(mappedVarMappingParts);
CMemoryPool* mappedKeyBatchHashVecMemPools = keyMappingBatchDataVector.takeMemoryPools();
CMemoryPool* mappedJoiningHashVecMemPools = mappedJoiningHashKeyVec.takeMemoryPools();
CMemoryPool* mappedJoinedHashVecMemPools = mappedJoinedHashKeyVec.takeMemoryPools();
CConcurrentTaskScheduler::run([=] {
clearMemoryPoolsFunc(mappedKeyBatchHashVecMemPools);
});
CConcurrentTaskScheduler::run([=] {
clearMemoryPoolsFunc(mappedJoiningHashVecMemPools);
});
CConcurrentTaskScheduler::run([=] {
clearMemoryPoolsFunc(mappedJoinedHashVecMemPools);
});
keyMappingBatchDataVector.clear();
if (stepTimer) {
cint64 partitionsUsed = 0;
for (cint64 i = 0; i < mappedVarMappingParts.size(); ++i) {
if (mappedVarMappingParts[i]) {
++partitionsUsed;
}
}
QString x6 = QString("Insertion step 4 of concurrent join computation for item %3 with %4/%1 tasks took %2 ms.").arg(mappedVarMappingParts.size()).arg(stepTimer->elapsed()).arg(joiningItem->getComputationStepId()).arg(partitionsUsed);
if (mConfDirectPerformanceConsoleOutput) {
cout << x6.toLocal8Bit().constData() << endl;
} else {
LOG(INFO, getLogDomain(), x6, this);
}
delete stepTimer;
}
}
};
function(const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData* const & batchData)> bindingBatchesHashingDirectJoiningMapFunc =
[&](const CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVectorData* const & batchData) ->CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector {
CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector bindingsMappedVec(bindingsMappingSize);
if (batchData) {
COptimizedComplexVariableIndividualBindings* tmpJoinedBindings = nullptr;
CQtConcurrentVariableMappingsCompositionJoinBindingsCardinalityLinkerBatchLinker* batchLinker = batchData->getNextBatchLinker();
while (batchLinker) {
bool leftSide = batchLinker->isLeftSide();
COptimizedComplexVariableIndividualBindingsCardinalityLinker* bindingsCardinalityCurrentLinker = batchLinker->getNextBatchBindingsCardinalityLinker(true);
while (bindingsCardinalityCurrentLinker) {
CXLinker* keyBindingLinker = leftKeyBindingLinker;
CXLinker* remainingBindingLinker = leftRemainingBindingLinker;
cint64 splitCount = leftSplitCount;
cint64 splitPos = leftSplitPos;
if (!leftSide) {
keyBindingLinker = rightKeyBindingLinker;
remainingBindingLinker = rightRemainingBindingLinker;
splitPos = rightSplitPos;
splitCount = rightSplitCount;
}
COptimizedComplexVariableJoiningWithRemainingHasher bindingHasher(bindingsCardinalityCurrentLinker->getBindings()->getBindingArray(), keyBindingLinker, remainingBindingLinker);
cint64 keyHashValue = bindingHasher.getHashValue();
uint uintHashValue = uint((keyHashValue >> (8 * sizeof(uint) - 1)) ^ keyHashValue);
cint64 mappedKeyIndex = uintHashValue % keyMappingSize;
if (!splitComputationMode || bindingHasher.getRemainingHashValue() % splitCount == splitPos) {
COptimizedComplexVariableJoiningHashMemoryManaged* joiningHash = joiningHashVector[mappedKeyIndex];
if (joiningHash) {
const COptimizedComplexVariableJoiningData& joiningData = joiningHash->value(bindingHasher);
bool leftSide = batchLinker->isLeftSide();
COptimizedComplexVariableIndividualBindings* varMapping = bindingsCardinalityCurrentLinker->getBindings();
COptimizedComplexVariableIndividualBindingsCardinality* newCardinalites = bindingsCardinalityCurrentLinker->getInitialCardinalities();
COptimizedComplexVariableIndividualBindingsCardinality* prevCardinalites = nullptr;
COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker* linker = joiningHash->createDataLinker();
COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker tmpLinker(varMapping, newCardinalites, prevCardinalites);
for (COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker* existingOtherLinkerIt = joiningData.getBindingLinker(!leftSide); existingOtherLinkerIt; existingOtherLinkerIt = existingOtherLinkerIt->getNext()) {
joinedVariableMappingCreationRightLeftCopyingVecInsertionFunc(existingOtherLinkerIt, &tmpLinker, leftSide, bindingsMappedVec, bindingsMappedVec.getMemoryAllocationManager(), tmpJoinedBindings);
}
}
}
bindingsCardinalityCurrentLinker = batchLinker->getNextBatchBindingsCardinalityLinker(true);
}
COptimizedComplexVariableIndividualUpdateCardinalityLinker* cardinalityUpdateLinker = batchLinker->getNextCardinalityUpdateLinker(true);
while (cardinalityUpdateLinker) {
CXLinker* keyBindingLinker = leftKeyBindingLinker;
CXLinker* remainingBindingLinker = leftRemainingBindingLinker;
cint64 splitCount = leftSplitCount;
cint64 splitPos = leftSplitPos;
if (!leftSide) {
keyBindingLinker = rightKeyBindingLinker;
remainingBindingLinker = rightRemainingBindingLinker;
splitPos = rightSplitPos;
splitCount = rightSplitCount;
}
COptimizedComplexVariableJoiningWithRemainingHasher bindingHasher(cardinalityUpdateLinker->getUpdatedBindingsCardinalityLinker()->getBindings()->getBindingArray(), keyBindingLinker, remainingBindingLinker);
cint64 keyHashValue = bindingHasher.getHashValue();
uint uintHashValue = uint((keyHashValue >> (8 * sizeof(uint) - 1)) ^ keyHashValue);
cint64 mappedKeyIndex = uintHashValue % keyMappingSize;
if (!splitComputationMode || bindingHasher.getRemainingHashValue() % splitCount == splitPos) {
COptimizedComplexVariableJoiningHashMemoryManaged* joiningHash = joiningHashVector[mappedKeyIndex];
if (joiningHash) {
const COptimizedComplexVariableJoiningData& joiningData = joiningHash->value(bindingHasher);
bool leftSide = batchLinker->isLeftSide();
COptimizedComplexVariableIndividualBindings* varMapping = cardinalityUpdateLinker->getUpdatedBindingsCardinalityLinker()->getBindings();
COptimizedComplexVariableIndividualBindingsCardinality* newCardinalites = cardinalityUpdateLinker->getNewCardinality();
COptimizedComplexVariableIndividualBindingsCardinality* prevCardinalites = cardinalityUpdateLinker->getPreviousCardinality();
COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker* linker = joiningHash->createDataLinker();
COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker tmpLinker(varMapping, newCardinalites, prevCardinalites);
for (COptimizedComplexVariableJoiningBindingsCardinalitiesDataLinker* existingOtherLinkerIt = joiningData.getBindingLinker(!leftSide); existingOtherLinkerIt; existingOtherLinkerIt = existingOtherLinkerIt->getNext()) {
joinedVariableMappingCreationRightLeftCopyingVecInsertionFunc(existingOtherLinkerIt, &tmpLinker, leftSide, bindingsMappedVec, bindingsMappedVec.getMemoryAllocationManager(), tmpJoinedBindings);
}
}
}
cardinalityUpdateLinker = batchLinker->getNextCardinalityUpdateLinker(true);
}
batchLinker = batchLinker->getNext();
}
}
return bindingsMappedVec;
};
function& keyMappingBatchDataVector)> mapReducedConcurrentJoinCheckingComputation = [&](CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector& keyMappingBatchDataVector) -> void {
if (!keyMappingBatchDataVector.isEmpty()) {
QTime* stepTimer = nullptr;
if (mConfPerformanceLogging) {
stepTimer = new QTime();
stepTimer->start();
}
CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector mappedJoinedHashKeyVec = QtConcurrent::blockingMappedReduced>(keyMappingBatchDataVector, bindingBatchesHashingDirectJoiningMapFunc, bindingBatchesHashingJoinReduceFunc);
if (stepTimer) {
cint64 partitionsUsed = 0;
for (cint64 i = 0; i < joiningHashVector.size(); ++i) {
if (joiningHashVector[i]) {
++partitionsUsed;
}
}
QString x1 = QString("Checking step 1 of concurrent join computation for item %3 with %4/%1 tasks took %2 ms.").arg(joiningHashVector.size()).arg(stepTimer->elapsed()).arg(joiningItem->getComputationStepId()).arg(partitionsUsed);
if (mConfDirectPerformanceConsoleOutput) {
cout << x1.toLocal8Bit().constData() << endl;
} else {
LOG(INFO, getLogDomain(), x1, this);
}
delete stepTimer;
cint64 statMaxJoiningCount = 0;
cint64 statMaxProcessingTime = 0;
cint64 statTotalJoiningCount = 0;
cint64 statTotalProcessingTime = 0;
cint64 statUsedCount = 0;
for (cint64 i = 0; i < joiningHashVector.size(); ++i) {
COptimizedComplexVariableJoiningHashMemoryManaged* joiningHash = joiningHashVector[i];
if (joiningHash && joiningHash->mJoinedCount > 0) {
++statUsedCount;
statMaxJoiningCount = qMax(statMaxJoiningCount, joiningHash->mJoinedCount);
statMaxProcessingTime = qMax(statMaxProcessingTime, joiningHash->mProcessingTime);
statTotalJoiningCount += joiningHash->mJoinedCount;
statTotalProcessingTime += joiningHash->mProcessingTime;
}
}
cint64 averageJoiningCount = 0;
if (statUsedCount > 0) {
averageJoiningCount = statTotalJoiningCount / statUsedCount;
}
cint64 averageProcessingTime = 0;
if (statUsedCount > 0) {
averageProcessingTime = statTotalProcessingTime / statUsedCount;
}
QString x2 = QString("Checking step 1 of concurrent join computation for item %1 had in average %2 (max %3) created joins and took in average %4 (max %5) ms.").arg(joiningItem->getComputationStepId()).arg(averageJoiningCount).arg(statMaxJoiningCount).arg(averageProcessingTime).arg(statMaxProcessingTime);
if (mConfDirectPerformanceConsoleOutput) {
cout << x2.toLocal8Bit().constData() << endl;
} else {
LOG(INFO, getLogDomain(), x2, this);
}
}
if (mConfPerformanceLogging) {
stepTimer = new QTime();
stepTimer->start();
}
QVector mappedVarMappingParts = QtConcurrent::blockingMapped>(mappedJoinedHashKeyVec, bindingBatchesInsertingMapFunc);
if (stepTimer) {
cint64 partitionsUsed = 0;
for (cint64 i = 0; i < mappedJoinedHashKeyVec.size(); ++i) {
if (mappedJoinedHashKeyVec[i]) {
++partitionsUsed;
}
}
QString x3 = QString("Checking step 2 of concurrent join computation for item %3 with %4/%1 tasks took %2 ms.").arg(mappedJoinedHashKeyVec.size()).arg(stepTimer->elapsed()).arg(joiningItem->getComputationStepId()).arg(partitionsUsed);
if (mConfDirectPerformanceConsoleOutput) {
cout << x3.toLocal8Bit().constData() << endl;
} else {
LOG(INFO, getLogDomain(), x3, this);
}
delete stepTimer;
cint64 statMaxInsertionCount = 0;
cint64 statMaxProcessingTime = 0;
cint64 statTotalInsertionCount = 0;
cint64 statTotalProcessingTime = 0;
cint64 statUsedCount = 0;
for (cint64 i = 0; i < mappedJoinedHashKeyVec.size(); ++i) {
COptimizedComplexVariableIndividualMappingsMultiHashPart* joinedVarMappingMultiHashPart = joinedVarMapping->getMultiHashPart(i);
if (joinedVarMappingMultiHashPart && joinedVarMappingMultiHashPart->mStatInsertionCount > 0) {
++statUsedCount;
statMaxInsertionCount = qMax(statMaxInsertionCount, joinedVarMappingMultiHashPart->mStatInsertionCount);
statMaxProcessingTime = qMax(statMaxProcessingTime, joinedVarMappingMultiHashPart->mStatsProcessingTime);
statTotalInsertionCount += joinedVarMappingMultiHashPart->mStatInsertionCount;
statTotalProcessingTime += joinedVarMappingMultiHashPart->mStatsProcessingTime;
}
}
cint64 averageTotalInsertionCount = 0;
if (statUsedCount > 0) {
averageTotalInsertionCount = statTotalInsertionCount / statUsedCount;
}
cint64 averageProcessingTime = 0;
if (statUsedCount > 0) {
averageProcessingTime = statTotalProcessingTime / statUsedCount;
}
QString x4 = QString("Checking step 2 of concurrent join computation for item %1 had in average %2 (max %3) bindings insertions and took in average %4 (max %5) ms.").arg(joiningItem->getComputationStepId()).arg(averageTotalInsertionCount).arg(statMaxInsertionCount).arg(averageProcessingTime).arg(statMaxProcessingTime);
if (mConfDirectPerformanceConsoleOutput) {
cout << x4.toLocal8Bit().constData() << endl;
} else {
LOG(INFO, getLogDomain(), x4, this);
}
}
if (mConfPerformanceLogging) {
stepTimer = new QTime();
stepTimer->start();
}
bindingBatchesInsertingReduceFunc(mappedVarMappingParts);
CMemoryPool* mappedKeyBatchHashVecMemPools = keyMappingBatchDataVector.takeMemoryPools();
CMemoryPool* mappedJoinedHashVecMemPools = mappedJoinedHashKeyVec.takeMemoryPools();
CConcurrentTaskScheduler::run([=] {
clearMemoryPoolsFunc(mappedKeyBatchHashVecMemPools);
});
CConcurrentTaskScheduler::run([=] {
clearMemoryPoolsFunc(mappedJoinedHashVecMemPools);
});
keyMappingBatchDataVector.clear();
if (stepTimer) {
cint64 partitionsUsed = 0;
for (cint64 i = 0; i < mappedVarMappingParts.size(); ++i) {
if (mappedVarMappingParts[i]) {
++partitionsUsed;
}
}
QString x5 = QString("Checking step 3 of concurrent join computation for item %3 with %4/%1 tasks took %2 ms.").arg(mappedVarMappingParts.size()).arg(stepTimer->elapsed()).arg(joiningItem->getComputationStepId()).arg(partitionsUsed);
if (mConfDirectPerformanceConsoleOutput) {
cout << x5.toLocal8Bit().constData() << endl;
} else {
LOG(INFO, getLogDomain(), x5, this);
}
delete stepTimer;
}
}
};
auto checkingHandlingFunc = mapReducedConcurrentJoinInsertionComputation;
if (mConfCheckingSideDirectJoining) {
checkingHandlingFunc = mapReducedConcurrentJoinCheckingComputation;
}
//if (leftVarMapping->size() > mConfSamplingBasedJoinMappingSize || rightVarMapping->size() > mConfSamplingBasedJoinMappingSize) {
if (leftItemDep->isBatchProcessed(false) && rightItemDep->isBatchProcessed(false)) {
leftItemDep->loadNextBatch(false);
rightItemDep->loadNextBatch(false);
}
bool mappingProcessed = false;
bool requiresScheduling = false;
while (!joiningItem->isSamplingCompleted() && requiresMoreVariableMappingComputation(joiningItem, true) && (!leftItemDep->isBatchProcessed(false) || !rightItemDep->isBatchProcessed(false) || leftItemDep->loadNextBatch(false) || rightItemDep->loadNextBatch(false)) && !requiresScheduling) {
CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector keyMappingBatchDataVector(currentBatchSizeCount);
while (requiresMoreVariableMappingComputation(joiningItem, true) && ((leftItemDep->getBatchCurrentBindingsCardinalityLinker(false) && leftSampleInsertionCount < mConfSamplingBasedJoinMappingSize) || (rightItemDep->getBatchCurrentBindingsCardinalityLinker(false) && rightSampleInsertionCount < mConfSamplingBasedJoinMappingSize))) {
if (requiresMoreVariableMappingComputation(joiningItem, true) && leftItemDep->getBatchCurrentBindingsCardinalityLinker(false) && leftSampleInsertionCount < mConfSamplingBasedJoinMappingSize) {
mappingProcessed = true;
joiningHashSampleBatchingFunc(true, leftItemDep, leftSampleInsertionCount, keyMappingBatchDataVector);
}
if (requiresMoreVariableMappingComputation(joiningItem, true) && rightItemDep->getBatchCurrentBindingsCardinalityLinker(false) && rightSampleInsertionCount < mConfSamplingBasedJoinMappingSize) {
mappingProcessed = true;
joiningHashSampleBatchingFunc(false, rightItemDep, rightSampleInsertionCount, keyMappingBatchDataVector);
}
}
mapReducedConcurrentJoinInsertionComputation(keyMappingBatchDataVector);
if ((leftSampleInsertionCount >= mConfSamplingBasedJoinMappingSize || leftItemDep->isProcessingFinished(false)) && (rightSampleInsertionCount >= mConfSamplingBasedJoinMappingSize || rightItemDep->isProcessingFinished(false))) {
joiningItem->setSamplingCompleted(true);
}
if (leftItemDep->isBatchProcessed(false)) {
leftItemDep->loadNextBatch(false);
}
if (rightItemDep->isBatchProcessed(false)) {
rightItemDep->loadNextBatch(false);
}
if (!joiningItem->isSamplingCompleted() && (rightSampleInsertionCount >= mConfSamplingBasedJoinMappingSize || rightItemDep->isProcessingFinished(false))) {
if (leftItemDep->isBatchProcessed(false) && !leftItemDep->isProcessingFinished(false) && leftSampleInsertionCount < mConfSamplingBasedJoinMappingSize) {
requiresScheduling = true;
}
}
if (!joiningItem->isSamplingCompleted() && (leftSampleInsertionCount >= mConfSamplingBasedJoinMappingSize || leftItemDep->isProcessingFinished(false))) {
if (rightItemDep->isBatchProcessed(false) && !rightItemDep->isProcessingFinished(false) && rightSampleInsertionCount < mConfSamplingBasedJoinMappingSize) {
requiresScheduling = true;
}
}
}
if ((leftSampleInsertionCount >= mConfSamplingBasedJoinMappingSize || leftItemDep->isProcessingFinished(false)) && (rightSampleInsertionCount >= mConfSamplingBasedJoinMappingSize || rightItemDep->isProcessingFinished(false))) {
joiningItem->setSamplingCompleted(true);
}
if (joiningItem->isSamplingCompleted()) {
leftSampleKeyCount = atomicLeftSampleKeyCount;
rightSampleKeyCount = atomicRightSampleKeyCount;
}
bool leftInsertion = isSamplingLeftInsertionDecision(joiningItem);
if (joiningItem->isSamplingCompleted()) {
COptimizedComplexVariableCompositionItemDependence* insertingItemDep = leftItemDep;
COptimizedComplexVariableCompositionItemDependence* checkingItemDep = rightItemDep;
CXLinker* insertingKeyBindingLinker = leftKeyBindingLinker;
CXLinker* checkingKeyBindingLinker = rightKeyBindingLinker;
if (!leftInsertion) {
insertingItemDep = rightItemDep;
checkingItemDep = leftItemDep;
insertingKeyBindingLinker = rightKeyBindingLinker;
checkingKeyBindingLinker = leftKeyBindingLinker;
}
while (requiresMoreVariableMappingComputation(joiningItem, false) && (!insertingItemDep->isBatchProcessed() || insertingItemDep->loadNextBatch())) {
while (requiresMoreVariableMappingComputation(joiningItem, false) && insertingItemDep->getBatchCurrentBindingsCardinalityLinker(false)) {
mappingProcessed = true;
CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector keyMappingBindingsBatchDataVector(currentBatchSizeCount);
joiningHashBindingsInsertingCheckingBatchingFunc(leftInsertion, insertingItemDep, true, keyMappingBindingsBatchDataVector);
mapReducedConcurrentJoinInsertionComputation(keyMappingBindingsBatchDataVector);
}
// inserting remaining cardinality updates
while (requiresMoreVariableMappingComputation(joiningItem, false) && insertingItemDep->getBatchCurrentUpdatedCardinalityLinker(false)) {
mappingProcessed = true;
CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector keyMappingCardinalitiesBatchDataVector(currentBatchSizeCount);
joiningHashCardinalitiesInsertingCheckingBatchingFunc(leftInsertion, insertingItemDep, true, keyMappingCardinalitiesBatchDataVector);
mapReducedConcurrentJoinInsertionComputation(keyMappingCardinalitiesBatchDataVector);
}
}
if (insertingItemDep->isProcessingFinished()) {
while (requiresMoreVariableMappingComputation(joiningItem, false) && (!checkingItemDep->isBatchProcessed() || checkingItemDep->loadNextBatch())) {
// checking remaining mappings
while (requiresMoreVariableMappingComputation(joiningItem, false) && checkingItemDep->getBatchCurrentBindingsCardinalityLinker(false)) {
mappingProcessed = true;
CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector keyMappingBindingsBatchDataVector(currentBatchSizeCount);
joiningHashBindingsInsertingCheckingBatchingFunc(!leftInsertion, checkingItemDep, false, keyMappingBindingsBatchDataVector);
checkingHandlingFunc(keyMappingBindingsBatchDataVector);
}
// checking remaining cardinality updates
while (requiresMoreVariableMappingComputation(joiningItem, false) && checkingItemDep->getBatchCurrentUpdatedCardinalityLinker(false)) {
mappingProcessed = true;
CQtConcurrentVariableMappingsCompositionBaseBatchLinkerVector keyMappingCardinalitiesBatchDataVector(currentBatchSizeCount);
joiningHashCardinalitiesInsertingCheckingBatchingFunc(!leftInsertion, checkingItemDep, false, keyMappingCardinalitiesBatchDataVector);
checkingHandlingFunc(keyMappingCardinalitiesBatchDataVector);
}
}
}
}
joinedVarMapping->incCurrentUpdateId();
return mappingProcessed;
}
}; // end namespace Composition
}; // end namespace Answerer
}; // end namespace Reasoner
}; // end namespace Konclude