/*
* Copyright (C) 2013-2015, 2019 by the Konclude Developer Team.
*
* This file is part of the reasoning system Konclude.
* For details and support, see .
*
* Konclude is free software: you can redistribute it and/or modify
* it under the terms of version 3 of the GNU Lesser General Public
* License (LGPLv3) as published by the Free Software Foundation.
*
* Konclude is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU (Lesser) General Public License for more details.
*
* You should have received a copy of the GNU (Lesser) General Public
* License along with Konclude. If not, see .
*
*/
#include "CBackendRepresentativeMemoryCache.h"
#include "CBackendRepresentativeMemoryCacheWriter.h"
namespace Konclude {
namespace Reasoner {
namespace Kernel {
namespace Cache {
CBackendRepresentativeMemoryCache::CBackendRepresentativeMemoryCache(CConfiguration* config, QString threadIdentifierName, CWatchDog *watchDogThread) : CThread(threadIdentifierName, watchDogThread) {
mConfig = config;
mSlotLinker = nullptr;
mLastUpdatedSlotLinker = nullptr;
mReaderLinker = nullptr;
mNextIndiUpdateId = 1;
mNextNomConnUpdateId = 1;
mReaderSlotUpdateCount = 0;
mOntologyDataUpdateCount = 0;
mOntologyDataReleasedCount = 0;
mOntologyDataReleasedWhileNewCreationCount = 0;
mOntologyDataReleasedWhileSlotUpdateCount = 0;
mReaderSlotReleasedCount = 0;
mCheckingRemainingIncompletelyHandledCount = 0;
mEmptyWriteDataCount = 0;
mWriteDataCount = 0;
mStartWriteCollectCount = 0;
mNextWriteCollectCount = 0;
mTmpPropCutIndiArrayNeighboursHandlingDataHash = nullptr;
mConfInstallingDeterministicSameHandlingLargeDifference = 50;
mConfInstallingDeterministicSameHandlingLargeDifferenceReached = false;
mConfWaitIndividualLabelAssociationIndexed = CConfigDataReader::readConfigBoolean(config, "Konclude.Cache.RepresentativeBackendCache.WaitIndividualLabelAssociationIndexed", true);
mConfLateIndividualLabelAssociationIndexing = CConfigDataReader::readConfigBoolean(config, "Konclude.Cache.RepresentativeBackendCache.LateIndividualLabelAssociationIndexing", true);
mConfDebugWriteRepresentativeCache = CConfigDataReader::readConfigBoolean(config, "Konclude.Debugging.WriteDebuggingRepresentativeCacheContent", false);
mStatCollectStatistics = CConfigDataReader::readConfigBoolean(config, "Konclude.Cache.RepresentativeBackendCache.LogStorageStatistics", false);
mConfDirectUpdateSynchronization = !CConfigDataReader::readConfigBoolean(config, "Konclude.Cache.RepresentativeBackendCache.ThreadedUpdateSynchronization", true);
cint64 limitCount = CConfigDataReader::readConfigInteger(config, "Konclude.Cache.RepresentativeBackendCache.PendingWriteDataLimit", -1);
if (limitCount >= 0) {
mLimitRemainingWritePending = true;
mRemainingWritePendingSemaphore.release(limitCount);
} else {
mLimitRemainingWritePending = false;
}
mConfIncrementUpdateIdForDeterministicSameAsCompletion = CConfigDataReader::readConfigBoolean(config, "Konclude.Cache.RepresentativeBackendCache.IncrementUpdateIdForDeterministicSameAsCompletion", true);
mConfMinRequiredDeterministicSameMergedHandledInstallationPossiblitiesForNeighbourCompletion = CConfigDataReader::readConfigInteger(config, "Konclude.Cache.RepresentativeBackendCache.MinRequiredDeterministicSameMergedHandledInstallationPossiblitiesForNeighbourCompletion", 1);
mConfUnchangedDeterministicSameMergeUpdatesForDeterministicSameNeighbourCompletion = CConfigDataReader::readConfigInteger(config, "Konclude.Cache.RepresentativeBackendCache.UnchangedDeterministicSameMergeUpdatesForDeterministicSameNeighbourCompletion", 1);
mConfInstallingDeterministicSameHandlingLargeDifference = CConfigDataReader::readConfigInteger(config, "Konclude.Cache.RepresentativeBackendCache.TryingDeterministicSameMergedHandlingInstallationDifferenceCount", -1);
if (mConfInstallingDeterministicSameHandlingLargeDifference != -1) {
mConfInstallingDeterministicSameHandlingLargeDifferenceReached = true;
}
mConfBasicPrecomputationModeActivationUpdateMergesRatio = CConfigDataReader::readConfigInteger(config, "Konclude.Cache.RepresentativeBackendCache.BasicPrecomputationModeActivationUpdateMergesPercentRatio", 5) / 100.;
mConfMaxIncompletelyHandledIndividualsRetrievalCount = CConfigDataReader::readConfigInteger(config, "Konclude.Cache.RepresentativeBackendCache.MaxIncompletelyHandledIndividualsRetrievalCount", -1);
mConfMaxCacheDataUpdateWritingCount = CConfigDataReader::readConfigInteger(config, "Konclude.Cache.RepresentativeBackendCache.MaxCacheDataUpdateWritingCount", -1);
mConfUpdateRejectingIncompatibleIndividualAssociationsRatio = CConfigDataReader::readConfigDouble(config, "Konclude.Cache.RepresentativeBackendCache.RejectingUpdateIncompatibleIndividualAssociationsRatio", 1.0);
mConfUpdateRejectingIncompatiblePropagationCuttedIndividualLinkedNeighbourRatio = CConfigDataReader::readConfigDouble(config, "Konclude.Cache.RepresentativeBackendCache.RejectingUpdateIncompatiblePropagationCuttedIndividualLinkedNeighbourRatio", 1.0);
mConfUpdateRejectingIncompatiblePropagationCuttedIndividualLinkedNeighbourCount = CConfigDataReader::readConfigInteger(config, "Konclude.Cache.RepresentativeBackendCache.RejectingUpdateIncompatiblePropagationCuttedIndividualLinkedNeighbourCount", -1);
mConfInterpretUnchangedLabelsAsCompatible = CConfigDataReader::readConfigBoolean(config, "Konclude.Cache.RepresentativeBackendCache.InterpretUnchangedLabelsAsCompatibleIndividualAssociationUpdates", false);
mSlotUpdateWaitingMaxCount = CConfigDataReader::readConfigInteger(config, "Konclude.Cache.RepresentativeBackendCache.SlotUpdateWaitingIncreaseMaximumCount", 20);
mStatAddingNeighbourLinksAssociationUpdateCount = 0;
mStatUpdatedOrRemovedNeighbourLinksAssociationUpdateCount = 0;
mStatMaxSameAsMergedCount = 0;
mStatMaxLabelValueCount = 0;
for (cint64 i = 0; i < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_TYPE_COUNT; ++i) {
mStatLabelTypeCount[i] = 0;
mStatLabelTypeMaxValueCount[i] = 0;
mStatLabelTypeAllValueCount[i] = 0;
}
mStatLabelCount = 0;
mStatMaxAssociationUpdateCount = 0;
mStatCreatedNeighbourLinks = 0;
mStatMaxNeighbourLinksCount = 0;
mStatIncompatibleLabelNeighbourCompletionCount = 0;
mStatChangedLabelNeighbourCompletionCount = 0;
mStatNeighbourCompletionDetSameSuccededCount = 0;
mStatNeighbourCompletionDetSameChangedCount = 0;
mStatNeighbourCompletionDetSameIncompatibleCount = 0;
mStatNeighbourCompletionDetSameUnchangedCount = 0;
mStatDetSameAssociationInstallCount = 0;
mStatDetSameAssociationFailedCount = 0;
mStatDetSameAssociationDifferentUpdateIdFailedCount = 0;
mStatDetSameAssociationDifferentDestIdFailedCount = 0;
mStatDetSameAssociationIncompleteHandledDestFailedCount = 0;
mStatDetSameAssociationRepMergedDestFailedCount = 0;
mStatDetSameRepresentativeMergingCount = 0;
mSlotUpdateWaitingIncreaseCount = 1;
CMemoryAllocationManager* memMan = mContext.getMemoryAllocationManager();
//mIndiIdAssoDataHash = nullptr;
mOntologyIdentifierDataHash = CObjectParameterizingAllocator< CCACHINGHASH, CBackendRepresentativeMemoryCacheBaseContext* >::allocateAndConstructAndParameterize(memMan, &mContext);
startThread(QThread::HighestPriority);
}
CBackendRepresentativeMemoryCache::~CBackendRepresentativeMemoryCache() {
}
CCacheStatistics* CBackendRepresentativeMemoryCache::getCacheStatistics() {
return &mCacheStat;
}
void CBackendRepresentativeMemoryCache::deleteExpiredIndividualAssociationMemoryContexts(CBackendRepresentativeMemoryCacheOntologyData* ontologyData, CBackendRepresentativeMemoryCacheBaseContext* context) {
CBackendRepresentativeMemoryCacheContext* ontologyContext = ontologyData->getOntologyContext();
if (ontologyData->isBasicPrecomputationMode()) {
return;
}
cint64 statIndividualAssociationSeparateMemoryManagmentDeletionCount = 0;
cint64 statIndividualAssociationSeparateMemoryManagmentSlotReferredCheckingCount = 0;
CCACHINGMAP* recomputationIdReleasingIndividualAssociationContextMap = ontologyData->getRecomputationIdReleasingIndividualAssociationContextMap();
bool recomputationIdReleasingIndividualAssociationContextMapLocated = false;
CBackendRepresentativeMemoryCacheIndividualAssociationContext* releaseQueuedIndividualAssociationContextLinker = ontologyData->getReleaseQueuedIndividualAssociationContextLinker();
if (releaseQueuedIndividualAssociationContextLinker) {
cint64 ontologyIdentifier = ontologyData->getOntologyIdentifer();
CBackendRepresentativeMemoryCacheIndividualAssociationContext* prevReleaseQueuedIndividualAssociationContextLinker = nullptr;
for (CBackendRepresentativeMemoryCacheIndividualAssociationContext* releaseQueuedIndividualAssociationContextLinkerIt = releaseQueuedIndividualAssociationContextLinker; releaseQueuedIndividualAssociationContextLinkerIt; ) {
CBackendRepresentativeMemoryCacheIndividualAssociationContext* nextReleaseQueuedIndividualAssociationContextLinker = releaseQueuedIndividualAssociationContextLinkerIt->getNext();
CBackendRepresentativeMemoryCacheOntologyDataRecomputationReferenceLinker* firstRecomputationReferenceLinker = releaseQueuedIndividualAssociationContextLinkerIt->getFirstRecomputationReferenceLinker();
CBackendRepresentativeMemoryCacheOntologyDataRecomputationReferenceLinker* firstNextRecomputationReferenceLinker = firstRecomputationReferenceLinker->getNext();
CBackendRepresentativeMemoryCacheOntologyDataRecomputationReferenceLinker* lastRecomputationReferenceLinker = releaseQueuedIndividualAssociationContextLinkerIt->getLastRecomputationReferenceLinker();
bool noneOntologyDataUpdateIdSlotReferred = true;
cint64 maxReferedRecomputationId = CINT64_MIN;
cint64 minReferedRecomputationId = CINT64_MAX;
for (CBackendRepresentativeMemoryCacheOntologyDataRecomputationReferenceLinker* recomputationReferenceLinkerIt = lastRecomputationReferenceLinker; recomputationReferenceLinkerIt && noneOntologyDataUpdateIdSlotReferred && recomputationReferenceLinkerIt != firstNextRecomputationReferenceLinker; recomputationReferenceLinkerIt = recomputationReferenceLinkerIt->getNext()) {
cint64 ontologyDataUpdateId = recomputationReferenceLinkerIt->getOntologyDataUpdateId();
if (!recomputationReferenceLinkerIt->isOntologyDataActive()) {
mStatIndividualAssociationSeparateMemoryManagmentSlotReferredCheckingCount++;
statIndividualAssociationSeparateMemoryManagmentSlotReferredCheckingCount++;
bool ontologyDataUpdateIdSlotReferred = false;
for (CBackendRepresentativeMemoryCacheSlotItem* slotLinkerIt = mSlotLinker; slotLinkerIt && !ontologyDataUpdateIdSlotReferred; slotLinkerIt = slotLinkerIt->getNext()) {
CBackendRepresentativeMemoryCacheOntologyData* referredOntologyData = slotLinkerIt->getOntologyData(ontologyIdentifier);
cint64 ontologyUpdateId = referredOntologyData->getOntologyDataUpdateId();
if (ontologyUpdateId == ontologyDataUpdateId) {
ontologyDataUpdateIdSlotReferred = true;
}
}
if (ontologyDataUpdateIdSlotReferred) {
noneOntologyDataUpdateIdSlotReferred = false;
}
maxReferedRecomputationId = qMax(recomputationReferenceLinkerIt->getMaxUsedRecomputationId(), maxReferedRecomputationId);
minReferedRecomputationId = qMin(recomputationReferenceLinkerIt->getMinUsedRecomputationId(), minReferedRecomputationId);
} else {
noneOntologyDataUpdateIdSlotReferred = false;
}
}
if (noneOntologyDataUpdateIdSlotReferred) {
--mStatMemoryManagmentQueuedCheckingCount;
++mStatMemoryManagmentScheduledReleasingCount;
mStatIndividualAssociationSeparateMemoryManagmentUnreferredSlotCount++;
// move to recomputation id releasing map
if (!recomputationIdReleasingIndividualAssociationContextMap || !recomputationIdReleasingIndividualAssociationContextMapLocated) {
CCACHINGMAP* tmpMap = CObjectParameterizingAllocator< CCACHINGMAP , CContext* >::allocateAndConstructAndParameterize(ontologyContext->getMemoryAllocationManager(), ontologyContext);
if (recomputationIdReleasingIndividualAssociationContextMap) {
*tmpMap = *recomputationIdReleasingIndividualAssociationContextMap;
}
recomputationIdReleasingIndividualAssociationContextMap = tmpMap;
recomputationIdReleasingIndividualAssociationContextMapLocated = true;
ontologyData->setRecomputationIdReleasingIndividualAssociationContextMap(recomputationIdReleasingIndividualAssociationContextMap);
}
if (!prevReleaseQueuedIndividualAssociationContextLinker) {
ontologyData->setReleaseQueuedIndividualAssociationContextLinker(nextReleaseQueuedIndividualAssociationContextLinker);
} else {
prevReleaseQueuedIndividualAssociationContextLinker->setNext(nextReleaseQueuedIndividualAssociationContextLinker);
}
releaseQueuedIndividualAssociationContextLinkerIt->setNext(nullptr);
CBackendRepresentativeMemoryCacheIndividualAssociationContext* prevReleasingIndividualAssociationContextLinker = recomputationIdReleasingIndividualAssociationContextMap->value(maxReferedRecomputationId);
releaseQueuedIndividualAssociationContextLinkerIt = releaseQueuedIndividualAssociationContextLinkerIt->append(prevReleasingIndividualAssociationContextLinker);
recomputationIdReleasingIndividualAssociationContextMap->insert(maxReferedRecomputationId, releaseQueuedIndividualAssociationContextLinkerIt);
} else {
prevReleaseQueuedIndividualAssociationContextLinker = releaseQueuedIndividualAssociationContextLinkerIt;
}
releaseQueuedIndividualAssociationContextLinkerIt = nextReleaseQueuedIndividualAssociationContextLinker;
}
}
if (recomputationIdReleasingIndividualAssociationContextMap && !recomputationIdReleasingIndividualAssociationContextMap->isEmpty()) {
cint64 minValidRecompId = getMinimumSlotReferreringInstalledValidRecomputationId(ontologyData);
mLastMemoryContextDeletionMinValidRecompId = minValidRecompId;
cint64 statDeletedMemoryPoolsCount = 0;
cint64 statDeletedMemoryPoolSize = 0;
CMemoryPoolAllocationManager* memMan = context->getMemoryPoolAllocationManager();
while (!recomputationIdReleasingIndividualAssociationContextMap->isEmpty() && recomputationIdReleasingIndividualAssociationContextMap->constBegin().key() < minValidRecompId) {
CBackendRepresentativeMemoryCacheIndividualAssociationContext* unusedIndiAssContextLinker = recomputationIdReleasingIndividualAssociationContextMap->begin().value();
for (CBackendRepresentativeMemoryCacheIndividualAssociationContext* unusedIndiAssContextLinkerIt = unusedIndiAssContextLinker; unusedIndiAssContextLinkerIt; ) {
CBackendRepresentativeMemoryCacheIndividualAssociationContext* nextUnusedIndiAssContextLinker = unusedIndiAssContextLinkerIt->getNext();
CMemoryPoolContainer* memPoolContainer = unusedIndiAssContextLinkerIt->getMemoryPoolContainer();
CMemoryPool* memoryPools = memPoolContainer->getMemoryPools();
statDeletedMemoryPoolsCount += memPoolContainer->getMemoryPoolCount();
statDeletedMemoryPoolSize += memPoolContainer->getMemorySize();
if (mStatCollectStatistics) {
#ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
mIndiContextCountHash[unusedIndiAssContextLinkerIt->mIndiId]--;
#endif
}
--mStatMemoryManagmentScheduledReleasingCount;
//#ifndef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
memMan->releaseTemporaryMemoryPools(memoryPools);
//#endif
mStatIndividualAssociationSeparateMemoryManagmentDeletionCount++;
statIndividualAssociationSeparateMemoryManagmentDeletionCount++;
unusedIndiAssContextLinkerIt = nextUnusedIndiAssContextLinker;
}
if (!recomputationIdReleasingIndividualAssociationContextMapLocated) {
CCACHINGMAP* tmpMap = CObjectParameterizingAllocator< CCACHINGMAP, CContext* >::allocateAndConstructAndParameterize(ontologyContext->getMemoryAllocationManager(), ontologyContext);
if (recomputationIdReleasingIndividualAssociationContextMap) {
*tmpMap = *recomputationIdReleasingIndividualAssociationContextMap;
}
recomputationIdReleasingIndividualAssociationContextMap = tmpMap;
recomputationIdReleasingIndividualAssociationContextMapLocated = true;
ontologyData->setRecomputationIdReleasingIndividualAssociationContextMap(recomputationIdReleasingIndividualAssociationContextMap);
}
recomputationIdReleasingIndividualAssociationContextMap->erase(recomputationIdReleasingIndividualAssociationContextMap->begin());
}
if (statDeletedMemoryPoolsCount > 0) {
LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Deleted %1 memory pools for %2 individual associations, %5 with separate memory management remaining, checked %3 (overall %4) slot reference usages.")
.arg(statDeletedMemoryPoolsCount).arg(statIndividualAssociationSeparateMemoryManagmentDeletionCount).arg(statIndividualAssociationSeparateMemoryManagmentSlotReferredCheckingCount).arg(mStatIndividualAssociationSeparateMemoryManagmentSlotReferredCheckingCount).arg(mStatIndividualAssociationSeparateMemoryManagmentContextCreationCount - mStatIndividualAssociationSeparateMemoryManagmentDeletionCount), this);
}
if (mStatCollectStatistics) {
cint64 maxIndi = -1;
cint64 maxCount = -1;
for (QHash::const_iterator it = mIndiContextCountHash.constBegin(), itEnd = mIndiContextCountHash.constEnd(); it != itEnd; ++it) {
if (it.value() > maxCount) {
maxCount = it.value();
maxIndi = it.key();
}
}
LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("%1 individuals with separate memory management, max memory contexts (%2) for individual %3, %4 remaining deletion queued, %5 remaining deletion scheduled.")
.arg(mIndiContextCountHash.size()).arg(maxCount).arg(maxIndi).arg(mStatMemoryManagmentQueuedCheckingCount).arg(mStatMemoryManagmentScheduledReleasingCount), this);
}
}
}
void CBackendRepresentativeMemoryCache::queueIndividualAssociationMemoryContextDeletion(CBackendRepresentativeMemoryCacheIndividualAssociationContext* indiAssMemContext, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
ontologyData->addReleaseQueuedIndividualAssociationContextLinker(indiAssMemContext);
++mStatIndividualAssociationSeparateMemoryManagmentSlotReferredCheckingQueuingCount;
++mStatMemoryManagmentQueuedCheckingCount;
}
void CBackendRepresentativeMemoryCache::createReaderSlotUpdate(CBackendRepresentativeMemoryCacheOntologyData* ontologyData, CBackendRepresentativeMemoryCacheBaseContext* context) {
ontologyData->setMinimumValidRecomputationId(ontologyData->getNextUpdateMinimumValidRecomputationId());
ontologyData->setSlotUpdateIntegrated(true);
CMemoryPoolProvider* memProv = context->getMemoryPoolAllocationManager();
CBackendRepresentativeMemoryCacheSlotItem* slot = CObjectMemoryPoolAllocator::allocateAndConstructWithMemroyPool(memProv);
CMemoryPoolContainerAllocationManager slotMemMan(slot, memProv);
CMemoryAllocationManager* memMan = &slotMemMan;
CContextBase tmpContext(memMan);
CCACHINGHASH* ontologyIdentifierDataHash = CObjectParameterizingAllocator< CCACHINGHASH, CContext* >::allocateAndConstructAndParameterize(memMan, &tmpContext);
*ontologyIdentifierDataHash = *mOntologyIdentifierDataHash;
ontologyIdentifierDataHash->detach();
slot->setOntologyIdentifierDataHash(ontologyIdentifierDataHash);
mReaderSlotUpdateCount++;
mLastUpdatedSlotLinker = slot;
if (mSlotLinker) {
mSlotLinker->append(slot);
} else {
mSlotLinker = slot;
}
for (CCACHINGHASH::const_iterator it = ontologyIdentifierDataHash->constBegin(), itEnd = ontologyIdentifierDataHash->constEnd(); it != itEnd; ++it) {
CBackendRepresentativeMemoryCacheOntologyData* ontologyData = it.value();
ontologyData->incUsageCount();
}
CBackendRepresentativeMemoryCacheReader* readerLinkerIt = mReaderLinker;
while (readerLinkerIt) {
slot->incReader();
readerLinkerIt->updateSlot(slot);
readerLinkerIt = readerLinkerIt->getNext();
}
}
CBackendRepresentativeMemoryCacheOntologyData* CBackendRepresentativeMemoryCache::prepareOntologyDataUpdate(cint64 ontologyIdentifier, cint64 minIndiCount) {
CBackendRepresentativeMemoryCacheOntologyData*& ontologyData = (*mOntologyIdentifierDataHash)[ontologyIdentifier];
if (!ontologyData || ontologyData->isSlotUpdateIntegrated()) {
CBackendRepresentativeMemoryCacheOntologyData* prevOntologyData = ontologyData;
ontologyData = new CBackendRepresentativeMemoryCacheOntologyData(&mContext);
ontologyData->initOntologyData(ontologyIdentifier, !mConfLateIndividualLabelAssociationIndexing);
mOntologyDataUpdateCount++;
CBackendRepresentativeMemoryCacheContext* tmpContext = ontologyData->getTemporaryContext();
CMemoryAllocationManager* tmpMemMan = tmpContext->getMemoryAllocationManager();
CCACHINGHASH* slotSigLabelItemHashArray[CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_TYPE_COUNT];
for (cint64 i = 0; i < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_TYPE_COUNT; ++i) {
CCACHINGHASH* ontoSigLabelItemHash = CObjectParameterizingAllocator< CCACHINGHASH, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(tmpMemMan, tmpContext);
slotSigLabelItemHashArray[i] = ontoSigLabelItemHash;
ontologyData->setSignatureLabelItemHash(i, ontoSigLabelItemHash);
if (prevOntologyData) {
*ontoSigLabelItemHash = *prevOntologyData->getSignatureLabelItemHash(i);
ontoSigLabelItemHash->detach();
}
}
CCACHINGHASH* ontologyNominalIndiIdIndirectConnectionDataHash = CObjectParameterizingAllocator< CCACHINGHASH, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(tmpMemMan, tmpContext);
if (prevOntologyData) {
*ontologyNominalIndiIdIndirectConnectionDataHash = *prevOntologyData->getNominaIIndividualdIndirectConnectionDataHash();
ontologyNominalIndiIdIndirectConnectionDataHash->detach();
}
ontologyData->setNominaIIndividualdIndirectConnectionDataHash(ontologyNominalIndiIdIndirectConnectionDataHash);
if (!prevOntologyData) {
CBackendRepresentativeMemoryCacheOntologyContext* ontologyContext = new CBackendRepresentativeMemoryCacheOntologyContext(&mContext);
ontologyData->setOntologyContext(ontologyContext);
} else {
ontologyData->setOntologyContext(prevOntologyData->getOntologyContext());
}
if (!prevOntologyData) {
cint64 indiIdAssoDataVectorSize = qMax((cint64)1000, minIndiCount+1);
CBackendRepresentativeMemoryCacheIndividualAssociationData** indiIdAssoDataVector = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualAssociationData* >::allocateAndConstructArray(ontologyData->getOntologyContext()->getMemoryAllocationManager(), indiIdAssoDataVectorSize);
for (cint64 i = 0; i < indiIdAssoDataVectorSize; ++i) {
indiIdAssoDataVector[i] = nullptr;
}
ontologyData->setIndividualIdAssoiationDataVector(indiIdAssoDataVectorSize, indiIdAssoDataVector);
} else {
cint64 indiIdAssoDataVectorSize = prevOntologyData->getIndividualIdAssoiationDataVectorSize();
CBackendRepresentativeMemoryCacheIndividualAssociationData** indiIdAssoDataVector = prevOntologyData->getIndividualIdAssoiationDataVector();
ontologyData->setIndividualIdAssoiationDataVector(indiIdAssoDataVectorSize, indiIdAssoDataVector);
}
cint64 indiIdAssoDataVectorSize = ontologyData->getIndividualIdAssoiationDataVectorSize();
if (minIndiCount > indiIdAssoDataVectorSize) {
cint64 prevSize = indiIdAssoDataVectorSize;
CBackendRepresentativeMemoryCacheIndividualAssociationData** indiIdAssoDataVector = ontologyData->getIndividualIdAssoiationDataVector();
CBackendRepresentativeMemoryCacheIndividualAssociationData** prevIndiIdAssoDataVector = indiIdAssoDataVector;
indiIdAssoDataVectorSize = minIndiCount+1;
indiIdAssoDataVector = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualAssociationData* >::allocateAndConstructArray(ontologyData->getOntologyContext()->getMemoryAllocationManager(), indiIdAssoDataVectorSize);
for (cint64 i = 0; i < prevSize; ++i) {
indiIdAssoDataVector[i] = prevIndiIdAssoDataVector[i];
}
for (cint64 i = prevSize; i < indiIdAssoDataVectorSize; ++i) {
indiIdAssoDataVector[i] = nullptr;
}
ontologyData->setIndividualIdAssoiationDataVector(indiIdAssoDataVectorSize, indiIdAssoDataVector);
}
if (prevOntologyData) {
ontologyData->copyOntologyData(prevOntologyData);
}
CBackendRepresentativeMemoryCacheOntologyDataRecomputationReferenceLinker* linker = new CBackendRepresentativeMemoryCacheOntologyDataRecomputationReferenceLinker(ontologyData->getOntologyDataUpdateId());
ontologyData->setRecomputationReferenceLinker(linker);
ontologyData->incUsageCount();
if (prevOntologyData) {
prevOntologyData->decUsageCount();
if (prevOntologyData->getUsageCount() <= 0) {
++mOntologyDataReleasedCount;
++mOntologyDataReleasedWhileNewCreationCount;
CMemoryPoolAllocationManager* memMan = mContext.getMemoryPoolAllocationManager();
CMemoryPool* memoryPools = prevOntologyData->getTemporaryContext()->getMemoryPoolContainer()->getMemoryPools();
memMan->releaseTemporaryMemoryPools(memoryPools);
delete prevOntologyData;
}
}
}
return ontologyData;
}
void CBackendRepresentativeMemoryCache::cleanUnusedSlots(CBackendRepresentativeMemoryCacheBaseContext* context) {
CMemoryPoolAllocationManager* memMan = context->getMemoryPoolAllocationManager();
CBackendRepresentativeMemoryCacheSlotItem* slotLinkerIt = mSlotLinker;
CBackendRepresentativeMemoryCacheSlotItem* lastSlotLinker = nullptr;
while (slotLinkerIt) {
CBackendRepresentativeMemoryCacheSlotItem* tmpSlotLinker = nullptr;
if (!slotLinkerIt->hasCacheReaders()) {
tmpSlotLinker = slotLinkerIt;
}
CBackendRepresentativeMemoryCacheSlotItem* nextSlotLinker = slotLinkerIt->getNext();
if (tmpSlotLinker) {
if (!lastSlotLinker) {
mSlotLinker = nextSlotLinker;
} else {
lastSlotLinker->setNext(nextSlotLinker);
}
CCACHINGHASH* ontologyIdentifierDataHash = slotLinkerIt->getOntologyIdentifierDataHash();
for (CCACHINGHASH::const_iterator it = ontologyIdentifierDataHash->constBegin(), itEnd = ontologyIdentifierDataHash->constEnd(); it != itEnd; ++it) {
CBackendRepresentativeMemoryCacheOntologyData* ontologyData = it.value();
ontologyData->decUsageCount();
if (ontologyData->getUsageCount() <= 0) {
++mOntologyDataReleasedCount;
++mOntologyDataReleasedWhileSlotUpdateCount;
ontologyData->getRecomputationReferenceLinker()->setOntologyDataInactive();
CMemoryPool* memoryPools = ontologyData->getTemporaryContext()->getMemoryPoolContainer()->getMemoryPools();
memMan->releaseTemporaryMemoryPools(memoryPools);
delete ontologyData;
}
}
CMemoryPool* memoryPools = tmpSlotLinker->getMemoryPools();
memMan->releaseTemporaryMemoryPools(memoryPools);
++mReaderSlotReleasedCount;
}
else {
lastSlotLinker = slotLinkerIt;
}
slotLinkerIt = nextSlotLinker;
}
}
CBackendRepresentativeMemoryCacheReader* CBackendRepresentativeMemoryCache::createCacheReader() {
CBackendRepresentativeMemoryCacheReader* reader = new CBackendRepresentativeMemoryCacheReader();
mReaderSyncMutex.lock();
mReaderLinker = reader->append(mReaderLinker);
mReaderSyncMutex.unlock();
return reader;
}
CBackendRepresentativeMemoryCacheReader* CBackendRepresentativeMemoryCache::createOntologyFixedCacheReader(cint64 ontologyIdentifier) {
CBackendRepresentativeMemoryCacheReader* reader = new CBackendRepresentativeMemoryCacheReader();
mFixedOntologyIdentifierDataHashLock.lockForRead();
CBackendRepresentativeMemoryCacheOntologyData* ontologyData = mFixedOntologyIdentifierDataHash.value(ontologyIdentifier);
if (ontologyData) {
ontologyData->incUsageCount();
}
reader->fixOntologyData(ontologyData);
mFixedOntologyIdentifierDataHashLock.unlock();
ontologyData->waitIndividualLabelAssociationIndexed();
return reader;
}
CBackendRepresentativeMemoryCacheWriter* CBackendRepresentativeMemoryCache::createCacheWriter() {
CBackendRepresentativeMemoryCacheWriter* writer = new CBackendRepresentativeMemoryCacheWriter(this);
return writer;
}
CBackendRepresentativeMemoryCache* CBackendRepresentativeMemoryCache::writeCachedData(CBackendRepresentativeMemoryCacheWriteData* writeData, CMemoryPool* memoryPools) {
if (mStatCollectStatistics) {
mPendingUpdateCount.ref();
}
if (mLimitRemainingWritePending) {
mRemainingWritePendingSemaphore.acquire();
}
if (mConfDirectUpdateSynchronization) {
mDirectUpdateSyncMutex.lock();
CWriteBackendAssociationCachedEvent* procEvent = new CWriteBackendAssociationCachedEvent(writeData, memoryPools);
processCustomsEvents(CWriteBackendAssociationCachedEvent::EVENTTYPE, procEvent);
mDirectUpdateSyncMutex.unlock();
} else {
postEvent(new CWriteBackendAssociationCachedEvent(writeData, memoryPools));
}
return this;
}
void CBackendRepresentativeMemoryCache::installTemporaryCardinalities(CBackendRepresentativeMemoryCacheTemporaryCardinalityWriteDataLinker* tempCardWriteDataLinker, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
for (CBackendRepresentativeMemoryCacheTemporaryCardinalityWriteDataLinker* tempCardWriteDataLinkerIt = tempCardWriteDataLinker; tempCardWriteDataLinkerIt; tempCardWriteDataLinkerIt = tempCardWriteDataLinkerIt->getNext()) {
CBackendRepresentativeMemoryCacheTemporaryLabelWriteDataLinker* labelWriteDataLinker = tempCardWriteDataLinkerIt->getLabelWriteDataLinker();
CBackendRepresentativeMemoryLabelCacheItem* labelCacheItem = (CBackendRepresentativeMemoryLabelCacheItem*)labelWriteDataLinker->getTemporaryData();
CBackendRepresentativeMemoryCardinalityValueLinker* cardValueLinker = tempCardWriteDataLinkerIt->getCardinalityCacheValueLinker();
bool handled = false;
CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData* extensionData = (CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData*)labelCacheItem->getExtensionData(CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData::CARDINALITY_HASH);
if (extensionData) {
handled = true;
for (CBackendRepresentativeMemoryCardinalityValueLinker* cardValueLinkerIt = cardValueLinker; cardValueLinkerIt; cardValueLinkerIt = cardValueLinkerIt->getNext()) {
cint64 roleTag = cardValueLinkerIt->getRoleTag();
CBackendRepresentativeMemoryLabelCacheItemCardinalityData* roleCardData = extensionData->getRoleCardinalityData(roleTag);
if (roleCardData) {
roleCardData->updateExistentialMaxUsedCardinality(cardValueLinkerIt->getExistentialMaxUsedCardinality());
roleCardData->updateMinimumRestrictingCardinality(cardValueLinkerIt->getMinimalRestrictingCardinality());
} else {
handled = false;
}
}
}
if (!handled) {
CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData* newExtensionData = CObjectParameterizingAllocator< CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
newExtensionData->initCardinalityExtensionData();
for (CBackendRepresentativeMemoryCardinalityValueLinker* cardValueLinkerIt = cardValueLinker; cardValueLinkerIt; cardValueLinkerIt = cardValueLinkerIt->getNext()) {
cint64 roleTag = cardValueLinkerIt->getRoleTag();
cint64 cardCount = cardValueLinkerIt->getExistentialMaxUsedCardinality();
cint64 minRestCount = cardValueLinkerIt->getMinimalRestrictingCardinality();
CBackendRepresentativeMemoryLabelCacheItemCardinalityData* cardData = CObjectAllocator< CBackendRepresentativeMemoryLabelCacheItemCardinalityData >::allocateAndConstruct(context->getMemoryAllocationManager());
cardData->initCardinalityData(cardCount, minRestCount);
if (extensionData) {
CBackendRepresentativeMemoryLabelCacheItemCardinalityData* roleCardData = extensionData->getRoleCardinalityData(roleTag);
cardData->updateExistentialMaxUsedCardinality(roleCardData->getExistentialMaxUsedCardinality());
cardData->updateMinimumRestrictingCardinality(roleCardData->getMinimumRestrictingCardinality());
}
newExtensionData->setRoleCardinalityData(roleTag, cardData);
}
labelCacheItem->setExtensionData(CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData::CARDINALITY_HASH, newExtensionData);
}
}
}
void CBackendRepresentativeMemoryCache::installTemporaryLabels(CBackendRepresentativeMemoryCacheTemporaryLabelWriteDataLinker* tempLabelWriteDataLinker, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
for (CBackendRepresentativeMemoryCacheTemporaryLabelWriteDataLinker* tempLabelWriteDataLinkerIt = tempLabelWriteDataLinker; tempLabelWriteDataLinkerIt; tempLabelWriteDataLinkerIt = tempLabelWriteDataLinkerIt->getNext()) {
cint64 signature = tempLabelWriteDataLinkerIt->getSignature();
if (tempLabelWriteDataLinkerIt->getLabelType() == CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL) {
signature = 0;
for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = tempLabelWriteDataLinkerIt->getCacheValueLinker(); labelValueLinkerIt; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
CCacheValue cacheValue = labelValueLinkerIt->getCacheValue();
if (cacheValue.getCacheValueIdentifier() == CCacheValue::CACHE_VALUE_TAG_AND_TEMPORARY_ENTRY) {
CBackendRepresentativeMemoryCacheTemporaryLabelWriteDataLinker* tmpLabelWriteData = (CBackendRepresentativeMemoryCacheTemporaryLabelWriteDataLinker*)cacheValue.getIdentification();
CBackendRepresentativeMemoryLabelCacheItem* labelCacheItem = (CBackendRepresentativeMemoryLabelCacheItem*)tmpLabelWriteData->getTemporaryData();
cacheValue.initCacheValue(labelCacheItem->getCacheEntryID(), (cint64)labelCacheItem, CCacheValue::CACHE_VALUE_TAG_AND_ENTRY);
labelValueLinkerIt->setCacheValue(cacheValue);
}
signature = CBackendRepresentativeMemoryCacheUtilities::getSignatureExtensionByCacheValue(signature, cacheValue);
}
// update signature
tempLabelWriteDataLinkerIt->setSignature(signature);
}
CBackendRepresentativeMemoryLabelValueLinker* labelValueLinker = tempLabelWriteDataLinkerIt->getCacheValueLinker();
cint64 labelValueCount = tempLabelWriteDataLinkerIt->getCacheValueCount();
CBackendRepresentativeMemoryLabelSignatureResolveCacheItem& sigResolveItem = (*ontologyData->getSignatureLabelItemHash(tempLabelWriteDataLinkerIt->getLabelType()))[signature];
CBackendRepresentativeMemoryLabelCacheItem* refLabelItem = nullptr;
for (CBackendRepresentativeMemoryLabelCacheItem* labelItemLinkerIt = sigResolveItem.getLabelItems(); labelItemLinkerIt && !refLabelItem; labelItemLinkerIt = labelItemLinkerIt->getNext()) {
if (labelItemLinkerIt->getCacheValueCount() == labelValueCount) {
CCACHINGHASH* labelValueHash = labelItemLinkerIt->getTagCacheValueHash(false);
bool identical = true;
for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = labelValueLinker; labelValueLinkerIt && identical; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
const CCacheValue& cacheValue = labelValueLinkerIt->getCacheValue();
cint64 valueTag = cacheValue.getTag();
CBackendRepresentativeMemoryLabelValueLinker* labelValue = labelValueHash->value(valueTag);
if (!labelValue || labelValue->getCacheValue() != cacheValue) {
identical = false;
}
}
if (identical) {
refLabelItem = labelItemLinkerIt;
}
}
}
if (!refLabelItem) {
refLabelItem = CObjectParameterizingAllocator< CBackendRepresentativeMemoryLabelCacheItem, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
refLabelItem->initCacheEntry(signature, ontologyData->getNextEntryID(), (CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_TYPE)tempLabelWriteDataLinkerIt->getLabelType());
CCACHINGHASH* labelValueHash = refLabelItem->getTagCacheValueHash(true);
for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = labelValueLinker; labelValueLinkerIt; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
CBackendRepresentativeMemoryLabelValueLinker*& newValueLinker = (*labelValueHash)[labelValueLinkerIt->getCacheValue().getTag()];
if (!newValueLinker) {
CBackendRepresentativeMemoryLabelValueLinker* newValueLinker = CObjectAllocator< CBackendRepresentativeMemoryLabelValueLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
newValueLinker->initLabelValueLinker(labelValueLinkerIt->getCacheValue());
labelValueHash->insert(labelValueLinkerIt->getCacheValue().getTag(), newValueLinker);
refLabelItem->addCacheValueLinker(newValueLinker);
}
}
addCreatedLabelStatistics(tempLabelWriteDataLinkerIt->getLabelType(), refLabelItem, ontologyData);
sigResolveItem.appendLabelItem(refLabelItem);
}
if (!refLabelItem->isCompletelyHandled() && tempLabelWriteDataLinkerIt->isCompletelyHandled()) {
refLabelItem->setCompletelyHandled(true);
}
if (!refLabelItem->isCompletelySaturated() && tempLabelWriteDataLinkerIt->isCompletelySaturated()) {
refLabelItem->setCompletelySaturated(true);
}
if (!refLabelItem->hasNondeterministicElements() && tempLabelWriteDataLinkerIt->hasNondeterministicElements()) {
refLabelItem->setNondeterministicElements(true);
}
if (tempLabelWriteDataLinkerIt->getLabelType() == CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL) {
getNeighbourArrayRoleTagResolvingLabelExtensionData(refLabelItem, ontologyData);
}
tempLabelWriteDataLinkerIt->setTemporaryData(refLabelItem);
}
}
CBackendRepresentativeMemoryCache* CBackendRepresentativeMemoryCache::addCreatedLabelStatistics(cint64 labelType, CBackendRepresentativeMemoryLabelCacheItem* label, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
++mStatLabelCount;
mStatMaxLabelValueCount = qMax(mStatMaxLabelValueCount, label->getCacheValueCount());
mStatLabelTypeCount[labelType]++;
mStatLabelTypeMaxValueCount[labelType] = qMax(mStatLabelTypeMaxValueCount[labelType], label->getCacheValueCount());
mStatLabelTypeAllValueCount[labelType] += label->getCacheValueCount();
return this;
}
CBackendRepresentativeMemoryLabelCacheItem* CBackendRepresentativeMemoryCache::getExtendedLabel(cint64 labelType, CBackendRepresentativeMemoryLabelCacheItem* label, CCacheValue extendingCacheValue, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
if (label->getTagCacheValueHash(false)->contains(extendingCacheValue.getTag())) {
return label;
}
CBackendRepresentativeMemoryLabelValueLinker* labelValueLinker = label->getCacheValueLinker();
cint64 signature = label->getSignature() + qHash((qint64)extendingCacheValue.getTag());
cint64 newCount = label->getCacheValueCount() + 1;
// test with signature whether extended label already exists
CBackendRepresentativeMemoryLabelSignatureResolveCacheItem& sigResolveItem = (*ontologyData->getSignatureLabelItemHash(labelType))[signature];
for (CBackendRepresentativeMemoryLabelCacheItem* labelItemLinkerIt = sigResolveItem.getLabelItems(); labelItemLinkerIt; labelItemLinkerIt = labelItemLinkerIt->getNext()) {
if (labelItemLinkerIt->getCacheValueCount() == newCount) {
CCACHINGHASH* labelValueHash = labelItemLinkerIt->getTagCacheValueHash(false);
bool identical = true;
CBackendRepresentativeMemoryLabelValueLinker* labelValue = labelValueHash->value(extendingCacheValue.getTag());
if (!labelValue || labelValue->getCacheValue() != extendingCacheValue) {
identical = false;
}
for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = labelValueLinker; labelValueLinkerIt && identical; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
const CCacheValue& cacheValue = labelValueLinkerIt->getCacheValue();
cint64 valueTag = cacheValue.getTag();
CBackendRepresentativeMemoryLabelValueLinker* labelValue = labelValueHash->value(valueTag);
if (!labelValue || labelValue->getCacheValue() != cacheValue) {
identical = false;
}
}
if (identical) {
return labelItemLinkerIt;
}
}
}
// create new label item
CBackendRepresentativeMemoryLabelCacheItem* refLabelItem = CObjectParameterizingAllocator< CBackendRepresentativeMemoryLabelCacheItem, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
refLabelItem->initCacheEntry(signature, ontologyData->getNextEntryID(), (CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_TYPE)labelType);
CCACHINGHASH* labelValueHash = refLabelItem->getTagCacheValueHash(true);
for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = labelValueLinker; labelValueLinkerIt; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
const CCacheValue& cacheValue = labelValueLinkerIt->getCacheValue();
CBackendRepresentativeMemoryLabelValueLinker* newValueLinker = CObjectAllocator< CBackendRepresentativeMemoryLabelValueLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
newValueLinker->initLabelValueLinker(cacheValue);
labelValueHash->insert(cacheValue.getTag(), newValueLinker);
refLabelItem->addCacheValueLinker(newValueLinker);
}
CBackendRepresentativeMemoryLabelValueLinker* newValueLinker = CObjectAllocator< CBackendRepresentativeMemoryLabelValueLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
newValueLinker->initLabelValueLinker(extendingCacheValue);
labelValueHash->insert(extendingCacheValue.getTag(), newValueLinker);
refLabelItem->addCacheValueLinker(newValueLinker);
if (refLabelItem->getLabelType() == CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL) {
getNeighbourArrayRoleTagResolvingLabelExtensionData(refLabelItem, ontologyData);
}
if (refLabelItem->getLabelType() == CBackendRepresentativeMemoryLabelCacheItem::FULL_CONCEPT_SET_LABEL) {
CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData* extensionData = (CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData*)label->getExtensionData(CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData::CARDINALITY_HASH);
if (extensionData) {
CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData* newExtensionData = CObjectParameterizingAllocator< CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
newExtensionData->initCardinalityExtensionData();
CCACHINGHASH* newExtensionDataHash = newExtensionData->getRoleCardinalityDataHash();
for (CCACHINGHASH::const_iterator it = extensionData->getRoleCardinalityDataHash()->constBegin(), itEnd = extensionData->getRoleCardinalityDataHash()->constEnd(); it != itEnd; ++it) {
cint64 roleTag = it.key();
CBackendRepresentativeMemoryLabelCacheItemCardinalityData* cardData = it.value();
cint64 cardCount = cardData->getExistentialMaxUsedCardinality();
cint64 minRestCount = cardData->getMinimumRestrictingCardinality();
CBackendRepresentativeMemoryLabelCacheItemCardinalityData*& roleCardData = (*newExtensionDataHash)[roleTag];
if (!roleCardData) {
roleCardData = CObjectAllocator< CBackendRepresentativeMemoryLabelCacheItemCardinalityData >::allocateAndConstruct(context->getMemoryAllocationManager());
roleCardData->initCardinalityData(cardCount, minRestCount);
} else {
roleCardData->updateExistentialMaxUsedCardinality(cardCount);
roleCardData->updateMinimumRestrictingCardinality(minRestCount);
}
}
refLabelItem->setExtensionData(CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData::CARDINALITY_HASH, newExtensionData);
}
}
addCreatedLabelStatistics(labelType, refLabelItem, ontologyData);
sigResolveItem.appendLabelItem(refLabelItem);
return refLabelItem;
}
CBackendRepresentativeMemoryLabelCacheItem* CBackendRepresentativeMemoryCache::getReducedLabel(cint64 labelType, CBackendRepresentativeMemoryLabelCacheItem* label, function reduceCheckFunction, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
CBackendRepresentativeMemoryLabelValueLinker* labelValueLinker = label->getCacheValueLinker();
cint64 signature = 0;
cint64 newCount = 0;
for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = labelValueLinker; labelValueLinkerIt; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
CCacheValue cacheValue = labelValueLinkerIt->getCacheValue();
if (!reduceCheckFunction(cacheValue)) {
// extend signature/hash value
signature += qHash((qint64)cacheValue.getTag());
newCount++;
}
}
if (newCount == label->getCacheValueCount()) {
return label;
}
// test with signature whether reduced label already exists
CBackendRepresentativeMemoryLabelSignatureResolveCacheItem& sigResolveItem = (*ontologyData->getSignatureLabelItemHash(labelType))[signature];
for (CBackendRepresentativeMemoryLabelCacheItem* labelItemLinkerIt = sigResolveItem.getLabelItems(); labelItemLinkerIt; labelItemLinkerIt = labelItemLinkerIt->getNext()) {
if (labelItemLinkerIt->getCacheValueCount() == newCount) {
CCACHINGHASH* labelValueHash = labelItemLinkerIt->getTagCacheValueHash(false);
bool identical = true;
for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = labelValueLinker; labelValueLinkerIt && identical; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
const CCacheValue& cacheValue = labelValueLinkerIt->getCacheValue();
if (!reduceCheckFunction(cacheValue)) {
cint64 valueTag = cacheValue.getTag();
CBackendRepresentativeMemoryLabelValueLinker* labelValue = labelValueHash->value(valueTag);
if (!labelValue || labelValue->getCacheValue() != cacheValue) {
identical = false;
}
}
}
if (identical) {
return labelItemLinkerIt;
}
}
}
// create new label item
CBackendRepresentativeMemoryLabelCacheItem* refLabelItem = CObjectParameterizingAllocator< CBackendRepresentativeMemoryLabelCacheItem, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
refLabelItem->initCacheEntry(signature, ontologyData->getNextEntryID(), (CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_TYPE)labelType);
CCACHINGHASH* labelValueHash = refLabelItem->getTagCacheValueHash(true);
for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = labelValueLinker; labelValueLinkerIt; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
const CCacheValue& cacheValue = labelValueLinkerIt->getCacheValue();
if (!reduceCheckFunction(cacheValue)) {
CBackendRepresentativeMemoryLabelValueLinker* newValueLinker = CObjectAllocator< CBackendRepresentativeMemoryLabelValueLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
newValueLinker->initLabelValueLinker(cacheValue);
labelValueHash->insert(cacheValue.getTag(), newValueLinker);
refLabelItem->addCacheValueLinker(newValueLinker);
}
}
if (refLabelItem->getLabelType() == CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL) {
getNeighbourArrayRoleTagResolvingLabelExtensionData(refLabelItem, ontologyData);
}
addCreatedLabelStatistics(labelType, refLabelItem, ontologyData);
sigResolveItem.appendLabelItem(refLabelItem);
return refLabelItem;
}
CBackendRepresentativeMemoryLabelCacheItem* CBackendRepresentativeMemoryCache::getAdditionMergedLabel(cint64 labelType, CBackendRepresentativeMemoryLabelCacheItem* additionLabel, CBackendRepresentativeMemoryLabelCacheItem* associatedlabel, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
if (!additionLabel) {
return associatedlabel;
}
if (!associatedlabel) {
return additionLabel;
}
if (additionLabel == associatedlabel) {
return associatedlabel;
}
CBackendRepresentativeMemoryLabelValueLinker* additionLabelValueLinker = additionLabel->getCacheValueLinker();
CBackendRepresentativeMemoryLabelValueLinker* associatedLabelValueLinker = associatedlabel->getCacheValueLinker();
CCACHINGHASH* associatedLabelValueHash = associatedlabel->getTagCacheValueHash(false);
// check whether all values are already included
bool valuesAlreadyIncluded = true;
cint64 signature = associatedlabel->getSignature();
cint64 newCount = associatedlabel->getCacheValueCount();
for (CBackendRepresentativeMemoryLabelValueLinker* additionLabelValueLinkerIt = additionLabelValueLinker; additionLabelValueLinkerIt; additionLabelValueLinkerIt = additionLabelValueLinkerIt->getNext()) {
CCacheValue cacheValue = additionLabelValueLinkerIt->getCacheValue();
if (!associatedLabelValueHash || !associatedLabelValueHash->contains(cacheValue.getTag())) {
valuesAlreadyIncluded = false;
// extend signature/hash value
signature += qHash((qint64)cacheValue.getTag());
newCount++;
}
}
if (valuesAlreadyIncluded) {
return associatedlabel;
}
// test with signature whether extended label already exists
CBackendRepresentativeMemoryLabelSignatureResolveCacheItem& sigResolveItem = (*ontologyData->getSignatureLabelItemHash(labelType))[signature];
for (CBackendRepresentativeMemoryLabelCacheItem* labelItemLinkerIt = sigResolveItem.getLabelItems(); labelItemLinkerIt; labelItemLinkerIt = labelItemLinkerIt->getNext()) {
if (labelItemLinkerIt->getCacheValueCount() == newCount) {
CCACHINGHASH* labelValueHash = labelItemLinkerIt->getTagCacheValueHash(false);
bool identical = true;
for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = associatedLabelValueLinker; labelValueLinkerIt && identical; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
const CCacheValue& cacheValue = labelValueLinkerIt->getCacheValue();
cint64 valueTag = cacheValue.getTag();
CBackendRepresentativeMemoryLabelValueLinker* labelValue = labelValueHash->value(valueTag);
if (!labelValue || labelValue->getCacheValue() != cacheValue) {
identical = false;
}
}
for (CBackendRepresentativeMemoryLabelValueLinker* additionLabelValueLinkerIt = additionLabelValueLinker; additionLabelValueLinkerIt && identical; additionLabelValueLinkerIt = additionLabelValueLinkerIt->getNext()) {
const CCacheValue& cacheValue = additionLabelValueLinkerIt->getCacheValue();
cint64 valueTag = cacheValue.getTag();
CBackendRepresentativeMemoryLabelValueLinker* labelValue = labelValueHash->value(valueTag);
if (!labelValue || labelValue->getCacheValue() != cacheValue) {
identical = false;
}
}
if (identical) {
return labelItemLinkerIt;
}
}
}
// create new label item
CBackendRepresentativeMemoryLabelCacheItem* refLabelItem = CObjectParameterizingAllocator< CBackendRepresentativeMemoryLabelCacheItem, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
refLabelItem->initCacheEntry(signature, ontologyData->getNextEntryID(), (CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_TYPE)labelType);
CCACHINGHASH* labelValueHash = refLabelItem->getTagCacheValueHash(true);
for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = associatedLabelValueLinker; labelValueLinkerIt; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
CBackendRepresentativeMemoryLabelValueLinker* newValueLinker = CObjectAllocator< CBackendRepresentativeMemoryLabelValueLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
newValueLinker->initLabelValueLinker(labelValueLinkerIt->getCacheValue());
labelValueHash->insert(labelValueLinkerIt->getCacheValue().getTag(), newValueLinker);
refLabelItem->addCacheValueLinker(newValueLinker);
}
for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = additionLabelValueLinker; labelValueLinkerIt; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
CBackendRepresentativeMemoryLabelValueLinker*& newValueLinker = (*labelValueHash)[labelValueLinkerIt->getCacheValue().getTag()];
if (!newValueLinker) {
newValueLinker = CObjectAllocator< CBackendRepresentativeMemoryLabelValueLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
newValueLinker->initLabelValueLinker(labelValueLinkerIt->getCacheValue());
labelValueHash->insert(labelValueLinkerIt->getCacheValue().getTag(), newValueLinker);
refLabelItem->addCacheValueLinker(newValueLinker);
}
}
if (refLabelItem->getLabelType() == CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL) {
getNeighbourArrayRoleTagResolvingLabelExtensionData(refLabelItem, ontologyData);
}
if (refLabelItem->getLabelType() == CBackendRepresentativeMemoryLabelCacheItem::FULL_CONCEPT_SET_LABEL) {
CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData* extensionData1 = (CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData*)additionLabel->getExtensionData(CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData::CARDINALITY_HASH);
CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData* extensionData2 = (CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData*)associatedlabel->getExtensionData(CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData::CARDINALITY_HASH);
if (extensionData1 || extensionData2) {
CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData* newExtensionData = CObjectParameterizingAllocator< CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
newExtensionData->initCardinalityExtensionData();
CCACHINGHASH* newExtensionDataHash = newExtensionData->getRoleCardinalityDataHash();
for (CCACHINGHASH::const_iterator it = extensionData1->getRoleCardinalityDataHash()->constBegin(), itEnd = extensionData1->getRoleCardinalityDataHash()->constEnd(); it != itEnd; ++it) {
cint64 roleTag = it.key();
CBackendRepresentativeMemoryLabelCacheItemCardinalityData* cardData = it.value();
cint64 cardCount = cardData->getExistentialMaxUsedCardinality();
cint64 minRestCount = cardData->getMinimumRestrictingCardinality();
CBackendRepresentativeMemoryLabelCacheItemCardinalityData*& roleCardData = (*newExtensionDataHash)[roleTag];
if (!roleCardData) {
roleCardData = CObjectAllocator< CBackendRepresentativeMemoryLabelCacheItemCardinalityData >::allocateAndConstruct(context->getMemoryAllocationManager());
roleCardData->initCardinalityData(cardCount, minRestCount);
} else {
roleCardData->updateExistentialMaxUsedCardinality(cardCount);
roleCardData->updateMinimumRestrictingCardinality(minRestCount);
}
}
for (CCACHINGHASH::const_iterator it = extensionData2->getRoleCardinalityDataHash()->constBegin(), itEnd = extensionData2->getRoleCardinalityDataHash()->constEnd(); it != itEnd; ++it) {
cint64 roleTag = it.key();
CBackendRepresentativeMemoryLabelCacheItemCardinalityData* cardData = it.value();
cint64 cardCount = cardData->getExistentialMaxUsedCardinality();
cint64 minRestCount = cardData->getMinimumRestrictingCardinality();
CBackendRepresentativeMemoryLabelCacheItemCardinalityData*& roleCardData = (*newExtensionDataHash)[roleTag];
if (!roleCardData) {
roleCardData = CObjectAllocator< CBackendRepresentativeMemoryLabelCacheItemCardinalityData >::allocateAndConstruct(context->getMemoryAllocationManager());
roleCardData->initCardinalityData(cardCount, minRestCount);
} else {
roleCardData->updateExistentialMaxUsedCardinality(cardCount);
roleCardData->updateMinimumRestrictingCardinality(minRestCount);
}
}
refLabelItem->setExtensionData(CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData::CARDINALITY_HASH, newExtensionData);
}
}
addCreatedLabelStatistics(labelType, refLabelItem, ontologyData);
sigResolveItem.appendLabelItem(refLabelItem);
return refLabelItem;
}
bool CBackendRepresentativeMemoryCache::checkAssociationUsage(CBackendRepresentativeMemoryCacheTemporaryAssociationUseDataLinker* tempAssUseDataLinker, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
bool associationsUpdated = false;
for (CBackendRepresentativeMemoryCacheTemporaryAssociationUseDataLinker* tempAssUseDataLinkerIt = tempAssUseDataLinker; tempAssUseDataLinkerIt; tempAssUseDataLinkerIt = tempAssUseDataLinkerIt->getNext()) {
cint64 individualID = tempAssUseDataLinkerIt->getIndividualID();
cint64 usedUpateID = tempAssUseDataLinkerIt->getUsedAssociationUpdateId();
CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData = nullptr;
if (ontologyData->getIndividualIdAssoiationDataVector() && individualID < ontologyData->getIndividualIdAssoiationDataVectorSize()) {
associationData = ontologyData->getIndividualIdAssoiationDataVector()[individualID];
}
mCheckedIndiCount++;
if (!associationData || (associationData->getAssociationDataUpdateId() != usedUpateID && associationData->isCompletelyHandled())) {
associationsUpdated |= markRepresentativeReferencedIndividualAssociationIncompletelyHandled(individualID, associationData, ontologyData);
mCheckIncompatibleIndiCount++;
}
}
return associationsUpdated;
}
bool CBackendRepresentativeMemoryCache::markRepresentativeReferencedIndividualAssociationIncompletelyHandled(cint64 indiId, CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
bool associationsUpdated = false;
if (!associationData || !associationData->hasRepresentativeSameIndividualMerging()) {
DEBUG_HANDLING_STRING(QString("Marking %1 incompletly handled").arg(associationData->getAssociatedIndividualId()))
associationsUpdated = markIndividualAssociationIncompletelyHandled(indiId, associationData, ontologyData);
} else {
DEBUG_HANDLING_STRING(QString("Marking same representative individual %2 incompletly handled for %1").arg(associationData->getAssociatedIndividualId()).arg(associationData->getRepresentativeSameIndividualId()))
CBackendRepresentativeMemoryCacheIndividualAssociationData* repAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[associationData->getRepresentativeSameIndividualId()];
associationsUpdated = markIndividualAssociationIncompletelyHandled(repAssociationData->getRepresentativeSameIndividualId(), repAssociationData, ontologyData);
}
return associationsUpdated;
}
bool CBackendRepresentativeMemoryCache::markIndividualAssociationIncompletelyHandled(cint64 individualID, CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
bool associationsUpdated = false;
if (!associationData || associationData->isCompletelyHandled()) {
CBackendRepresentativeMemoryCacheIndividualAssociationData* locAssociationData = nullptr;
locAssociationData = createLocalizedIndividualAssociationData(individualID, associationData, ontologyData, false);
CBackendRepresentativeMemoryCacheContext* context = getIndividualAssociationDataMemoryContext(locAssociationData, ontologyData);
locAssociationData->setCacheUpdateId(mNextIndiUpdateId++);
#ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
locAssociationData->mDebugUpdateFunction = QString("markIndividualAssociationIncompletelyHandled");
locAssociationData->mDebugUpdateRecomputationId = mCurrentUpdateHandlingRecomputationId;
#endif
if (associationData && associationData->getNeighbourRoleSetHash()) {
locAssociationData->setNeighbourRoleSetHash(associationData->getNeighbourRoleSetHash());
locAssociationData->setRoleSetNeighbourArray(associationData->getRoleSetNeighbourArray());
}
locAssociationData->setCompletelyHandled(false);
associationsUpdated = true;
storeIndividualIncompletelyMarked(locAssociationData, !locAssociationData->isCompletelyHandled(), ontologyData);
setUpdatedIndividualAssociationData(individualID, locAssociationData, ontologyData);
}
return associationsUpdated;
}
bool CBackendRepresentativeMemoryCache::markIndividualAssociationCompletelyHandled(cint64 individualID, CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
bool associationsUpdated = false;
if (!associationData || !associationData->isCompletelyHandled()) {
CBackendRepresentativeMemoryCacheIndividualAssociationData* locAssociationData = nullptr;
locAssociationData = createLocalizedIndividualAssociationData(individualID, associationData, ontologyData, false);
CBackendRepresentativeMemoryCacheContext* context = getIndividualAssociationDataMemoryContext(locAssociationData, ontologyData);
locAssociationData->setCacheUpdateId(mNextIndiUpdateId++);
#ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
locAssociationData->mDebugUpdateFunction = QString("markIndividualAssociationCompletelyHandled");
locAssociationData->mDebugUpdateRecomputationId = mCurrentUpdateHandlingRecomputationId;
#endif
if (associationData && associationData->getNeighbourRoleSetHash()) {
locAssociationData->setNeighbourRoleSetHash(associationData->getNeighbourRoleSetHash());
locAssociationData->setRoleSetNeighbourArray(associationData->getRoleSetNeighbourArray());
}
locAssociationData->setCompletelyHandled(true);
associationsUpdated = true;
storeIndividualIncompletelyMarked(locAssociationData, !locAssociationData->isCompletelyHandled(), ontologyData);
setUpdatedIndividualAssociationData(individualID, locAssociationData, ontologyData);
}
return associationsUpdated;
}
bool CBackendRepresentativeMemoryCache::checkUpdateRejection(CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker* tempAssWriteDataLinker, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
bool rejection = false;
cint64 indiUpdateCount = 0;
cint64 incompatibleIndiUpdateCount = 0;
cint64 neighbourLinkCount = 0;
for (CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker* tempAssWriteDataLinkerIt = tempAssWriteDataLinker; tempAssWriteDataLinkerIt && !rejection; tempAssWriteDataLinkerIt = tempAssWriteDataLinkerIt->getNext()) {
CIndividual* individual = tempAssWriteDataLinkerIt->getIndividual();
cint64 individualID = tempAssWriteDataLinkerIt->getIndividualID();
cint64 sameAsIndividualId = tempAssWriteDataLinkerIt->getDeterministicSameIndividualId();
cint64 usedUpdateId = tempAssWriteDataLinkerIt->getUsedAssociationUpdateId();
if (tempAssWriteDataLinkerIt->getDeterministicSameIndividualId() == individualID) {
++indiUpdateCount;
CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData = nullptr;
if (ontologyData->getIndividualIdAssoiationDataVector() && individualID < ontologyData->getIndividualIdAssoiationDataVectorSize()) {
associationData = ontologyData->getIndividualIdAssoiationDataVector()[individualID];
}
if (associationData && associationData->getAssociationDataUpdateId() != usedUpdateId) {
++incompatibleIndiUpdateCount;
if (associationData->getLastPropagationCuttingUpdateId() > usedUpdateId) {
neighbourLinkCount += associationData->getNeighbourRoleSetHash()->getNeighbourCount();
cint64 totalIndiCount = ontologyData->getIndividualAssociationsCount();
if (mConfUpdateRejectingIncompatiblePropagationCuttedIndividualLinkedNeighbourCount >= 0 && neighbourLinkCount > mConfUpdateRejectingIncompatiblePropagationCuttedIndividualLinkedNeighbourCount) {
rejection = true;
}
double propCutLinkRatio = (double)neighbourLinkCount / (double)totalIndiCount;
if (propCutLinkRatio > mConfUpdateRejectingIncompatiblePropagationCuttedIndividualLinkedNeighbourRatio) {
rejection = true;
}
}
}
}
}
double incomUpdateRatio = (double)incompatibleIndiUpdateCount / (double)indiUpdateCount;
if (incomUpdateRatio > mConfUpdateRejectingIncompatibleIndividualAssociationsRatio) {
rejection = true;
}
return rejection;
}
bool CBackendRepresentativeMemoryCache::handleUpdateRejection(CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker* tempAssWriteDataLinker, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
for (CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker* tempAssWriteDataLinkerIt = tempAssWriteDataLinker; tempAssWriteDataLinkerIt; tempAssWriteDataLinkerIt = tempAssWriteDataLinkerIt->getNext()) {
CIndividual* individual = tempAssWriteDataLinkerIt->getIndividual();
cint64 individualID = tempAssWriteDataLinkerIt->getIndividualID();
if (tempAssWriteDataLinkerIt->isScheduledIndividual()) {
CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData = nullptr;
if (ontologyData->getIndividualIdAssoiationDataVector() && individualID < ontologyData->getIndividualIdAssoiationDataVectorSize()) {
associationData = ontologyData->getIndividualIdAssoiationDataVector()[individualID];
}
if (associationData) {
associationData->setCacheTouchId(mNextIndiUpdateId++);
}
}
}
return true;
}
bool CBackendRepresentativeMemoryCache::analyseDeterministicSameAsAssociationInstallation(CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker* tempAssWriteDataLinker, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
bool installable = false;
for (CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker* tempAssWriteDataLinkerIt = tempAssWriteDataLinker; tempAssWriteDataLinkerIt; tempAssWriteDataLinkerIt = tempAssWriteDataLinkerIt->getNext()) {
CIndividual* individual = tempAssWriteDataLinkerIt->getIndividual();
cint64 individualID = tempAssWriteDataLinkerIt->getIndividualID();
cint64 sameAsIndividualId = tempAssWriteDataLinkerIt->getDeterministicSameIndividualId();
cint64 usedUpdateId = tempAssWriteDataLinkerIt->getUsedAssociationUpdateId();
if (tempAssWriteDataLinkerIt->getDeterministicSameIndividualId() != individualID) {
CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData = nullptr;
if (ontologyData->getIndividualIdAssoiationDataVector() && individualID < ontologyData->getIndividualIdAssoiationDataVectorSize()) {
associationData = ontologyData->getIndividualIdAssoiationDataVector()[individualID];
}
if (!associationData || (associationData->getAssociationDataUpdateId() == usedUpdateId && !associationData->hasDeterministicSameIndividualMerging())) {
CDeterministicSameHandlingInstallationData& data = mDeterministicSameHandlingInstallationDataHash[sameAsIndividualId];
data.mIdPossibleInstallationSet.insert(individualID);
if (!associationData || !associationData->hasDeterministicSameIndividualMerging()) {
data.mIdFirstPossibleInstallationSet.insert(individualID);
DEBUG_HANDLING_STRING(QString("Adding %1 as first deterministic same handling installation possiblity to %2").arg(individualID).arg(sameAsIndividualId))
} else {
DEBUG_HANDLING_STRING(QString("Adding %1 as deterministic same handling installation possiblity to %2").arg(individualID).arg(sameAsIndividualId))
}
installable = true;
if (associationData) {
CBackendRepresentativeMemoryLabelCacheItem* detSameHandledLabel = associationData->getDeterministicMergedSameConsideredLabelCacheEntry();
if (detSameHandledLabel) {
for (CBackendRepresentativeMemoryLabelValueLinker* valueLinkerIt = detSameHandledLabel->getCacheValueLinker(); valueLinkerIt; valueLinkerIt = valueLinkerIt->getNext()) {
cint64 otherId = valueLinkerIt->getData().getTag();
data.mIdPossibleInstallationSet.insert(otherId);
DEBUG_HANDLING_STRING(QString("Adding %1 referenced from %3 as deterministic same handling installation possiblity to %2").arg(otherId).arg(sameAsIndividualId).arg(individualID))
}
}
}
}
}
}
return installable;
}
bool CBackendRepresentativeMemoryCache::installDeterministicSameAsAssociationUpdates(CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker* tempAssWriteDataLinker, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
bool associationsUpdated = false;
for (CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker* tempAssWriteDataLinkerIt = tempAssWriteDataLinker; tempAssWriteDataLinkerIt; tempAssWriteDataLinkerIt = tempAssWriteDataLinkerIt->getNext()) {
CIndividual* individual = tempAssWriteDataLinkerIt->getIndividual();
cint64 individualID = tempAssWriteDataLinkerIt->getIndividualID();
cint64 sameAsIndividualId = tempAssWriteDataLinkerIt->getDeterministicSameIndividualId();
cint64 usedUpdateId = tempAssWriteDataLinkerIt->getUsedAssociationUpdateId();
if (tempAssWriteDataLinkerIt->getDeterministicSameIndividualId() != individualID) {
ontologyData->setSameIndividualsMergings(true);
CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData = nullptr;
if (ontologyData->getIndividualIdAssoiationDataVector() && individualID < ontologyData->getIndividualIdAssoiationDataVectorSize()) {
associationData = ontologyData->getIndividualIdAssoiationDataVector()[individualID];
}
if (associationData) {
associationData->setCacheTouchId(mNextIndiUpdateId++);
}
DEBUG_HANDLING_STRING(QString("Checking determinitic same reference update for %1").arg(individualID))
if (!associationData || tempAssWriteDataLinkerIt->getDeterministicSameIndividualId() != associationData->getDeterministicSameIndividualId()) {
CBackendRepresentativeMemoryCacheIndividualAssociationData* detSameAsAssociationData = nullptr;
if (ontologyData->getIndividualIdAssoiationDataVector() && sameAsIndividualId < ontologyData->getIndividualIdAssoiationDataVectorSize()) {
detSameAsAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[sameAsIndividualId];
}
//if (detSameAsAssociationData && detSameAsAssociationData->hasDeterministicSameIndividualMerging()) {
// sameAsIndividualId = detSameAsAssociationData->getDeterministicSameIndividualId();
// detSameAsAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[sameAsIndividualId];
//}
if (detSameAsAssociationData) {
CBackendRepresentativeMemoryLabelCacheItem* detSameHandledLabel = detSameAsAssociationData->getDeterministicMergedSameConsideredLabelCacheEntry();
//if (!detSameAsAssociationData->hasRepresentativeSameIndividualMerging() && detSameAsAssociationData->isCompletelyHandled() && (!associationData || associationData->getAssociationDataUpdateId() == usedUpdateId || associationData->hasDeterministicSameIndividualMerging())) {
if (detSameHandledLabel && detSameHandledLabel->hasCachedTagValue(individualID)) {
associationsUpdated = installDeterministicSameAsAssociationUpdate(associationData, individualID, detSameAsAssociationData, sameAsIndividualId, ontologyData);
mStatDetSameAssociationInstallCount++;
} else {
DEBUG_HANDLING_STRING(QString("Determinitic same reference update for %1 failed").arg(individualID))
++mStatDetSameAssociationFailedCount;
if (detSameAsAssociationData->hasRepresentativeSameIndividualMerging()) {
++mStatDetSameAssociationRepMergedDestFailedCount;
} else if (!detSameAsAssociationData->isCompletelyHandled()) {
++mStatDetSameAssociationIncompleteHandledDestFailedCount;
} else {
if (associationData && associationData->getAssociationDataUpdateId() == usedUpdateId) {
++mStatDetSameAssociationDifferentUpdateIdFailedCount;
}
if (associationData && !associationData->hasDeterministicSameIndividualMerging()) {
++mStatDetSameAssociationDifferentDestIdFailedCount;
}
}
if (associationData && detSameAsAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL) != associationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL)) {
mTmpDetSameMergingCompletionReferenceHash.insert(individualID, sameAsIndividualId);
} else if (!associationData || !associationData->isCompletelyHandled()) {
markIndividualAssociationCompletelyHandled(individualID, associationData, ontologyData);
}
markRepresentativeReferencedIndividualAssociationIncompletelyHandled(sameAsIndividualId, detSameAsAssociationData, ontologyData);
}
}
} else if (associationData && associationData->getAssociationDataUpdateId() == usedUpdateId && !associationData->isCompletelyHandled()) {
markIndividualAssociationCompletelyHandled(individualID, associationData, ontologyData);
}
}
}
return associationsUpdated;
}
bool CBackendRepresentativeMemoryCache::checkRequiresDeterministicSameAsAssociationUpdateInstallation(CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData, cint64 individualID, CBackendRepresentativeMemoryCacheIndividualAssociationData* detSameAsAssociationData, cint64 sameAsIndividualId, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
if (associationData->getDeterministicSameIndividualId() != sameAsIndividualId) {
return true;
}
if (associationData->getRepresentativeSameIndividualId() != detSameAsAssociationData->getRepresentativeSameIndividualId()) {
return true;
}
if (associationData->getRoleSetNeighbourArray() != detSameAsAssociationData->getRoleSetNeighbourArray()) {
return true;
}
if (!associationData->isCompletelyHandled()) {
return true;
}
for (cint64 i = 0; i < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_ASSOCIATABLE_TYPE_COUNT; ++i) {
if (associationData->getLabelCacheEntry(i) != detSameAsAssociationData->getLabelCacheEntry(i)) {
return true;
}
}
return false;
}
bool CBackendRepresentativeMemoryCache::installDeterministicSameAsAssociationUpdate(CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData, cint64 individualID, CBackendRepresentativeMemoryCacheIndividualAssociationData* detSameAsAssociationData, cint64 sameAsIndividualId, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
DEBUG_HANDLING_STRING(QString("Updating deterministic same handled references for %1 to %2").arg(associationData->getAssociatedIndividualId()).arg(sameAsIndividualId))
CBackendRepresentativeMemoryCacheIndividualAssociationData* locAssociationData = createLocalizedIndividualAssociationData(individualID, associationData, ontologyData, false);
CBackendRepresentativeMemoryCacheContext* context = getIndividualAssociationDataMemoryContext(locAssociationData, ontologyData);
#ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
locAssociationData->mDebugUpdateFunction = QString("installDeterministicSameAsAssociationUpdate");
locAssociationData->mDebugUpdateRecomputationId = mCurrentUpdateHandlingRecomputationId;
#endif
locAssociationData->setCacheUpdateId(mNextIndiUpdateId++);
locAssociationData->setDeterministicSameIndividualId(sameAsIndividualId);
locAssociationData->setRepresentativeSameIndividualId(detSameAsAssociationData->getRepresentativeSameIndividualId());
if (locAssociationData->hasRepresentativeSameIndividualMerging()) {
if (!associationData || !associationData->hasRepresentativeSameIndividualMerging()) {
mStatDetSameRepresentativeMergingCount++;
ontologyData->incIndividualAssociationMergingCount();
}
}
locAssociationData->setRoleSetNeighbourArray(detSameAsAssociationData->getRoleSetNeighbourArray());
locAssociationData->setNeighbourRoleSetHash(detSameAsAssociationData->getNeighbourRoleSetHash());
locAssociationData->setCompletelyHandled(true);
locAssociationData->setDeterministicConceptSetLabelCacheEntry(nullptr);
for (cint64 i = 0; i < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_ASSOCIATABLE_TYPE_COUNT; ++i) {
locAssociationData->setLabelCacheEntry(i, detSameAsAssociationData->getLabelCacheEntry(i));
updateIndexedAssociationCount(locAssociationData, associationData, i, ontologyData);
}
CBackendRepresentativeMemoryLabelCacheItem* mergedSameAsLabelItem = nullptr;
if (associationData->hasDeterministicSameIndividualMerging() && associationData->getDeterministicSameIndividualId() != sameAsIndividualId) {
CBackendRepresentativeMemoryLabelCacheItem* detSameAsLabelItem = detSameAsAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL);
CBackendRepresentativeMemoryLabelCacheItem* prevSameAsLabelItem = associationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL);
mergedSameAsLabelItem = getAdditionMergedLabel(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL, prevSameAsLabelItem, detSameAsLabelItem, ontologyData);
if (mergedSameAsLabelItem != detSameAsLabelItem) {
locAssociationData->setLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL, mergedSameAsLabelItem);
updateIndexedAssociationCount(locAssociationData, detSameAsLabelItem, CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL, ontologyData);
mTmpDetSameMergingCompletionReferenceHash.insert(individualID, sameAsIndividualId);
}
}
storeIndividualIncompletelyMarked(locAssociationData, !locAssociationData->isCompletelyHandled(), ontologyData);
setUpdatedIndividualAssociationData(individualID, locAssociationData, ontologyData);
return true;
}
bool CBackendRepresentativeMemoryCache::installAssociationUpdates(CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker* tempAssWriteDataLinker, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
bool associationsUpdated = false;
for (CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker* tempAssWriteDataLinkerIt = tempAssWriteDataLinker; tempAssWriteDataLinkerIt; tempAssWriteDataLinkerIt = tempAssWriteDataLinkerIt->getNext()) {
cint64 individualID = tempAssWriteDataLinkerIt->getIndividualID();
DEBUG_HANDLING_STRING(QString("Handling update data for individual %1").arg(individualID))
if (tempAssWriteDataLinkerIt->getDeterministicSameIndividualId() == individualID) {
cint64 usedUpdateId = tempAssWriteDataLinkerIt->getUsedAssociationUpdateId();
CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData = nullptr;
if (ontologyData->getIndividualIdAssoiationDataVector() && individualID < ontologyData->getIndividualIdAssoiationDataVectorSize()) {
associationData = ontologyData->getIndividualIdAssoiationDataVector()[individualID];
}
if (associationData && associationData->hasDeterministicSameIndividualMerging()) {
if (!associationData->isCompletelyHandled()) {
markIndividualAssociationCompletelyHandled(individualID, associationData, ontologyData);
}
individualID = associationData->getDeterministicSameIndividualId();
associationData = ontologyData->getIndividualIdAssoiationDataVector()[individualID];
DEBUG_HANDLING_STRING(QString("Resolving update data for individual %1 to %2").arg(tempAssWriteDataLinkerIt->getIndividualID()).arg(individualID))
}
associationsUpdated |= installAssociationUpdate(individualID, associationData, tempAssWriteDataLinkerIt, ontologyData);
}
}
return associationsUpdated;
}
CBackendRepresentativeMemoryCacheIndividualAssociationData* CBackendRepresentativeMemoryCache::createLocalizedIndividualAssociationData(cint64 individualID, CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData, CBackendRepresentativeMemoryCacheOntologyData* ontologyData, bool allowSeparatedManagement, bool incrementUpdateId) {
CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
bool separatedMemoryManagementReusing = false;
bool separatedMemoryManagementNew = false;
CBackendRepresentativeMemoryCacheIndividualAssociationContext* existingIndiAssContext = nullptr;
if (associationData) {
existingIndiAssContext = associationData->getIndividualAssociationMemoryContext();
}
if (associationData && existingIndiAssContext) {
CMemoryPoolContainer* memPoolContainer = existingIndiAssContext->getMemoryPoolContainer();
if (existingIndiAssContext->getIndividualAssociationDataUsageCount() < 50 && memPoolContainer->getMemoryPoolCount() < 10 && memPoolContainer->getMemorySize() <= 10000000) {
separatedMemoryManagementReusing = true;
}
}
if (associationData && existingIndiAssContext && !allowSeparatedManagement) {
separatedMemoryManagementReusing = true;
}
if (!separatedMemoryManagementReusing && associationData && existingIndiAssContext && allowSeparatedManagement) {
separatedMemoryManagementNew = true;
}
if (!separatedMemoryManagementReusing && !separatedMemoryManagementNew && associationData && associationData->getAssociationDataUpdateId() > 10 && ontologyData->isFirstIncompletelyHandledIndividualsRetrieved() && allowSeparatedManagement) {
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* roleSetNeighArray = associationData->getRoleSetNeighbourArray();
cint64 neighbourCount = 0;
if (roleSetNeighArray) {
CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* roleSetNeighArrayInd = roleSetNeighArray->getIndexData();
if (roleSetNeighArrayInd) {
for (cint64 i = 0; i < roleSetNeighArrayInd->getArraySize(); ++i) {
neighbourCount += roleSetNeighArray->at(i).getIndividualCount();
}
}
}
if (neighbourCount > 100) {
separatedMemoryManagementNew = true;
}
}
if (!separatedMemoryManagementReusing && !separatedMemoryManagementNew && associationData && associationData->getAssociationDataUpdateId() > 20 && ontologyData->isFirstIncompletelyHandledIndividualsRetrieved() && !existingIndiAssContext) {
separatedMemoryManagementNew = true;
}
CBackendRepresentativeMemoryCacheIndividualAssociationData* locAssociationData = nullptr;
if (separatedMemoryManagementReusing) {
locAssociationData = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualAssociationData >::allocateAndConstruct(existingIndiAssContext->getMemoryAllocationManager());
locAssociationData->setIndividualAssociationMemoryContext(existingIndiAssContext);
existingIndiAssContext->incIndividualAssociationDataUsageCount();
existingIndiAssContext->setLastRecomputationReferenceLinker(ontologyData->getRecomputationReferenceLinker());
++mStatIndividualAssociationSeparateMemoryManagmentContextReuseCount;
} else if (separatedMemoryManagementNew) {
CBackendRepresentativeMemoryCacheIndividualAssociationContext* indiAssContext = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualAssociationContext, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
if (mStatCollectStatistics) {
#ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
indiAssContext->mIndiId = individualID;
mIndiContextCountHash[individualID]++;
#endif
}
locAssociationData = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualAssociationData >::allocateAndConstruct(indiAssContext->getMemoryAllocationManager());
locAssociationData->setIndividualAssociationMemoryContext(indiAssContext);
indiAssContext->incIndividualAssociationDataUsageCount();
indiAssContext->setFirstRecomputationReferenceLinker(ontologyData->getLastActiveRecomputationReferenceLinker());
indiAssContext->setLastRecomputationReferenceLinker(ontologyData->getRecomputationReferenceLinker());
++mStatIndividualAssociationSeparateMemoryManagmentContextCreationCount;
if (associationData && existingIndiAssContext) {
indiAssContext->setPreviousMemoryManagementCount(existingIndiAssContext->getPreviousMemoryManagementCount() + 1);
existingIndiAssContext->setLastRecomputationReferenceLinker(ontologyData->getRecomputationReferenceLinker());
queueIndividualAssociationMemoryContextDeletion(existingIndiAssContext, ontologyData);
}
} else {
locAssociationData = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualAssociationData >::allocateAndConstruct(context->getMemoryAllocationManager());
++mStatIndividualAssociationWithoutSeparateMemoryManagmentCount;
}
if (associationData) {
locAssociationData->initAssociationData(associationData, incrementUpdateId);
} else {
locAssociationData->initAssociationData(individualID);
}
return locAssociationData;
}
CBackendRepresentativeMemoryCacheContext* CBackendRepresentativeMemoryCache::getIndividualAssociationDataMemoryContext(CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData, CBackendRepresentativeMemoryCacheOntologyData* ontologyData, bool* requiresDataCopying) {
CBackendRepresentativeMemoryCacheIndividualAssociationContext* context = associationData->getIndividualAssociationMemoryContext();
if (context) {
if (requiresDataCopying && context->getIndividualAssociationDataUsageCount() <= 1 && context->getPreviousMemoryManagementCount() > 0) {
*requiresDataCopying = true;
}
return context;
}
return ontologyData->getOntologyContext();
}
bool CBackendRepresentativeMemoryCache::isCacheValueRoleInverse(const CCacheValue& cacheValue) {
CCacheValue::CACHEVALUEIDENTIFIER cacheValueIdentifier = cacheValue.getCacheValueIdentifier();
return cacheValueIdentifier == CCacheValue::CACHEVALTAGANDINVERSEDROLE || cacheValueIdentifier == CCacheValue::CACHEVALTAGANDINVERSEDASSERTEDROLE || cacheValueIdentifier == CCacheValue::CACHEVALTAGANDINVERSED_NOMINAL_CONNECTED_ROLE
|| cacheValueIdentifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_INVERSEDROLE || cacheValueIdentifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_INVERSEDASSERTEDROLE || cacheValueIdentifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_INVERSED_NOMINAL_CONNECTED_ROLE;
}
bool CBackendRepresentativeMemoryCache::isCacheValueRoleNondeterministic(const CCacheValue& cacheValue) {
CCacheValue::CACHEVALUEIDENTIFIER cacheValueIdentifier = cacheValue.getCacheValueIdentifier();
return cacheValueIdentifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_ROLE || cacheValueIdentifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_INVERSEDROLE || cacheValueIdentifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_ASSERTEDROLE
|| cacheValueIdentifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_INVERSEDASSERTEDROLE || cacheValueIdentifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_NOMINAL_CONNECTED_ROLE || cacheValueIdentifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_INVERSED_NOMINAL_CONNECTED_ROLE;
}
bool CBackendRepresentativeMemoryCache::isRoleNeighbourLinkLabelItemCompatibility(CBackendRepresentativeMemoryLabelCacheItem* itemPrev, CBackendRepresentativeMemoryLabelCacheItem* itemNew) {
if (itemPrev->getCacheValueCount() != itemNew->getCacheValueCount()) {
return false;
}
bool sameRolesCompatible = true;
for (CBackendRepresentativeMemoryLabelValueLinker* linkerIt = itemNew->getCacheValueLinker(); linkerIt && sameRolesCompatible; linkerIt = linkerIt->getNext()) {
cint64 tag = linkerIt->getCacheValue().getTag();
if (!itemPrev->getTagCacheValueHash(false)->contains(tag)) {
sameRolesCompatible = false;
} else {
const CCacheValue& prevCacheValue = itemPrev->getTagCacheValueHash(false)->value(tag)->getCacheValue();
const CCacheValue& newCacheValue = linkerIt->getCacheValue();
if (isCacheValueRoleInverse(prevCacheValue) != isCacheValueRoleInverse(newCacheValue)) {
sameRolesCompatible = false;
}
bool prevNondeterministic = isCacheValueRoleNondeterministic(prevCacheValue);
bool newNondeterministic = isCacheValueRoleNondeterministic(newCacheValue);
if (!prevNondeterministic && newNondeterministic) {
sameRolesCompatible = false;
}
}
}
return sameRolesCompatible;
}
bool CBackendRepresentativeMemoryCache::installAssociationUpdate(cint64 individualID, CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData, CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker* tempAssWriteDataLinkerIt, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
bool associationsUpdated = false;
CIndividual* individual = tempAssWriteDataLinkerIt->getIndividual();
cint64 usedUpdateId = tempAssWriteDataLinkerIt->getUsedAssociationUpdateId();
if (associationData) {
associationData->setCacheTouchId(mNextIndiUpdateId++);
}
bool labelAddition = tempAssWriteDataLinkerIt->getLabelUpdateType() == CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker::ADDITION;
bool labelReplacement = tempAssWriteDataLinkerIt->getLabelUpdateType() == CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker::REPLACEMENT;
bool labelRemoval = tempAssWriteDataLinkerIt->getLabelUpdateType() == CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker::REMOVAL;
bool linksAddition = tempAssWriteDataLinkerIt->getLinksUpdateType() == CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker::ADDITION;
bool linksReplacement = tempAssWriteDataLinkerIt->getLinksUpdateType() == CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker::REPLACEMENT;
bool linksRemoval = tempAssWriteDataLinkerIt->getLinksUpdateType() == CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker::REMOVAL;
CBackendRepresentativeMemoryLabelCacheItem* referredLabels[CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_ASSOCIATABLE_TYPE_COUNT];
for (cint64 i = 0; i < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_ASSOCIATABLE_TYPE_COUNT; ++i) {
CBackendRepresentativeMemoryLabelCacheItem* referredLabelCacheItem = tempAssWriteDataLinkerIt->getReferredLabelData(i);
if (!referredLabelCacheItem) {
CBackendRepresentativeMemoryCacheTemporaryLabelWriteDataLinker* tmpRefferedLabelItem = tempAssWriteDataLinkerIt->getReferredTemporaryLabelData(i);
if (tmpRefferedLabelItem) {
referredLabelCacheItem = (CBackendRepresentativeMemoryLabelCacheItem*)tmpRefferedLabelItem->getTemporaryData();
}
}
referredLabels[i] = referredLabelCacheItem;
}
bool propCut = mPropagationCutIndiSet.contains(individualID);
cint64 integratedIndirectlyConnectedIndividualsChangeId = tempAssWriteDataLinkerIt->getIntegratedIndirectlyConnectedIndividualsChangeId();
bool incompatibleChanges = false;
bool detSameNeighbourCompletion = tempAssWriteDataLinkerIt->requireSameAsNeighboursCompletion();
if (associationData && associationData->getAssociationDataUpdateId() != usedUpdateId || associationData && associationData->getAssociatedIndividualId() != tempAssWriteDataLinkerIt->getIndividualID()) {
bool labelCompatible = false;
if (associationData && mConfInterpretUnchangedLabelsAsCompatible && associationData->isCompletelyHandled()) {
CBackendRepresentativeMemoryCacheIndividualAssociationData* usedReferredAssociationData = associationData->getPreviousData();
while (usedReferredAssociationData && usedReferredAssociationData->getAssociationDataUpdateId() > usedUpdateId) {
usedReferredAssociationData = usedReferredAssociationData->getPreviousData();
}
if (usedReferredAssociationData && usedReferredAssociationData->getAssociationDataUpdateId() != usedUpdateId) {
usedReferredAssociationData = nullptr;
}
if (usedReferredAssociationData) {
labelCompatible = true;
for (cint64 i = 0; associationData && labelCompatible && i < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_ASSOCIATABLE_TYPE_COUNT; ++i) {
if (referredLabels[i] != associationData->getLabelCacheEntry(i)) {
if (i == CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL) {
if (labelReplacement && (linksAddition || propCut)) {
labelAddition = true;
labelReplacement = false;
}
} else if (i == CBackendRepresentativeMemoryLabelCacheItem::NONDETERMINISTIC_COMBINED_NEIGHBOUR_INSTANTIATED_ROLE_SET_LABEL || i == CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_COMBINED_NEIGHBOUR_INSTANTIATED_ROLE_SET_LABEL) {
labelAddition = true;
labelReplacement = false;
} else {
labelCompatible = false;
}
}
}
if (labelCompatible && (linksReplacement && propCut || linksAddition)) {
// only replacing/adding linkers must be checked and the same as for the referred association
CBackendRepresentativeMemoryLabelCacheItem* conSetLabel = associationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::FULL_CONCEPT_SET_LABEL);
CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData* extensionData = (CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData*)conSetLabel->getExtensionData(CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData::CARDINALITY_HASH);
if (extensionData) {
labelCompatible = false;
}
CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash* currentNeighbourRoleSetHash = associationData->getNeighbourRoleSetHash();
CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash* referredNeighbourRoleSetHash = usedReferredAssociationData->getNeighbourRoleSetHash();
for (CBackendRepresentativeMemoryCacheTemporaryIndividualRoleSetNeighbourUpdateDataLinker* roleSetNeighbourUpdateDataLinkerIt = tempAssWriteDataLinkerIt->getRoleSetNeighbourUpdateDataLinker(); labelCompatible && roleSetNeighbourUpdateDataLinkerIt; roleSetNeighbourUpdateDataLinkerIt = roleSetNeighbourUpdateDataLinkerIt->getNext()) {
CIndividualReference neighbourIndiRef = roleSetNeighbourUpdateDataLinkerIt->getNeighbourIndividualReference();
cint64 neighbourId = roleSetNeighbourUpdateDataLinkerIt->getNeighbourIndividualReference().getIndividualID();
CBackendRepresentativeMemoryLabelCacheItem* newNeighbourLabel = roleSetNeighbourUpdateDataLinkerIt->getNeighbourRoleInstantiatedCompinationLabelReference().getReferredLabelData();
if (!newNeighbourLabel) {
CBackendRepresentativeMemoryCacheTemporaryLabelWriteDataLinker* tmpRefferedLabelItem = roleSetNeighbourUpdateDataLinkerIt->getNeighbourRoleInstantiatedCompinationLabelReference().getReferredTemporaryLabelData();
if (tmpRefferedLabelItem) {
newNeighbourLabel = (CBackendRepresentativeMemoryLabelCacheItem*)tmpRefferedLabelItem->getTemporaryData();
}
}
CBackendRepresentativeMemoryLabelCacheItem* referredNeighbourLabel = nullptr;
if (referredNeighbourRoleSetHash) {
referredNeighbourLabel = referredNeighbourRoleSetHash->getNeighbourRoleSetLabel(neighbourId);
}
CBackendRepresentativeMemoryLabelCacheItem* currentNeighbourLabel = nullptr;
if (currentNeighbourRoleSetHash) {
currentNeighbourLabel = currentNeighbourRoleSetHash->getNeighbourRoleSetLabel(neighbourId);
}
if (currentNeighbourLabel && referredNeighbourLabel != currentNeighbourLabel && !isRoleNeighbourLinkLabelItemCompatibility(currentNeighbourLabel, newNeighbourLabel)) {
labelCompatible = false;
} else if (!currentNeighbourLabel && referredNeighbourLabel) {
labelCompatible = false;
}
}
} else {
labelCompatible = false;
}
}
}
if (!labelCompatible) {
labelAddition = true;
labelRemoval = false;
labelReplacement = false;
linksAddition = true;
linksRemoval = false;
linksReplacement = false;
incompatibleChanges = true;
DEBUG_HANDLING_STRING(QString("Installing incompatible update data to %1").arg(individualID))
}
}
bool repLabelUpdateRequired = false;
for (cint64 i = 0; associationData && !repLabelUpdateRequired && i < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_ASSOCIATABLE_TYPE_COUNT; ++i) {
if ((referredLabels[i] || labelReplacement || labelRemoval) && (!associationData || referredLabels[i] != associationData->getLabelCacheEntry(i))) {
DEBUG_HANDLING_STRING(QString("Requiring label %1 update for %2").arg(getRepresentativeCacheLabelItemString(i)).arg(individualID))
repLabelUpdateRequired = true;
}
}
bool statusFlagsUpdateRequired = false;
if ((tempAssWriteDataLinkerIt->getStatusFlags() != 0 || labelReplacement || labelRemoval) && (!associationData || associationData->getStatusFlags() != tempAssWriteDataLinkerIt->getStatusFlags()) || incompatibleChanges) {
statusFlagsUpdateRequired = true;
}
bool linksUpdateRequired = false;
if ((tempAssWriteDataLinkerIt->getRoleSetNeighbourUpdateDataLinker() != nullptr || linksReplacement || linksRemoval)) {
CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash* existingNeighbourRoleSetHash = nullptr;
if (associationData) {
existingNeighbourRoleSetHash = associationData->getNeighbourRoleSetHash();
}
cint64 updateLinksCount = 0;
for (CBackendRepresentativeMemoryCacheTemporaryIndividualRoleSetNeighbourUpdateDataLinker* roleSetNeighbourUpdateDataLinkerIt = tempAssWriteDataLinkerIt->getRoleSetNeighbourUpdateDataLinker(); !linksUpdateRequired && roleSetNeighbourUpdateDataLinkerIt; roleSetNeighbourUpdateDataLinkerIt = roleSetNeighbourUpdateDataLinkerIt->getNext()) {
CIndividualReference neighbourIndiRef = roleSetNeighbourUpdateDataLinkerIt->getNeighbourIndividualReference();
cint64 neighbourId = roleSetNeighbourUpdateDataLinkerIt->getNeighbourIndividualReference().getIndividualID();
CBackendRepresentativeMemoryLabelCacheItem* referredLabelCacheItem = roleSetNeighbourUpdateDataLinkerIt->getNeighbourRoleInstantiatedCompinationLabelReference().getReferredLabelData();
if (!referredLabelCacheItem) {
CBackendRepresentativeMemoryCacheTemporaryLabelWriteDataLinker* tmpRefferedLabelItem = roleSetNeighbourUpdateDataLinkerIt->getNeighbourRoleInstantiatedCompinationLabelReference().getReferredTemporaryLabelData();
if (tmpRefferedLabelItem) {
referredLabelCacheItem = (CBackendRepresentativeMemoryLabelCacheItem*)tmpRefferedLabelItem->getTemporaryData();
}
}
if (linksAddition) {
if (!existingNeighbourRoleSetHash) {
linksUpdateRequired = true;
} else if (existingNeighbourRoleSetHash->getNeighbourRoleSetLabel(neighbourId) != referredLabelCacheItem) {
linksUpdateRequired = true;
}
}
if (linksRemoval) {
if (existingNeighbourRoleSetHash && existingNeighbourRoleSetHash->getNeighbourRoleSetLabel(neighbourId)) {
linksUpdateRequired = true;
}
}
updateLinksCount++;
}
if (!linksUpdateRequired && (updateLinksCount > 0 && !existingNeighbourRoleSetHash || existingNeighbourRoleSetHash && updateLinksCount != existingNeighbourRoleSetHash->getNeighbourCount())) {
linksUpdateRequired = true;
}
}
QSet newCompletingDetSameNeighbours;
QSet newlyCompletedDetSameNeighbours;
bool completeOnlyCompatibleNeighbourRoleLabels = false;
if (detSameNeighbourCompletion) {
CDeterministicSameHandlingInstallationData& installData = mDeterministicSameHandlingInstallationDataHash[individualID];
CBackendRepresentativeMemoryLabelCacheItem* existDetSameHandlIdLabel = nullptr;
if (associationData) {
existDetSameHandlIdLabel = associationData->getDeterministicMergedSameConsideredLabelCacheEntry();
}
if (!existDetSameHandlIdLabel) {
newCompletingDetSameNeighbours = installData.mIdPossibleInstallationSet;
} else {
CBackendRepresentativeMemoryLabelCacheItem* newDetSameLabel = referredLabels[CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL];
for (CBackendRepresentativeMemoryLabelValueLinker* valueLinkerIt = existDetSameHandlIdLabel->getCacheValueLinker(); valueLinkerIt && detSameNeighbourCompletion; valueLinkerIt = valueLinkerIt->getNext()) {
if (!newDetSameLabel->hasCachedTagValue(valueLinkerIt->getCacheValue().getTag())) {
detSameNeighbourCompletion = false;
DEBUG_HANDLING_STRING(QString("Previous deterministic same handled individual %1 for %2 not newly suggested, installation update not possible").arg(valueLinkerIt->getCacheValue().getTag()).arg(individualID))
}
}
for (cint64 possDetSameInstallId : installData.mIdPossibleInstallationSet) {
if (!existDetSameHandlIdLabel->hasCachedTagValue(possDetSameInstallId)) {
newCompletingDetSameNeighbours.insert(possDetSameInstallId);
}
}
}
if (!newCompletingDetSameNeighbours.isEmpty()) {
cint64 unchangedDetMergedUpdateCount = 0;
CBackendRepresentativeMemoryCacheIndividualAssociationData* prevAssociationData = associationData;
while (prevAssociationData && prevAssociationData->getRepresentativeSameIndividualId() == tempAssWriteDataLinkerIt->getRepresentativeSameIndividualId() && unchangedDetMergedUpdateCount < mConfUnchangedDeterministicSameMergeUpdatesForDeterministicSameNeighbourCompletion) {
++unchangedDetMergedUpdateCount;
prevAssociationData = prevAssociationData->getPreviousData();
}
if (!incompatibleChanges && newCompletingDetSameNeighbours.size() < mConfMinRequiredDeterministicSameMergedHandledInstallationPossiblitiesForNeighbourCompletion && unchangedDetMergedUpdateCount < mConfUnchangedDeterministicSameMergeUpdatesForDeterministicSameNeighbourCompletion) {
incompatibleChanges = true;
statusFlagsUpdateRequired = true;
detSameNeighbourCompletion = false;
} else {
if (incompatibleChanges && (!mConfInstallingDeterministicSameHandlingLargeDifferenceReached || installData.mIdFirstPossibleInstallationSet.size() < mConfInstallingDeterministicSameHandlingLargeDifference)) {
detSameNeighbourCompletion = false;
} else {
DEBUG_HANDLING_STRING(QString("Scheduling installation of %1 newly deterministic same handled individuals for %2").arg(installData.mIdFirstPossibleInstallationSet.size()).arg(individualID))
linksUpdateRequired = true;
}
if (incompatibleChanges && detSameNeighbourCompletion) {
completeOnlyCompatibleNeighbourRoleLabels = true;
}
}
} else {
detSameNeighbourCompletion = false;
}
}
bool integrationUpdated = false;
if (!associationData || (associationData && (tempAssWriteDataLinkerIt->hasIndirectlyConnectedIndividualIntegration() && !associationData->hasIndirectlyConnectedIndividualIntegration() ||
tempAssWriteDataLinkerIt->isIndirectlyConnectedNominalIndividual() && !associationData->isIndirectlyConnectedNominalIndividual() ||
integratedIndirectlyConnectedIndividualsChangeId > 0 && integratedIndirectlyConnectedIndividualsChangeId != associationData->getLastIntegratedIndirectlyConnectedIndividualsChangeId()))) {
integrationUpdated = true;
}
if (!associationData || repLabelUpdateRequired || statusFlagsUpdateRequired || linksUpdateRequired || integrationUpdated) {
++mUpdatedIndiCount;
++mAssociationUpdatedIndiCount;
CBackendRepresentativeMemoryCacheIndividualAssociationData* locAssociationData = nullptr;
locAssociationData = createLocalizedIndividualAssociationData(individualID, associationData, ontologyData, true);
bool requiesDataCopying = false;
CBackendRepresentativeMemoryCacheContext* context = getIndividualAssociationDataMemoryContext(locAssociationData, ontologyData, &requiesDataCopying);
if (propCut) {
locAssociationData->setLastPropagationCuttingUpdateId(locAssociationData->getAssociationDataUpdateId());
}
#ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
locAssociationData->mDebugUpdateFunction = QString("installAssociationUpdate");
if (!locAssociationData->mDebugIndi) {
locAssociationData->mDebugIndi = individual;
}
locAssociationData->mDebugUpdateRecomputationId = mCurrentUpdateHandlingRecomputationId;
locAssociationData->mDebugNeighbourCompletionRequested = tempAssWriteDataLinkerIt->requireSameAsNeighboursCompletion();
#endif
DEBUG_HANDLING_STRING(QString("Updating data for %1").arg(individualID))
locAssociationData->setCacheUpdateId(mNextIndiUpdateId++);
ontologyData->setMaxIndividualAssociationDataUpdateCount(qMax(locAssociationData->getAssociationDataUpdateId(), ontologyData->getMaxIndividualAssociationDataUpdateCount()));
locAssociationData->setRepresentativeSameIndividualId(qMin(tempAssWriteDataLinkerIt->getRepresentativeSameIndividualId(), locAssociationData->getRepresentativeSameIndividualId()));
bool sameIndiMergedChanged = false;
if (locAssociationData->getRepresentativeSameIndividualId() != locAssociationData->getAssociatedIndividualId()) {
ontologyData->setSameIndividualsMergings(true);
if (!associationData || !associationData->hasRepresentativeSameIndividualMerging()) {
mStatDetSameRepresentativeMergingCount++;
ontologyData->incIndividualAssociationMergingCount();
}
}
if (!associationData || locAssociationData->getRepresentativeSameIndividualId() != associationData->getRepresentativeSameIndividualId()) {
sameIndiMergedChanged = true;
}
if (incompatibleChanges && locAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL) && locAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL) != referredLabels[CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL]) {
DEBUG_HANDLING_STRING(QString("Remembering deterministic same mergings required for %1").arg(locAssociationData->getAssociatedIndividualId()))
mTmpDetSameMergingCompletionReferenceHash.insert(locAssociationData->getAssociatedIndividualId(), locAssociationData->getAssociatedIndividualId());
}
bool labelsUpdated = false;
if (!associationData || repLabelUpdateRequired) {
DEBUG_HANDLING_STRING(QString("Updating labels for %1").arg(individualID))
for (cint64 i = 0; i < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_ASSOCIATABLE_TYPE_COUNT; ++i) {
if (referredLabels[i] != locAssociationData->getLabelCacheEntry(i)) {
if (referredLabels[i] || labelReplacement || labelRemoval) {
if (locAssociationData->getLabelCacheEntry(i) && labelAddition) {
CBackendRepresentativeMemoryLabelCacheItem* mergedLabel = getAdditionMergedLabel(i, referredLabels[i], locAssociationData->getLabelCacheEntry(i), ontologyData);
if (mergedLabel != locAssociationData->getLabelCacheEntry(i)) {
DEBUG_HANDLING_STRING(QString("Replacing label %1 for %4 with %5 (merged from %2 and %3)").arg(getRepresentativeCacheLabelItemString(i)).arg(locAssociationData->getLabelCacheEntry(i) ? locAssociationData->getLabelCacheEntry(i)->getCacheEntryID() : 0).arg(referredLabels[i] ? referredLabels[i]->getCacheEntryID() : 0).arg(locAssociationData->getAssociatedIndividualId()).arg(mergedLabel->getCacheEntryID()))
locAssociationData->setLabelCacheEntry(i, mergedLabel);
}
} else {
DEBUG_HANDLING_STRING(QString("Replacing label %1 for %4 with %2 from %3").arg(getRepresentativeCacheLabelItemString(i)).arg(locAssociationData->getLabelCacheEntry(i) ? locAssociationData->getLabelCacheEntry(i)->getCacheEntryID() : 0).arg(referredLabels[i] ? referredLabels[i]->getCacheEntryID() : 0).arg(locAssociationData->getAssociatedIndividualId()))
locAssociationData->setLabelCacheEntry(i, referredLabels[i]);
}
}
}
updateIndexedAssociationCount(locAssociationData, associationData, i, ontologyData);
}
labelsUpdated = true;
associationsUpdated = true;
if (associationData) {
CBackendRepresentativeMemoryLabelCacheItem* indConnNomLabelItem = associationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::INDIRECTLY_CONNECTED_NOMINAL_INDIVIDUAL_SET_LABEL);
if (indConnNomLabelItem) {
mTmpIndiIndirectlyConnNomLabelItemHash.insert(individualID, indConnNomLabelItem);
}
}
} else if (sameIndiMergedChanged) {
for (cint64 i = 0; i < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_ASSOCIATABLE_TYPE_COUNT; ++i) {
bool exactIndiAssocTracking = requiresIndividualAssociations(i);
if (exactIndiAssocTracking && !mConfLateIndividualLabelAssociationIndexing) {
CBackendRepresentativeMemoryLabelCacheItem* labelItem = locAssociationData->getLabelCacheEntry(i);
CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData* indiAssoExtData = getIndividualAssociationsExtensionData(labelItem, ontologyData);
indiAssoExtData->removeIndividualIdAssociation(locAssociationData);
indiAssoExtData->addIndividualIdAssociation(locAssociationData);
}
}
}
if (integrationUpdated) {
if (integratedIndirectlyConnectedIndividualsChangeId > 0) {
locAssociationData->setLastIntegratedIndirectlyConnectedIndividualsChangeId(integratedIndirectlyConnectedIndividualsChangeId);
}
if (tempAssWriteDataLinkerIt->hasIndirectlyConnectedIndividualIntegration()) {
locAssociationData->setIndirectlyConnectedIndividualIntegration(true);
}
if (tempAssWriteDataLinkerIt->isIndirectlyConnectedNominalIndividual()) {
locAssociationData->setIndirectlyConnectedNominalIndividual(true);
}
associationsUpdated = true;
}
if (statusFlagsUpdateRequired) {
locAssociationData->setStatusFlags(tempAssWriteDataLinkerIt->getStatusFlags());
associationsUpdated = true;
}
CBackendRepresentativeMemoryLabelCacheItem* newNeighbourRoleSetCompLabel = locAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL);
CBackendRepresentativeMemoryLabelCacheItem* prevNeighbourRoleSetCompLabel = nullptr;
if (associationData) {
prevNeighbourRoleSetCompLabel = associationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL);
}
if (!linksUpdateRequired && prevNeighbourRoleSetCompLabel && newNeighbourRoleSetCompLabel != prevNeighbourRoleSetCompLabel) {
linksUpdateRequired = true;
associationsUpdated = true;
}
QSet< cint64 >* updateOrRemovalNeighbourIndiIdSet = nullptr;
bool needsNeighCompLabelCleanUp = false;
bool neighbourRoleSetCompLabelReductionRequired = false;
bool neighbourRoleSetCompLabelReductionChecking = false;
bool propCutReadding = false;
QSet* propCutReaddingArrayIds = nullptr;
QSet* propCutRemovingArrayIds = nullptr;
bool propCutRemovedNeighbourIndiLinkerCopied = false;
QHash* propCutArrayNewPrevReductionIdHash = nullptr;
bool hasProblematicLeveledNeighbours = locAssociationData->hasProblematicLeveledNeigbour();
//QHash< cint64, cint64 >* idProblematicLeveledNeighboursHash = nullptr;
QSet< cint64 >** idProblematicLeveledNeighboursSetArray = nullptr;
// update neighbour data
bool linksUpdated = false;
if (linksUpdateRequired) {
linksUpdated = true;
DEBUG_HANDLING_STRING(QString("Updating links for %1").arg(individualID))
CBackendRepresentativeMemoryLabelCacheItem* propMarkLabelItem = ontologyData->getPrioritizedPropagationMarkedNeighbourLabelItem();
CBackendRepresentativeMemoryCacheTemporaryIndividualRoleSetNeighbourUpdateDataLinker* roleSetNeighbourUpdateDataLinker = tempAssWriteDataLinkerIt->getRoleSetNeighbourUpdateDataLinker();
// create role set neighbour array and neighbour role set hash, if addition, then copy previous data
CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* newArrayIndexData = getIndividualNeighbourArrayIndexExtensionData(newNeighbourRoleSetCompLabel, ontologyData);
CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash* newNeighbourRoleSetHash = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash* prevNeighbourRoleSetHash = nullptr;
if (associationData) {
prevNeighbourRoleSetHash = associationData->getNeighbourRoleSetHash();
}
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* newArray = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* prevArray = nullptr;
CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* prevArrayIndexData = nullptr;
if (associationData) {
prevArray = associationData->getRoleSetNeighbourArray();
}
if (prevArray) {
prevArrayIndexData = prevArray->getIndexData();
}
if ((linksRemoval || linksReplacement || tempAssWriteDataLinkerIt->requireSameAsNeighboursCompletion() && !detSameNeighbourCompletion) && prevArray) {
updateOrRemovalNeighbourIndiIdSet = new QSet< cint64 >();
++mStatUpdatedOrRemovedNeighbourLinksAssociationUpdateCount;
} else {
++mStatAddingNeighbourLinksAssociationUpdateCount;
}
if (updateOrRemovalNeighbourIndiIdSet && mTmpPropCutIndiArrayNeighboursHandlingDataHash) {
CPropagationCutNeighbourArrayHandlingData* handlingData = mTmpPropCutIndiArrayNeighboursHandlingDataHash->value(individualID);
if (handlingData) {
propCutReadding = true;
if (!handlingData->mReaddingArrayPosSet.isEmpty()) {
propCutReaddingArrayIds = &handlingData->mReaddingArrayPosSet;
}
if (!handlingData->mReductionArrayPosSet.isEmpty()) {
for (cint64 i : handlingData->mReductionArrayPosSet) {
CBackendRepresentativeMemoryLabelCacheItem* prevNeighLabelItem = prevArrayIndexData->getNeighbourRoleSetLabel(i);
CBackendRepresentativeMemoryLabelCacheItem* reducedNeighLabelItem = getReducedLabel(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_LABEL, prevNeighLabelItem, [&](const CCacheValue& cacheValue)->bool {
if (isCacheValueRoleNondeterministic(cacheValue)) {
return true;
} else {
return false;
}
}, ontologyData);
cint64 newIndex = newArrayIndexData->getIndex(reducedNeighLabelItem);
//KONCLUDE_ASSERT_X(newIndex < 0, "BackendCache", "reduction label not available");
if (newIndex >= 0) {
if (!propCutArrayNewPrevReductionIdHash) {
propCutArrayNewPrevReductionIdHash = new QHash();
}
propCutArrayNewPrevReductionIdHash->insertMulti(newIndex, i);
}
}
}
if (!handlingData->mRemovalArrayPosSet.isEmpty()) {
propCutRemovingArrayIds = &handlingData->mRemovalArrayPosSet;
}
}
}
for (CBackendRepresentativeMemoryCacheTemporaryIndividualRoleSetNeighbourUpdateDataLinker* roleSetNeighbourUpdateDataLinkerIt = roleSetNeighbourUpdateDataLinker; roleSetNeighbourUpdateDataLinkerIt; roleSetNeighbourUpdateDataLinkerIt = roleSetNeighbourUpdateDataLinkerIt->getNext()) {
CIndividualReference neighbourIndiRef = roleSetNeighbourUpdateDataLinkerIt->getNeighbourIndividualReference();
CBackendRepresentativeMemoryLabelCacheItem* referredLabelCacheItem = roleSetNeighbourUpdateDataLinkerIt->getNeighbourRoleInstantiatedCompinationLabelReference().getReferredLabelData();
if (!referredLabelCacheItem) {
CBackendRepresentativeMemoryCacheTemporaryLabelWriteDataLinker* tmpRefferedLabelItem = roleSetNeighbourUpdateDataLinkerIt->getNeighbourRoleInstantiatedCompinationLabelReference().getReferredTemporaryLabelData();
if (tmpRefferedLabelItem) {
referredLabelCacheItem = (CBackendRepresentativeMemoryLabelCacheItem*)tmpRefferedLabelItem->getTemporaryData();
}
}
if (referredLabelCacheItem && prevNeighbourRoleSetHash) {
if (!updateOrRemovalNeighbourIndiIdSet && prevNeighbourRoleSetHash->getNeighbourRoleSetLabel(neighbourIndiRef.getIndividualID())) {
updateOrRemovalNeighbourIndiIdSet = new QSet< cint64 >();
++mStatUpdatedOrRemovedNeighbourLinksAssociationUpdateCount;
--mStatAddingNeighbourLinksAssociationUpdateCount;
} else if (!hasProblematicLeveledNeighbours && ontologyData->hasInvolvedIndividuals()) {
CBackendRepresentativeMemoryCacheIndividualAssociationData* neighbourAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[neighbourIndiRef.getIndividualID()];
if (neighbourAssociationData && neighbourAssociationData->hasProblematicLevel()) {
locAssociationData->setProblematicLeveledNeigbour(true);
hasProblematicLeveledNeighbours = true;
}
}
}
}
cint64 prevPrioPropMarkIndex = -1;
cint64 newPrioPropMarkIndex = -1;
bool readdingPrioPropMarkNeighbours = false;
if (prevNeighbourRoleSetCompLabel && prevArray && propMarkLabelItem && linksAddition && incompatibleChanges) {
prevPrioPropMarkIndex = prevArrayIndexData->getIndex(propMarkLabelItem);
if (prevPrioPropMarkIndex >= 0 && prevArray->at(prevPrioPropMarkIndex).getIndividualCount() > 0) {
readdingPrioPropMarkNeighbours = true;
}
}
if (readdingPrioPropMarkNeighbours) {
newPrioPropMarkIndex = newArrayIndexData->getIndex(propMarkLabelItem);
if (newPrioPropMarkIndex < 0) {
CCacheValue extendingCacheValue;
extendingCacheValue.initCacheValue(propMarkLabelItem->getCacheEntryID(), (cint64)propMarkLabelItem, CCacheValue::CACHE_VALUE_TAG_AND_ENTRY);
CBackendRepresentativeMemoryLabelCacheItem* prevNewNeighbourRoleSetCompLabel = newNeighbourRoleSetCompLabel;
newNeighbourRoleSetCompLabel = getExtendedLabel(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL, prevNewNeighbourRoleSetCompLabel, extendingCacheValue, ontologyData);
locAssociationData->setLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL, newNeighbourRoleSetCompLabel);
updateIndexedAssociationCount(locAssociationData, prevNewNeighbourRoleSetCompLabel, CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL, ontologyData);
newArrayIndexData = getIndividualNeighbourArrayIndexExtensionData(newNeighbourRoleSetCompLabel, ontologyData);
newPrioPropMarkIndex = newArrayIndexData->getIndex(propMarkLabelItem);
}
}
if (hasProblematicLeveledNeighbours) {
//idProblematicLeveledNeighboursHash = new QHash< cint64, cint64 >();
idProblematicLeveledNeighboursSetArray = new QSet*[newArrayIndexData->getArraySize()];
for (cint64 i = 0; i < newArrayIndexData->getArraySize(); ++i) {
idProblematicLeveledNeighboursSetArray[i] = new QSet();
}
}
if (prevNeighbourRoleSetCompLabel == newNeighbourRoleSetCompLabel && prevArray && linksAddition && !updateOrRemovalNeighbourIndiIdSet) {
newArray->initNeighbourArray(prevArray);
newNeighbourRoleSetHash->initNeighbourRoleSetHash(prevNeighbourRoleSetHash, requiesDataCopying);
if (requiesDataCopying) {
for (cint64 i = 0; i < newArrayIndexData->getArraySize(); ++i) {
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourData& neighbourData = newArray->at(i);
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* firstNewIndiIdLinker = copyNeighbourIndividualIdLinkers(neighbourData.getIndividualIdLinker(), context, locAssociationData, i);
neighbourData.setIndividualIdLinker(firstNewIndiIdLinker, false);
}
}
if (hasProblematicLeveledNeighbours) {
for (cint64 i = 0; i < newArrayIndexData->getArraySize(); ++i) {
if (i != newPrioPropMarkIndex) {
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourData& neighbourData = newArray->at(i);
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiIdLinker = neighbourData.getIndividualIdLinker();
for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiIdLinkerIt = indiIdLinker; indiIdLinkerIt; indiIdLinkerIt = indiIdLinkerIt->getNext()) {
cint64 neighbourId = indiIdLinkerIt->getIndividualId();
CBackendRepresentativeMemoryCacheIndividualAssociationData* neighbourAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[neighbourId];
if (neighbourAssociationData->hasProblematicLevel()) {
//idProblematicLeveledNeighboursHash->insertMulti(i, neighbourId);
idProblematicLeveledNeighboursSetArray[i]->insert(neighbourId);
indiIdLinker = indiIdLinkerIt->getNext();
} else {
break;
}
}
neighbourData.setIndividualIdLinker(indiIdLinker, false);
}
}
}
} else {
newArray->initNeighbourArray(newArrayIndexData);
// copy data from previous array
if (linksAddition && prevArray && !updateOrRemovalNeighbourIndiIdSet) {
CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* prevArrayIndexData = prevArray->getIndexData();
for (cint64 i = 0; i < newArrayIndexData->getArraySize(); ++i) {
cint64 prevIndex = prevArrayIndexData->getIndex(newArrayIndexData->getNeighbourRoleSetLabel(i));
if (prevIndex >= 0 && i != newPrioPropMarkIndex) {
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourData& neighbourData = newArray->at(i);
neighbourData = prevArray->at(prevIndex);
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiIdLinker = neighbourData.getIndividualIdLinker();
if (hasProblematicLeveledNeighbours) {
for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiIdLinkerIt = indiIdLinker; indiIdLinkerIt; indiIdLinkerIt = indiIdLinkerIt->getNext()) {
cint64 neighbourId = indiIdLinkerIt->getIndividualId();
CBackendRepresentativeMemoryCacheIndividualAssociationData* neighbourAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[neighbourId];
if (neighbourAssociationData->hasProblematicLevel()) {
//idProblematicLeveledNeighboursHash->insertMulti(i, neighbourId);
idProblematicLeveledNeighboursSetArray[i]->insert(neighbourId);
indiIdLinker = indiIdLinkerIt->getNext();
} else {
break;
}
}
}
if (requiesDataCopying) {
indiIdLinker = copyNeighbourIndividualIdLinkers(indiIdLinker, context, locAssociationData, i);
}
neighbourData.setIndividualIdLinker(indiIdLinker, false);
}
}
newNeighbourRoleSetHash->initNeighbourRoleSetHash(prevNeighbourRoleSetHash, requiesDataCopying);
} else {
if (!linksReplacement) {
newNeighbourRoleSetHash->initNeighbourRoleSetHash(prevNeighbourRoleSetHash, requiesDataCopying);
} else {
newNeighbourRoleSetHash->initNeighbourRoleSetHash(nullptr);
}
}
}
QHash updatedPrevArrayPosCountHash;
// add/update new data, TODO: first add non-problematic leveled neighbours
for (CBackendRepresentativeMemoryCacheTemporaryIndividualRoleSetNeighbourUpdateDataLinker* roleSetNeighbourUpdateDataLinkerIt = roleSetNeighbourUpdateDataLinker; roleSetNeighbourUpdateDataLinkerIt; roleSetNeighbourUpdateDataLinkerIt = roleSetNeighbourUpdateDataLinkerIt->getNext()) {
CIndividualReference neighbourIndiRef = roleSetNeighbourUpdateDataLinkerIt->getNeighbourIndividualReference();
CBackendRepresentativeMemoryLabelCacheItem* referredLabelCacheItem = roleSetNeighbourUpdateDataLinkerIt->getNeighbourRoleInstantiatedCompinationLabelReference().getReferredLabelData();
if (!referredLabelCacheItem) {
CBackendRepresentativeMemoryCacheTemporaryLabelWriteDataLinker* tmpRefferedLabelItem = roleSetNeighbourUpdateDataLinkerIt->getNeighbourRoleInstantiatedCompinationLabelReference().getReferredTemporaryLabelData();
if (tmpRefferedLabelItem) {
referredLabelCacheItem = (CBackendRepresentativeMemoryLabelCacheItem*)tmpRefferedLabelItem->getTemporaryData();
}
}
if (referredLabelCacheItem) {
cint64 index = newArrayIndexData->getIndex(referredLabelCacheItem);
CBackendRepresentativeMemoryCacheIndividualAssociationData* neighbourAssociationData = nullptr;
if (hasProblematicLeveledNeighbours) {
neighbourAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[neighbourIndiRef.getIndividualID()];
}
if (hasProblematicLeveledNeighbours && neighbourAssociationData && neighbourAssociationData->hasProblematicLevel()) {
cint64 prevIndex = newArrayIndexData->getIndex(newNeighbourRoleSetHash->getNeighbourRoleSetLabel(neighbourIndiRef.getIndividualID()));
if (prevIndex >= 0) {
idProblematicLeveledNeighboursSetArray[prevIndex]->remove(neighbourIndiRef.getIndividualID());
newArray->at(prevIndex).decIndividualCount();
}
//idProblematicLeveledNeighboursHash->insertMulti(index, neighbourIndiRef.getIndividualID());
idProblematicLeveledNeighboursSetArray[index]->insert(neighbourIndiRef.getIndividualID());
newArray->at(index).incIndividualCount();
} else {
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newLinker = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
++mStatCreatedNeighbourLinks;
newLinker->initIndividualIdLinker(neighbourIndiRef.getIndividualID());
newArray->at(index).addIndividualIdLinker(newLinker);
}
newNeighbourRoleSetHash->setNeighbourRoleSetLabel(neighbourIndiRef.getIndividualID(), referredLabelCacheItem);
if (updateOrRemovalNeighbourIndiIdSet) {
cint64 prevIndex = -1;
if (prevArrayIndexData && prevNeighbourRoleSetHash) {
prevIndex = prevArrayIndexData->getIndex(prevNeighbourRoleSetHash->getNeighbourRoleSetLabel(neighbourIndiRef.getIndividualID()));
if (prevIndex >= 0) {
updatedPrevArrayPosCountHash[prevIndex]++;
}
}
updateOrRemovalNeighbourIndiIdSet->insert(neighbourIndiRef.getIndividualID());
}
}
}
// copy previous data
if (updateOrRemovalNeighbourIndiIdSet && (!linksReplacement || propCutReadding)) {
CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* prevArrayIndexData = prevArray->getIndexData();
for (cint64 i = 0; i < newArrayIndexData->getArraySize(); ++i) {
CBackendRepresentativeMemoryLabelCacheItem* referredLabelCacheItem = newArrayIndexData->getNeighbourRoleSetLabel(i);
cint64 prevIndex = prevArrayIndexData->getIndex(referredLabelCacheItem);
if (prevIndex >= 0 && i != newPrioPropMarkIndex && (!propCutReadding || propCutReaddingArrayIds->contains(prevIndex))) {
bool firstNonProblematicNeighbourIndiFound = false;
cint64 remainingPrevUpdatedRemovedCount = updatedPrevArrayPosCountHash.value(prevIndex);
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourData& prevNeighbourData = prevArray->at(prevIndex);
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourData& newNeighbourData = newArray->at(i);
cint64 remainingNeighbourCount = prevNeighbourData.getIndividualCount();
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* addingNeighbourLinker = newNeighbourData.getIndividualIdLinker();
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* lastAddingNeighbourLinker = nullptr;
if (newNeighbourData.getIndividualIdLinker()) {
lastAddingNeighbourLinker = newNeighbourData.getIndividualIdLinker()->getLastListLink();
}
cint64 addingNeighbourCount = 0;
for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* neighbourLinkerIt = prevNeighbourData.getIndividualIdLinker(); neighbourLinkerIt; neighbourLinkerIt = neighbourLinkerIt->getNext()) {
cint64 neighbourIndiId = neighbourLinkerIt->getIndividualId();
if (!updateOrRemovalNeighbourIndiIdSet->contains(neighbourIndiId)) {
newNeighbourRoleSetHash->setNeighbourRoleSetLabel(neighbourIndiId, referredLabelCacheItem);
CBackendRepresentativeMemoryCacheIndividualAssociationData* neighbourAssociationData = nullptr;
if (!firstNonProblematicNeighbourIndiFound && hasProblematicLeveledNeighbours) {
neighbourAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[neighbourIndiId];
}
if (!firstNonProblematicNeighbourIndiFound && hasProblematicLeveledNeighbours && neighbourAssociationData && neighbourAssociationData->hasProblematicLevel()) {
idProblematicLeveledNeighboursSetArray[i]->insert(neighbourIndiId);
//idProblematicLeveledNeighboursHash->insertMulti(i, neighbourIndiId);
addingNeighbourCount++;
--remainingNeighbourCount;
} else {
firstNonProblematicNeighbourIndiFound = true;
if (remainingPrevUpdatedRemovedCount <= 0) {
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* readdingNeighbourLinker = neighbourLinkerIt;
if (propCutReadding) {
for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* readdingNeighbourLinkerIt = readdingNeighbourLinker; readdingNeighbourLinkerIt; readdingNeighbourLinkerIt = readdingNeighbourLinkerIt->getNext()) {
cint64 neighbourIndiId = readdingNeighbourLinkerIt->getIndividualId();
newNeighbourRoleSetHash->setNeighbourRoleSetLabel(neighbourIndiId, referredLabelCacheItem);
}
}
if (requiesDataCopying) {
readdingNeighbourLinker = copyNeighbourIndividualIdLinkers(readdingNeighbourLinker, context, locAssociationData, i);
}
if (!addingNeighbourLinker) {
addingNeighbourLinker = readdingNeighbourLinker;
} else {
lastAddingNeighbourLinker->setNext(readdingNeighbourLinker);
}
addingNeighbourCount += remainingNeighbourCount;
remainingNeighbourCount = 0;
break;
} else {
--remainingNeighbourCount;
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newLinker = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
++mStatCreatedNeighbourLinks;
newLinker->initIndividualIdLinker(neighbourIndiId);
addingNeighbourCount++;
if (!addingNeighbourLinker) {
lastAddingNeighbourLinker = addingNeighbourLinker = newLinker;
} else {
lastAddingNeighbourLinker->setNext(newLinker);
lastAddingNeighbourLinker = newLinker;
}
}
}
} else {
remainingPrevUpdatedRemovedCount--;
--remainingNeighbourCount;
}
//prevArray->at(prevIndex).visitNeighbourIndividualIds([&](cint64 neighbourIndiId)->bool {
// if (!updateOrRemovalNeighbourIndiIdSet->contains(neighbourIndiId)) {
// newNeighbourRoleSetHash->setNeighbourRoleSetLabel(neighbourIndiId, referredLabelCacheItem);
// CBackendRepresentativeMemoryCacheIndividualAssociationData* neighbourAssociationData = nullptr;
// if (!firstNonProblematicNeighbourIndiFound && hasProblematicLeveledNeighbours) {
// neighbourAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[neighbourIndiId];
// }
// if (!firstNonProblematicNeighbourIndiFound && hasProblematicLeveledNeighbours && neighbourAssociationData && neighbourAssociationData->hasProblematicLevel()) {
// idProblematicLeveledNeighboursSetArray[i]->insert(neighbourIndiId);
// //idProblematicLeveledNeighboursHash->insertMulti(i, neighbourIndiId);
// newArray->at(i).incIndividualCount();
// } else {
// firstNonProblematicNeighbourIndiFound = true;
// CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newLinker = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
// ++mStatCreatedNeighbourLinks;
// newLinker->initIndividualIdLinker(neighbourIndiId);
// newArray->at(i).addIndividualIdLinker(newLinker);
// }
// }
// return true;
//});
}
newNeighbourData.setIndividualIdLinker(addingNeighbourLinker, false);
newNeighbourData.incIndividualCount(addingNeighbourCount);
}
}
if (propCutArrayNewPrevReductionIdHash) {
for (QHash::const_iterator it = propCutArrayNewPrevReductionIdHash->constBegin(), itEnd = propCutArrayNewPrevReductionIdHash->constEnd(); it != itEnd; ++it) {
cint64 newArrayPos = it.key();
cint64 prevIndex = it.value();
if (prevIndex >= 0 && newArrayPos >= 0) {
CBackendRepresentativeMemoryLabelCacheItem* referredLabelCacheItem = newArrayIndexData->getNeighbourRoleSetLabel(newArrayPos);
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourData& newNeighbourData = newArray->at(newArrayPos);
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourData& prevNeighbourData = prevArray->at(prevIndex);
if (newNeighbourData.getIndividualCount() <= 0) {
newNeighbourData = prevNeighbourData;
} else {
bool firstNonProblematicNeighbourIndiFound = false;
for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* neighbourLinkerIt = prevNeighbourData.getIndividualIdLinker(); neighbourLinkerIt; neighbourLinkerIt = neighbourLinkerIt->getNext()) {
cint64 neighbourIndiId = neighbourLinkerIt->getIndividualId();
if (!updateOrRemovalNeighbourIndiIdSet->contains(neighbourIndiId)) {
newNeighbourRoleSetHash->setNeighbourRoleSetLabel(neighbourIndiId, referredLabelCacheItem);
CBackendRepresentativeMemoryCacheIndividualAssociationData* neighbourAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[neighbourIndiId];
if (!firstNonProblematicNeighbourIndiFound && hasProblematicLeveledNeighbours && neighbourAssociationData && neighbourAssociationData->hasProblematicLevel()) {
idProblematicLeveledNeighboursSetArray[newArrayPos]->insert(neighbourIndiId);
newNeighbourData.incIndividualCount();
} else {
firstNonProblematicNeighbourIndiFound = true;
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newLinker = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
++mStatCreatedNeighbourLinks;
newLinker->initIndividualIdLinker(neighbourIndiId);
newNeighbourData.addIndividualIdLinker(newLinker);
}
}
}
}
}
}
}
//if (propCutReaddingArrayIds) {
// delete propCutReaddingArrayIds;
//}
if (propCutArrayNewPrevReductionIdHash) {
delete propCutArrayNewPrevReductionIdHash;
}
delete updateOrRemovalNeighbourIndiIdSet;
if (propCutRemovingArrayIds) {
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* prevRemNeighIndiLinker = locAssociationData->getPropagationCutRemovedNeighbourIndividualLinker();
if (requiesDataCopying) {
propCutRemovedNeighbourIndiLinkerCopied = true;
prevRemNeighIndiLinker = copyNeighbourIndividualIdLinkers(prevRemNeighIndiLinker, context, locAssociationData, -1);
}
QSet prevRemNeighIndiSet;
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newRemNeighIndiLinker = nullptr;
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newLastRemNeighIndiLinker = nullptr;
for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* neighbourLinkerIt = prevRemNeighIndiLinker; neighbourLinkerIt; ) {
cint64 neighbourIndiId = neighbourLinkerIt->getIndividualId();
CBackendRepresentativeMemoryLabelCacheItem* neighbourLabelItem = newNeighbourRoleSetHash->getNeighbourRoleSetLabel(neighbourIndiId);
if (!neighbourLabelItem) {
prevRemNeighIndiSet.insert(neighbourIndiId);
if (!newLastRemNeighIndiLinker) {
newLastRemNeighIndiLinker = newRemNeighIndiLinker = neighbourLinkerIt;
} else {
newLastRemNeighIndiLinker->setNext(neighbourLinkerIt);
newLastRemNeighIndiLinker = neighbourLinkerIt;
}
neighbourLinkerIt = neighbourLinkerIt->getNext();
if (newLastRemNeighIndiLinker) {
newLastRemNeighIndiLinker->setNext(nullptr);
}
} else {
neighbourLinkerIt = neighbourLinkerIt->getNext();
}
}
for (cint64 prevIndex : *propCutRemovingArrayIds) {
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourData& prevNeighbourData = prevArray->at(prevIndex);
for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* neighbourLinkerIt = prevNeighbourData.getIndividualIdLinker(); neighbourLinkerIt; neighbourLinkerIt = neighbourLinkerIt->getNext()) {
cint64 neighbourIndiId = neighbourLinkerIt->getIndividualId();
CBackendRepresentativeMemoryLabelCacheItem* neighbourLabelItem = newNeighbourRoleSetHash->getNeighbourRoleSetLabel(neighbourIndiId);
if (!prevRemNeighIndiSet.contains(neighbourIndiId) && !neighbourLabelItem) {
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newLinker = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
++mStatCreatedNeighbourLinks;
newLinker->initIndividualIdLinker(neighbourIndiId);
newRemNeighIndiLinker = newLinker->append(newRemNeighIndiLinker);
}
}
}
locAssociationData->setPropagationCutRemovedNeighbourIndividualLinker(newRemNeighIndiLinker);
}
if (readdingPrioPropMarkNeighbours) {
newArray->at(newPrioPropMarkIndex) = prevArray->at(prevPrioPropMarkIndex);
if (requiesDataCopying) {
newArray->at(newPrioPropMarkIndex).setIndividualIdLinker(copyNeighbourIndividualIdLinkers(newArray->at(newPrioPropMarkIndex).getIndividualIdLinker(), context, locAssociationData, newPrioPropMarkIndex), false);
}
}
if (detSameNeighbourCompletion) {
locAssociationData->setNeighbourRoleSetHash(newNeighbourRoleSetHash);
locAssociationData->setRoleSetNeighbourArray(newArray);
if (completeSameAsNeighbours(locAssociationData, newCompletingDetSameNeighbours, newlyCompletedDetSameNeighbours, completeOnlyCompatibleNeighbourRoleLabels, ontologyData)) {
mTmpCompleteNeighbourSameIndiMergingSet.insert(locAssociationData->getAssociatedIndividualId());
}
}
neighbourRoleSetCompLabelReductionChecking = true;
} else {
if (detSameNeighbourCompletion) {
locAssociationData->setNeighbourRoleSetHash(newNeighbourRoleSetHash);
locAssociationData->setRoleSetNeighbourArray(newArray);
if (completeSameAsNeighbours(locAssociationData, newCompletingDetSameNeighbours, newlyCompletedDetSameNeighbours, completeOnlyCompatibleNeighbourRoleLabels, ontologyData)) {
mTmpCompleteNeighbourSameIndiMergingSet.insert(locAssociationData->getAssociatedIndividualId());
}
}
if (incompatibleChanges || detSameNeighbourCompletion) {
neighbourRoleSetCompLabelReductionChecking = true;
}
}
if (hasProblematicLeveledNeighbours) {
//for (QHash< cint64, cint64 >::const_iterator idProbNeighIt = idProblematicLeveledNeighboursHash->constBegin(), idProbNeighItEnd = idProblematicLeveledNeighboursHash->constEnd(); idProbNeighIt != idProbNeighItEnd; ++idProbNeighIt) {
// cint64 idx = idProbNeighIt.key();
// cint64 neighbourId = idProbNeighIt.value();
for (cint64 idx = 0; idx < newArrayIndexData->getArraySize(); ++idx) {
for (cint64 neighbourId : *(idProblematicLeveledNeighboursSetArray[idx])) {
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourData& neighbourData = newArray->at(idx);
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newLinker = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
++mStatCreatedNeighbourLinks;
newLinker->initIndividualIdLinker(neighbourId);
neighbourData.addIndividualIdLinker(newLinker, false);
}
delete (idProblematicLeveledNeighboursSetArray[idx]);
}
//delete idProblematicLeveledNeighboursHash;
delete[] idProblematicLeveledNeighboursSetArray;
}
if (neighbourRoleSetCompLabelReductionChecking) {
for (cint64 i = 0; i < newArrayIndexData->getArraySize(); ++i) {
if (!newArray->at(i).getIndividualIdLinker()) {
//if (!newArray->at(i).getIndividualIdLinker() && (!idProblematicLeveledNeighboursHash || !idProblematicLeveledNeighboursHash->contains(i))) {
neighbourRoleSetCompLabelReductionRequired = true;
}
}
if (neighbourRoleSetCompLabelReductionRequired) {
++mReducedNeighbourArrayCount;
CBackendRepresentativeMemoryLabelCacheItem* neighbourRoleSetCombinationLabelSet = locAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL);
CBackendRepresentativeMemoryLabelCacheItem* reducedNeighbourRoleSetCombinationLabelSet = getReducedLabel(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL, neighbourRoleSetCombinationLabelSet, [&](const CCacheValue& cacheValue)->bool {
CBackendRepresentativeMemoryLabelCacheItem* neighbourRoleSetLabel = (CBackendRepresentativeMemoryLabelCacheItem*)cacheValue.getIdentification();
cint64 index = newArrayIndexData->getIndex(neighbourRoleSetLabel);
if (index < 0) {
return true;
}
//if (!newArray->at(index).getIndividualIdLinker() && (!idProblematicLeveledNeighboursHash || !idProblematicLeveledNeighboursHash->contains(index))) {
if (!newArray->at(index).getIndividualIdLinker()) {
return true;
}
return false;
}, ontologyData);
if (neighbourRoleSetCombinationLabelSet != reducedNeighbourRoleSetCombinationLabelSet) {
locAssociationData->setLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL, reducedNeighbourRoleSetCombinationLabelSet);
updateIndexedAssociationCount(locAssociationData, neighbourRoleSetCombinationLabelSet, CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL, ontologyData);
CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* reducedArrayIndexData = getIndividualNeighbourArrayIndexExtensionData(reducedNeighbourRoleSetCombinationLabelSet, ontologyData);
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* reducedArray = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
reducedArray->initNeighbourArray(reducedArrayIndexData);
// copy data into reduced array
for (cint64 i = 0; i < reducedArrayIndexData->getArraySize(); ++i) {
cint64 prevIndex = newArrayIndexData->getIndex(reducedArrayIndexData->getNeighbourRoleSetLabel(i));
if (prevIndex >= 0) {
reducedArray->at(i) = newArray->at(prevIndex);
}
}
newArray = reducedArray;
newArrayIndexData = reducedArrayIndexData;
}
}
}
locAssociationData->setNeighbourRoleSetHash(newNeighbourRoleSetHash);
locAssociationData->setRoleSetNeighbourArray(newArray);
associationsUpdated = true;
} else if (associationData && associationData->getNeighbourRoleSetHash()) {
if (detSameNeighbourCompletion) {
linksUpdated = true;
CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash* newNeighbourRoleSetHash = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* newArray = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
newArray->initNeighbourArray(associationData->getRoleSetNeighbourArray());
newNeighbourRoleSetHash->initNeighbourRoleSetHash(associationData->getNeighbourRoleSetHash(), requiesDataCopying);
if (requiesDataCopying) {
CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* arrayIndexData = newArray->getIndexData();
for (cint64 i = 0; i < arrayIndexData->getArraySize(); ++i) {
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourData& neighbourData = newArray->at(i);
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* firstNewIndiIdLinker = copyNeighbourIndividualIdLinkers(neighbourData.getIndividualIdLinker(), context, locAssociationData, i);
neighbourData.setIndividualIdLinker(firstNewIndiIdLinker, false);
}
}
locAssociationData->setNeighbourRoleSetHash(newNeighbourRoleSetHash);
locAssociationData->setRoleSetNeighbourArray(newArray);
bool completed = completeSameAsNeighbours(locAssociationData, newCompletingDetSameNeighbours, newlyCompletedDetSameNeighbours, completeOnlyCompatibleNeighbourRoleLabels, ontologyData);
if (completed || requiesDataCopying) {
associationsUpdated = true;
mTmpCompleteNeighbourSameIndiMergingSet.insert(locAssociationData->getAssociatedIndividualId());
} else {
locAssociationData->setNeighbourRoleSetHash(associationData->getNeighbourRoleSetHash());
locAssociationData->setRoleSetNeighbourArray(associationData->getRoleSetNeighbourArray());
}
} else {
if (requiesDataCopying) {
CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash* newNeighbourRoleSetHash = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* newArray = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
newArray->initNeighbourArray(associationData->getRoleSetNeighbourArray());
newNeighbourRoleSetHash->initNeighbourRoleSetHash(associationData->getNeighbourRoleSetHash(), requiesDataCopying);
CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* arrayIndexData = newArray->getIndexData();
for (cint64 i = 0; i < arrayIndexData->getArraySize(); ++i) {
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourData& neighbourData = newArray->at(i);
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* firstNewIndiIdLinker = copyNeighbourIndividualIdLinkers(neighbourData.getIndividualIdLinker(), context, locAssociationData, i);
neighbourData.setIndividualIdLinker(firstNewIndiIdLinker, false);
}
locAssociationData->setNeighbourRoleSetHash(newNeighbourRoleSetHash);
locAssociationData->setRoleSetNeighbourArray(newArray);
} else {
locAssociationData->setNeighbourRoleSetHash(associationData->getNeighbourRoleSetHash());
locAssociationData->setRoleSetNeighbourArray(associationData->getRoleSetNeighbourArray());
}
}
}
if (requiesDataCopying && !propCutRemovedNeighbourIndiLinkerCopied) {
propCutRemovedNeighbourIndiLinkerCopied = true;
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* prevRemNeighIndiLinker = locAssociationData->getPropagationCutRemovedNeighbourIndividualLinker();
prevRemNeighIndiLinker = copyNeighbourIndividualIdLinkers(prevRemNeighIndiLinker, context, locAssociationData, -1);
locAssociationData->setPropagationCutRemovedNeighbourIndividualLinker(prevRemNeighIndiLinker);
}
CBackendRepresentativeMemoryLabelCacheItem* detSameLabItem = locAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL);
if (detSameLabItem) {
mStatMaxSameAsMergedCount = qMax(mStatMaxSameAsMergedCount, detSameLabItem->getCacheValueCount());
}
if (locAssociationData->getNeighbourRoleSetHash()) {
mStatMaxNeighbourLinksCount = qMax(mStatMaxNeighbourLinksCount, locAssociationData->getNeighbourRoleSetHash()->getNeighbourCount());
}
mStatMaxAssociationUpdateCount = qMax(mStatMaxAssociationUpdateCount, locAssociationData->getAssociationDataUpdateId());
if (tempAssWriteDataLinkerIt->hasIndirectlyConnectedIndividualIntegration()) {
CBackendRepresentativeMemoryCacheNominalIndividualIndirectConnectionData*& nomConnData = (*ontologyData->getNominaIIndividualdIndirectConnectionDataHash())[individualID];
if (nomConnData || integratedIndirectlyConnectedIndividualsChangeId > 0) {
if (integratedIndirectlyConnectedIndividualsChangeId != nomConnData->getLastChangeId()) {
locAssociationData->setCompletelyHandled(false);
associationsUpdated = true;
}
}
}
if (incompatibleChanges && locAssociationData->isCompletelyHandled() && (labelsUpdated || linksUpdated || tempAssWriteDataLinkerIt->requireSameAsNeighboursCompletion())) {
// TODO: may optimize by checking whether something has actually been updated, e.g., labels, links, etc., but requires neighbours completion then
mCheckIncompatibleIndiCount++;
if (locAssociationData->hasRepresentativeSameIndividualMerging()) {
markRepresentativeReferencedIndividualAssociationIncompletelyHandled(individualID, locAssociationData, ontologyData);
} else {
locAssociationData->setCompletelyHandled(false);
associationsUpdated = true;
}
}
if (locAssociationData->isCompletelyPropagated() && !tempAssWriteDataLinkerIt->isCompletelyPropagated()) {
locAssociationData->setCompletelyPropagated(false);
associationsUpdated = true;
}
if (!locAssociationData->isCompletelyHandled() && locAssociationData->hasRepresentativeSameIndividualMerging()) {
locAssociationData->setCompletelyHandled(true);
markRepresentativeReferencedIndividualAssociationIncompletelyHandled(individualID, locAssociationData, ontologyData);
associationsUpdated = true;
}
if (detSameNeighbourCompletion) {
if (referredLabels[CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL] != locAssociationData->getDeterministicMergedSameConsideredLabelCacheEntry()) {
CBackendRepresentativeMemoryLabelCacheItem* prevDetSameLabel = locAssociationData->getDeterministicMergedSameConsideredLabelCacheEntry();
CBackendRepresentativeMemoryLabelCacheItem* newDetSameLabel = referredLabels[CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL];
CBackendRepresentativeMemoryLabelCacheItem* reducedNewDetSameLabel = getReducedLabel(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL, newDetSameLabel, [&](const CCacheValue& cacheValue)-> bool {
cint64 detSameHandledId = cacheValue.getTag();
if (detSameHandledId == individualID || prevDetSameLabel && prevDetSameLabel->hasCachedTagValue(detSameHandledId) || newlyCompletedDetSameNeighbours.contains(detSameHandledId)) {
return false;
} else {
return true;
}
}, ontologyData);
locAssociationData->setDeterministicMergedSameConsideredLabelCacheEntry(reducedNewDetSameLabel);
cint64 prevDetSameLabelCacheEntryId = -1;
if (prevDetSameLabel) {
prevDetSameLabelCacheEntryId = prevDetSameLabel->getCacheEntryID();
}
DEBUG_HANDLING_STRING(QString("Updating deterministically same handled label for %1 to label %2 from label %3").arg(locAssociationData->getAssociatedIndividualId()).arg(locAssociationData->getDeterministicMergedSameConsideredLabelCacheEntry()->getCacheEntryID()).arg(prevDetSameLabelCacheEntryId))
associationsUpdated = true;
}
}
associationsUpdated |= storeIndividualIncompletelyMarked(locAssociationData, !locAssociationData->isCompletelyHandled(), ontologyData);
//if (mIncompletelyHandledIndiIdCount != mIncompletelyAssociatedIndividualSet.size()) {
// bool debug = true;
//}
//if (locAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL) && locAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL)->getCacheValueCount() > 0) {
// if (!locAssociationData->getRoleSetNeighbourArray()) {
// bool debug = true;
// }
//}
if (associationsUpdated) {
if (locAssociationData->getPreviousData() && !locAssociationData->getPreviousData()->getPreviousData()) {
ontologyData->incIndividualAssociationDataDirectUpdateCount();
}
setUpdatedIndividualAssociationData(individualID, locAssociationData, ontologyData);
if (!locAssociationData->hasRepresentativeSameIndividualMerging()) {
udateDeterministicSameAssociations(locAssociationData, ontologyData);
}
}
DEBUG_HANDLING_STRING(QString("Finishing updating data for %1").arg(locAssociationData->getAssociatedIndividualId()))
}
return associationsUpdated;
}
bool CBackendRepresentativeMemoryCache::updateIndexedAssociationCount(CBackendRepresentativeMemoryCacheIndividualAssociationData* locAssociationData, CBackendRepresentativeMemoryLabelCacheItem* prevLabelItem, cint64 i, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
bool updated = false;
bool exactIndiAssocTracking = requiresIndividualAssociations(i);
CBackendRepresentativeMemoryLabelCacheItem* newLabelItem = locAssociationData->getLabelCacheEntry(i);
if (prevLabelItem != newLabelItem) {
if (prevLabelItem) {
updated = true;
prevLabelItem->decIndividualAssociationCount();
if (exactIndiAssocTracking && !mConfLateIndividualLabelAssociationIndexing) {
CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData* indiAssoExtData = getIndividualAssociationsExtensionData(prevLabelItem, ontologyData);
indiAssoExtData->removeIndividualIdAssociation(locAssociationData);
}
}
if (newLabelItem) {
updated = true;
newLabelItem->incIndividualAssociationCount();
if (exactIndiAssocTracking && !mConfLateIndividualLabelAssociationIndexing) {
CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData* indiAssoExtData = getIndividualAssociationsExtensionData(newLabelItem, ontologyData);
indiAssoExtData->addIndividualIdAssociation(locAssociationData);
}
}
}
return true;
}
bool CBackendRepresentativeMemoryCache::updateIndexedAssociationCount(CBackendRepresentativeMemoryCacheIndividualAssociationData* locAssociationData, CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData, cint64 i, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
bool updated = false;
bool exactIndiAssocTracking = requiresIndividualAssociations(i);
bool sameIndiMergedChanged = false;
if (!associationData || associationData->getRepresentativeSameIndividualId() != locAssociationData->getRepresentativeSameIndividualId()) {
sameIndiMergedChanged = true;
}
CBackendRepresentativeMemoryLabelCacheItem* prevLabelItem = nullptr;
CBackendRepresentativeMemoryLabelCacheItem* newLabelItem = locAssociationData->getLabelCacheEntry(i);
if (associationData) {
prevLabelItem = associationData->getLabelCacheEntry(i);
}
if (prevLabelItem != newLabelItem || sameIndiMergedChanged) {
if (prevLabelItem) {
updated = true;
prevLabelItem->decIndividualAssociationCount();
if (exactIndiAssocTracking && !mConfLateIndividualLabelAssociationIndexing) {
CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData* indiAssoExtData = getIndividualAssociationsExtensionData(prevLabelItem, ontologyData);
indiAssoExtData->removeIndividualIdAssociation(associationData);
}
}
if (newLabelItem) {
updated = true;
newLabelItem->incIndividualAssociationCount();
if (exactIndiAssocTracking && !mConfLateIndividualLabelAssociationIndexing) {
CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData* indiAssoExtData = getIndividualAssociationsExtensionData(newLabelItem, ontologyData);
indiAssoExtData->addIndividualIdAssociation(locAssociationData);
}
}
}
return true;
}
bool CBackendRepresentativeMemoryCache::udateDeterministicSameAssociations(CBackendRepresentativeMemoryCacheIndividualAssociationData* locNeighbourAssociationData, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
bool updated = false;
DEBUG_HANDLING_STRING(QString("Updating deterministic same references for %1").arg(locNeighbourAssociationData->getAssociatedIndividualId()))
CBackendRepresentativeMemoryLabelCacheItem* detSameLabItem = locNeighbourAssociationData->getDeterministicMergedSameConsideredLabelCacheEntry();
if (detSameLabItem) {
for (CBackendRepresentativeMemoryLabelValueLinker* detSameValueLinkerIt = detSameLabItem->getCacheValueLinker(); detSameValueLinkerIt; detSameValueLinkerIt = detSameValueLinkerIt->getNext()) {
cint64 detSameNeighbourId = detSameValueLinkerIt->getCacheValue().getTag();
if (detSameNeighbourId != locNeighbourAssociationData->getAssociatedIndividualId()) {
CBackendRepresentativeMemoryCacheIndividualAssociationData* detSameNeighbourAssociationData = nullptr;
if (ontologyData->getIndividualIdAssoiationDataVector() && detSameNeighbourId < ontologyData->getIndividualIdAssoiationDataVectorSize()) {
detSameNeighbourAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[detSameNeighbourId];
}
if (detSameNeighbourAssociationData && detSameNeighbourAssociationData->getDeterministicSameIndividualId() == locNeighbourAssociationData->getAssociatedIndividualId()) {
if (checkRequiresDeterministicSameAsAssociationUpdateInstallation(detSameNeighbourAssociationData, detSameNeighbourId, locNeighbourAssociationData, locNeighbourAssociationData->getAssociatedIndividualId(), ontologyData)) {
updated |= installDeterministicSameAsAssociationUpdate(detSameNeighbourAssociationData, detSameNeighbourId, locNeighbourAssociationData, locNeighbourAssociationData->getAssociatedIndividualId(), ontologyData);
}
}
}
}
}
return updated;
}
bool CBackendRepresentativeMemoryCache::completeSameAsNeighbours(CBackendRepresentativeMemoryCacheIndividualAssociationData* locAssociationData, QSet newCompletingDetSameNeighbours, QSet& newlyCompletedDetSameNeighbours, bool completeOnlyCompatibleNeighbourRoleLabels, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
bool completed = false;
DEBUG_HANDLING_STRING(QString("Completing links for %1").arg(locAssociationData->getAssociatedIndividualId()))
cint64 linksCompleted = 0;
CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
CBackendRepresentativeMemoryLabelCacheItem* detSameLabItem = locAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL);
CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash* neighbourRoleSetLabelItemHash = locAssociationData->getNeighbourRoleSetHash();
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* neighbourArray = locAssociationData->getRoleSetNeighbourArray();
CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* arrayIndexData = neighbourArray->getIndexData();
if (detSameLabItem) {
#ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
locAssociationData->mDebugNeighbourCompletionPerformed = true;
#endif
for (CBackendRepresentativeMemoryLabelValueLinker* detSameValueLinkerIt = detSameLabItem->getCacheValueLinker(); detSameValueLinkerIt; detSameValueLinkerIt = detSameValueLinkerIt->getNext()) {
cint64 detSameId = detSameValueLinkerIt->getCacheValue().getTag();
if (detSameId != locAssociationData->getAssociatedIndividualId() && newCompletingDetSameNeighbours.contains(detSameId)) {
CBackendRepresentativeMemoryCacheIndividualAssociationData* detSameAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[detSameId];
if (!detSameAssociationData->hasDeterministicSameIndividualMerging()) {
DEBUG_HANDLING_STRING(QString("Trying links completion for %1 with %2").arg(locAssociationData->getAssociatedIndividualId()).arg(detSameId))
bool completedDetSameId = true;
bool oneChangedDetSameIdNeighbourLabelCompletion = false;
bool oneIncompatibleDetSameIdNeighbourLabelCompletion = false;
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* detSameNeighbourArray = detSameAssociationData->getRoleSetNeighbourArray();
if (detSameNeighbourArray) {
CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* detSameArrayIndexData = detSameNeighbourArray->getIndexData();
for (cint64 i = 0; i < detSameArrayIndexData->getArraySize() && completedDetSameId; ++i) {
CBackendRepresentativeMemoryLabelCacheItem* detSameNeighbourRoleSetLabelItem = detSameArrayIndexData->getNeighbourRoleSetLabel(i);
cint64 arrayIndex = arrayIndexData->getIndex(detSameNeighbourRoleSetLabelItem);
if (arrayIndex >= 0) {
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourData& neighbourData = neighbourArray->at(arrayIndex);
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* neighbourIndiLinker = detSameNeighbourArray->at(i).getIndividualIdLinker();
for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* neighbourIndiLinkerIt = neighbourIndiLinker; neighbourIndiLinkerIt && completedDetSameId; neighbourIndiLinkerIt = neighbourIndiLinkerIt->getNext()) {
cint64 neighbourIndiId = neighbourIndiLinkerIt->getIndividualId();
CBackendRepresentativeMemoryLabelCacheItem* neighbourRoleSetLabelItem = neighbourRoleSetLabelItemHash->getNeighbourRoleSetLabel(neighbourIndiId);
if (!neighbourRoleSetLabelItem) {
neighbourRoleSetLabelItemHash->setNeighbourRoleSetLabel(neighbourIndiId, detSameNeighbourRoleSetLabelItem);
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newLinker = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
++mStatCreatedNeighbourLinks;
++linksCompleted;
newLinker->initIndividualIdLinker(neighbourIndiId);
neighbourData.addIndividualIdLinker(newLinker, true);
} else if (neighbourRoleSetLabelItem != detSameNeighbourRoleSetLabelItem) {
oneChangedDetSameIdNeighbourLabelCompletion = true;
++mStatChangedLabelNeighbourCompletionCount;
if (completeOnlyCompatibleNeighbourRoleLabels) {
completedDetSameId = false;
++mStatIncompatibleLabelNeighbourCompletionCount;
oneIncompatibleDetSameIdNeighbourLabelCompletion = true;
}
}
}
}
}
}
if (completedDetSameId) {
DEBUG_HANDLING_STRING(QString("Links completed for %1 with %2").arg(locAssociationData->getAssociatedIndividualId()).arg(detSameId))
newlyCompletedDetSameNeighbours.insert(detSameId);
completed = true;
++mStatNeighbourCompletionDetSameSuccededCount;
}
if (oneChangedDetSameIdNeighbourLabelCompletion) {
++mStatNeighbourCompletionDetSameChangedCount;
}
if (oneIncompatibleDetSameIdNeighbourLabelCompletion) {
++mStatNeighbourCompletionDetSameIncompatibleCount;
}
if (!oneChangedDetSameIdNeighbourLabelCompletion) {
++mStatNeighbourCompletionDetSameUnchangedCount;
}
} else {
newlyCompletedDetSameNeighbours.insert(detSameId);
}
}
}
}
DEBUG_HANDLING_STRING(QString("Completed %2 links for %1").arg(locAssociationData->getAssociatedIndividualId()).arg(linksCompleted))
return completed;
}
bool CBackendRepresentativeMemoryCache::completeDeterministicSameAsMergingInformation(QHash& tmpDetSameMergingCompletionReferenceHash, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
bool updated = false;
cint64 completedDetSameLabelItems = 0;
cint64 completedDetSameIntegratedIndividuals = 0;
cint64 completedIndividualUpdates = 0;
DEBUG_HANDLING_STRING(QString("Starting deterministic same mergings completions for %1 entries").arg(tmpDetSameMergingCompletionReferenceHash.size()))
QSet mergedHandledIndiIdSet;
for (QHash::const_iterator it = tmpDetSameMergingCompletionReferenceHash.constBegin(), itEnd = tmpDetSameMergingCompletionReferenceHash.constEnd(); it != itEnd; ++it) {
cint64 refIndiId = it.key();
cint64 destIndiId = it.value();
DEBUG_HANDLING_STRING(QString("Completing deterministic same mergings for %1 to %2").arg(refIndiId).arg(destIndiId))
if (!mergedHandledIndiIdSet.contains(destIndiId)) {
mergedHandledIndiIdSet.insert(destIndiId);
++completedDetSameLabelItems;
CBackendRepresentativeMemoryCacheIndividualAssociationData* refIndiAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[destIndiId];
CBackendRepresentativeMemoryLabelCacheItem* detSameLabItem = refIndiAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL);
if (detSameLabItem) {
QSet mergingDetSameLabelItemSet;
QList processingDetSameLabelItemList;
processingDetSameLabelItemList.append(detSameLabItem);
mergingDetSameLabelItemSet.insert(detSameLabItem);
cint64 minRepIndiId = destIndiId;
while (!processingDetSameLabelItemList.isEmpty()) {
CBackendRepresentativeMemoryLabelCacheItem* procDetSameLabItem = processingDetSameLabelItemList.takeFirst();
for (CBackendRepresentativeMemoryLabelValueLinker* detSameValueLinkerIt = procDetSameLabItem->getCacheValueLinker(); detSameValueLinkerIt; detSameValueLinkerIt = detSameValueLinkerIt->getNext()) {
cint64 detSameId = detSameValueLinkerIt->getCacheValue().getTag();
mergedHandledIndiIdSet.insert(detSameId);
minRepIndiId = qMin(detSameId, minRepIndiId);
++completedDetSameIntegratedIndividuals;
CBackendRepresentativeMemoryCacheIndividualAssociationData* sameIndiAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[detSameId];
CBackendRepresentativeMemoryLabelCacheItem* otherDetSameLabItem = sameIndiAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL);
if (otherDetSameLabItem && otherDetSameLabItem != procDetSameLabItem && !mergingDetSameLabelItemSet.contains(otherDetSameLabItem)) {
mergingDetSameLabelItemSet.insert(otherDetSameLabItem);
processingDetSameLabelItemList.append(otherDetSameLabItem);
}
cint64 otherDestIndiId = tmpDetSameMergingCompletionReferenceHash.value(detSameId);
CBackendRepresentativeMemoryCacheIndividualAssociationData* otherDestIndiAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[otherDestIndiId];
CBackendRepresentativeMemoryLabelCacheItem* otherDestDetSameLabItem = otherDestIndiAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL);
if (otherDestDetSameLabItem && otherDestDetSameLabItem != procDetSameLabItem && !mergingDetSameLabelItemSet.contains(otherDestDetSameLabItem)) {
mergingDetSameLabelItemSet.insert(otherDestDetSameLabItem);
processingDetSameLabelItemList.append(otherDestDetSameLabItem);
}
}
}
CBackendRepresentativeMemoryLabelCacheItem* mergedDetSameLabItem = detSameLabItem;
for (CBackendRepresentativeMemoryLabelCacheItem* mergingLabelItem : mergingDetSameLabelItemSet) {
if (mergingLabelItem != detSameLabItem) {
mergedDetSameLabItem = getAdditionMergedLabel(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL, mergingLabelItem, mergedDetSameLabItem, ontologyData);
}
}
if (mergedDetSameLabItem) {
DEBUG_HANDLING_STRING(QString("Found new deterministic same mergings for %1 to %2 stored in label item %3").arg(refIndiId).arg(destIndiId).arg(mergedDetSameLabItem->getCacheEntryID()))
}
for (CBackendRepresentativeMemoryLabelValueLinker* detSameValueLinkerIt = mergedDetSameLabItem->getCacheValueLinker(); detSameValueLinkerIt; detSameValueLinkerIt = detSameValueLinkerIt->getNext()) {
cint64 detSameId = detSameValueLinkerIt->getCacheValue().getTag();
CBackendRepresentativeMemoryCacheIndividualAssociationData* sameIndiAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[detSameId];
if (sameIndiAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL) != mergedDetSameLabItem || sameIndiAssociationData->getRepresentativeSameIndividualId() != minRepIndiId || sameIndiAssociationData->getAssociatedIndividualId() == minRepIndiId && sameIndiAssociationData->isCompletelyHandled()) {
bool incrementUpdateIdForDeterministicSameAsCompletion = mConfIncrementUpdateIdForDeterministicSameAsCompletion;
if (sameIndiAssociationData->getAssociatedIndividualId() == minRepIndiId) {
incrementUpdateIdForDeterministicSameAsCompletion = true;
}
CBackendRepresentativeMemoryCacheIndividualAssociationData* locSameIndiAssociationData = createLocalizedIndividualAssociationData(sameIndiAssociationData->getAssociatedIndividualId(), sameIndiAssociationData, ontologyData, false, incrementUpdateIdForDeterministicSameAsCompletion);
CBackendRepresentativeMemoryCacheContext* context = getIndividualAssociationDataMemoryContext(locSameIndiAssociationData, ontologyData);
locSameIndiAssociationData->setCacheUpdateId(mNextIndiUpdateId++);
locSameIndiAssociationData->setNeighbourRoleSetHash(sameIndiAssociationData->getNeighbourRoleSetHash());
locSameIndiAssociationData->setRoleSetNeighbourArray(sameIndiAssociationData->getRoleSetNeighbourArray());
#ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
locSameIndiAssociationData->mDebugUpdateFunction = QString("completeDeterministicSameAsMergingInformation");
locSameIndiAssociationData->mDebugSameIndividualsCompletion = true;
locSameIndiAssociationData->mDebugUpdateRecomputationId = mCurrentUpdateHandlingRecomputationId;
#endif
DEBUG_HANDLING_STRING(QString("Updating new deterministic same mergings for %1").arg(detSameId))
locSameIndiAssociationData->setRepresentativeSameIndividualId(minRepIndiId);
if (locSameIndiAssociationData->hasRepresentativeSameIndividualMerging()) {
if (!sameIndiAssociationData || !sameIndiAssociationData->hasRepresentativeSameIndividualMerging()) {
mStatDetSameRepresentativeMergingCount++;
ontologyData->incIndividualAssociationMergingCount();
}
}
//locSameIndiAssociationData->setDeterministicSameIndividualId(minRepIndiId);
locSameIndiAssociationData->setLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL, mergedDetSameLabItem);
updateIndexedAssociationCount(locSameIndiAssociationData, sameIndiAssociationData, CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL, ontologyData);
if (detSameId != minRepIndiId) {
locSameIndiAssociationData->setCompletelyHandled(true);
} else {
locSameIndiAssociationData->setCompletelyHandled(false);
}
++mUpdatedIndiCount;
++completedIndividualUpdates;
ontologyData->setMaxIndividualAssociationDataUpdateCount(qMax(locSameIndiAssociationData->getAssociationDataUpdateId(), ontologyData->getMaxIndividualAssociationDataUpdateCount()));
storeIndividualIncompletelyMarked(locSameIndiAssociationData, !locSameIndiAssociationData->isCompletelyHandled(), ontologyData);
setUpdatedIndividualAssociationData(locSameIndiAssociationData->getAssociatedIndividualId(), locSameIndiAssociationData, ontologyData);
updated = true;
}
}
}
}
}
LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Completed %1 deterministic same individual labels over %2 individuals with %3 updated association data.").arg(completedDetSameLabelItems).arg(completedDetSameIntegratedIndividuals).arg(completedIndividualUpdates), this);
return updated;
}
bool CBackendRepresentativeMemoryCache::completeNeighboursForSameAsMerging(QSet& tmpCompleteNeighbourSameIndiMergingSet, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
cint64 checkingNeighbours = 0;
cint64 completedNeighbours = 0;
cint64 completedNeighbourLinks = 0;
bool updated = false;
QHash tmpLocalIndiAssocDataHash;
for (cint64 mergeIndiId : mTmpCompleteNeighbourSameIndiMergingSet) {
CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData = ontologyData->getIndividualIdAssoiationDataVector()[mergeIndiId];
CBackendRepresentativeMemoryLabelCacheItem* detSameLabItem = associationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL);
if (detSameLabItem) {
CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash* neighbourRoleSetLabelItemHash = associationData->getNeighbourRoleSetHash();
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* neighbourArray = associationData->getRoleSetNeighbourArray();
CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* arrayIndexData = neighbourArray->getIndexData();
for (cint64 i = 0; i < arrayIndexData->getArraySize(); ++i) {
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* neighbourIndiLinker = neighbourArray->at(i).getIndividualIdLinker();
for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* neighbourIndiLinkerIt = neighbourIndiLinker; neighbourIndiLinkerIt; neighbourIndiLinkerIt = neighbourIndiLinkerIt->getNext()) {
cint64 neighbourIndiId = neighbourIndiLinkerIt->getIndividualId();
++checkingNeighbours;
CBackendRepresentativeMemoryCacheIndividualAssociationData* neighbourAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[neighbourIndiId];
if (!neighbourAssociationData->hasDeterministicSameIndividualMerging()) {
CBackendRepresentativeMemoryCacheIndividualAssociationData* locNeighbourAssociationData = nullptr;
CBackendRepresentativeMemoryLabelCacheItem* sameNeighbourRoleSetLabelItem = nullptr;
cint64 sameNeighbourRoleSetLabelArrayId = -1;
for (CBackendRepresentativeMemoryLabelValueLinker* detSameValueLinkerIt = detSameLabItem->getCacheValueLinker(); detSameValueLinkerIt && !sameNeighbourRoleSetLabelItem; detSameValueLinkerIt = detSameValueLinkerIt->getNext()) {
cint64 detSameId = detSameValueLinkerIt->getCacheValue().getTag();
if (neighbourAssociationData->getNeighbourRoleSetHash()) {
CBackendRepresentativeMemoryLabelCacheItem* neighbourRoleSetLabelItem = neighbourAssociationData->getNeighbourRoleSetHash()->getNeighbourRoleSetLabel(detSameId);
if (neighbourRoleSetLabelItem) {
sameNeighbourRoleSetLabelItem = neighbourRoleSetLabelItem;
sameNeighbourRoleSetLabelArrayId = neighbourAssociationData->getRoleSetNeighbourArray()->getIndexData()->getIndex(sameNeighbourRoleSetLabelItem);
}
}
}
if (sameNeighbourRoleSetLabelItem && sameNeighbourRoleSetLabelArrayId >= 0) {
for (CBackendRepresentativeMemoryLabelValueLinker* detSameValueLinkerIt = detSameLabItem->getCacheValueLinker(); detSameValueLinkerIt; detSameValueLinkerIt = detSameValueLinkerIt->getNext()) {
cint64 detSameId = detSameValueLinkerIt->getCacheValue().getTag();
CBackendRepresentativeMemoryLabelCacheItem* neighbourRoleSetLabelItem = neighbourAssociationData->getNeighbourRoleSetHash()->getNeighbourRoleSetLabel(detSameId);
if (!neighbourRoleSetLabelItem) {
if (!locNeighbourAssociationData) {
CBackendRepresentativeMemoryCacheIndividualAssociationData*& tmpLlocNeighbourAssociationData = tmpLocalIndiAssocDataHash[neighbourIndiId];
if (!tmpLlocNeighbourAssociationData) {
tmpLlocNeighbourAssociationData = createLocalizedIndividualAssociationData(neighbourAssociationData->getAssociatedIndividualId(), neighbourAssociationData, ontologyData, false);
bool requiresDataCopying = false;
CBackendRepresentativeMemoryCacheContext* context = getIndividualAssociationDataMemoryContext(tmpLlocNeighbourAssociationData, ontologyData, &requiresDataCopying);
#ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
tmpLlocNeighbourAssociationData->mDebugUpdateFunction = QString("completeNeighboursForSameAsMerging");
tmpLlocNeighbourAssociationData->mDebugNeighbourCompletionExtended = true;
tmpLlocNeighbourAssociationData->mDebugUpdateRecomputationId = mCurrentUpdateHandlingRecomputationId;
#endif
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* newArray = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash* newNeighbourRoleSetHash = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
newNeighbourRoleSetHash->initNeighbourRoleSetHash(neighbourAssociationData->getNeighbourRoleSetHash(), requiresDataCopying);
newArray->initNeighbourArray(neighbourAssociationData->getRoleSetNeighbourArray());
tmpLlocNeighbourAssociationData->setNeighbourRoleSetHash(newNeighbourRoleSetHash);
tmpLlocNeighbourAssociationData->setRoleSetNeighbourArray(newArray);
tmpLlocNeighbourAssociationData->setCacheUpdateId(mNextIndiUpdateId++);
ontologyData->setMaxIndividualAssociationDataUpdateCount(qMax(tmpLlocNeighbourAssociationData->getAssociationDataUpdateId(), ontologyData->getMaxIndividualAssociationDataUpdateCount()));
++mUpdatedIndiCount;
completedNeighbours++;
}
locNeighbourAssociationData = tmpLlocNeighbourAssociationData;
}
CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash* locNeighbourRoleSetLabelItemHash = locNeighbourAssociationData->getNeighbourRoleSetHash();
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* locArray = locNeighbourAssociationData->getRoleSetNeighbourArray();
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newLinker = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
++mStatCreatedNeighbourLinks;
newLinker->initIndividualIdLinker(detSameId);
locArray->at(sameNeighbourRoleSetLabelArrayId).addIndividualIdLinker(newLinker);
locNeighbourRoleSetLabelItemHash->setNeighbourRoleSetLabel(detSameId, sameNeighbourRoleSetLabelItem);
completedNeighbourLinks++;
}
}
}
}
}
}
}
}
for (QHash::const_iterator it = tmpLocalIndiAssocDataHash.constBegin(), itEnd = tmpLocalIndiAssocDataHash.constEnd(); it != itEnd; ++it) {
CBackendRepresentativeMemoryCacheIndividualAssociationData* locNeighbourAssociationData = it.value();
storeIndividualIncompletelyMarked(locNeighbourAssociationData, !locNeighbourAssociationData->isCompletelyHandled(), ontologyData);
setUpdatedIndividualAssociationData(locNeighbourAssociationData->getAssociatedIndividualId(), locNeighbourAssociationData, ontologyData);
updated |= udateDeterministicSameAssociations(locNeighbourAssociationData, ontologyData);
}
LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Completed %1 neighbouring links for %2 neighbours (%4 checked) of %3 same as merged individuals.").arg(completedNeighbourLinks).arg(completedNeighbours).arg(mTmpCompleteNeighbourSameIndiMergingSet.size()).arg(checkingNeighbours), this);
return updated;
}
bool CBackendRepresentativeMemoryCache::storeIndividualIncompletelyMarked(CBackendRepresentativeMemoryCacheIndividualAssociationData* locAssociationData, bool markIncompletelyHandled, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
bool associationsUpdated = false;
cint64 individualID = locAssociationData->getAssociatedIndividualId();
if (!markIncompletelyHandled) {
if (locAssociationData->isIncompletelyMarked()) {
ontologyData->decIncompletelyHandledIndividualIdCount();
//if (!mIncompletelyAssociatedIndividualSet.contains(CIndividualReference(individualID))) {
// bool debug = true;
//}
//mIncompletelyAssociatedIndividualSet.remove(CIndividualReference(individualID));
locAssociationData->setIncompletelyMarked(false);
associationsUpdated = true;
if (locAssociationData->hasProblematicLevel()) {
CCACHINGSET* propIndiSet = ontologyData->getProblematicIncompletelyHandledIndividualSet();
if (propIndiSet) {
propIndiSet->remove(individualID);
}
}
}
} else {
ontologyData->setLastMinIncompletelyHandledIndvidualiId(qMin(ontologyData->getLastMinIncompletelyHandledIndvidualiId(), individualID));
if (!locAssociationData->isIncompletelyMarked()) {
ontologyData->incIncompletelyHandledIndividualIdCount();
//if (mIncompletelyAssociatedIndividualSet.contains(CIndividualReference(individualID))) {
// bool debug = true;
//}
//mIncompletelyAssociatedIndividualSet.insert(CIndividualReference(individualID));
locAssociationData->setIncompletelyMarked(true);
associationsUpdated = true;
if (locAssociationData->hasProblematicLevel()) {
CCACHINGSET* propIndiSet = ontologyData->getProblematicIncompletelyHandledIndividualSet();
if (!propIndiSet) {
CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
propIndiSet = CObjectParameterizingAllocator < CCACHINGSET, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
}
propIndiSet->insert(individualID);
}
}
}
return associationsUpdated;
}
bool CBackendRepresentativeMemoryCache::installNominalIndirectConncetionUpdates(CBackendRepresentativeMemoryCacheTemporaryNominalIndirectConnectionDataLinker* tempNomIndirectConnDataLinker, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
bool connectionDataUpdated = false;
for (CBackendRepresentativeMemoryCacheTemporaryNominalIndirectConnectionDataLinker* tempNomIndirectConnDataLinkerIt = tempNomIndirectConnDataLinker; tempNomIndirectConnDataLinkerIt; tempNomIndirectConnDataLinkerIt = tempNomIndirectConnDataLinkerIt->getNext()) {
cint64 nomIndiId = tempNomIndirectConnDataLinkerIt->getIndividualID();
CXLinker* indirectlyConnectedIndividualIdLinker = tempNomIndirectConnDataLinkerIt->getIndirectlyConnectedIndividualIdLinker();
cint64 lastChangeIntegrationId = tempNomIndirectConnDataLinkerIt->getLastChangeIntegrationId();
CBackendRepresentativeMemoryCacheNominalIndividualIndirectConnectionData*& nomConnData = (*ontologyData->getNominaIIndividualdIndirectConnectionDataHash())[nomIndiId];
CBackendRepresentativeMemoryCacheNominalIndividualIndirectConnectionData* prevNomConnData = nomConnData;
CBackendRepresentativeMemoryCacheIndividualAssociationData* nominalAssociationData = nullptr;
if (ontologyData->getIndividualIdAssoiationDataVector() && nomIndiId < ontologyData->getIndividualIdAssoiationDataVectorSize()) {
nominalAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[nomIndiId];
}
if (nominalAssociationData && nominalAssociationData->getLastIntegratedIndirectlyConnectedIndividualsChangeId() != lastChangeIntegrationId && lastChangeIntegrationId > 0 ||
nominalAssociationData && prevNomConnData && lastChangeIntegrationId == 0 && nominalAssociationData->hasIndirectlyConnectedIndividualIntegration() && prevNomConnData->getLastChangeId() != nominalAssociationData->getLastIntegratedIndirectlyConnectedIndividualsChangeId()) {
// mark as incompletely handled
connectionDataUpdated |= markRepresentativeReferencedIndividualAssociationIncompletelyHandled(nomIndiId, nominalAssociationData, ontologyData);
}
CBackendRepresentativeMemoryCacheNominalIndividualIndirectConnectionData* newNomConnData = CObjectAllocator< CBackendRepresentativeMemoryCacheNominalIndividualIndirectConnectionData >::allocateAndConstruct(context->getMemoryAllocationManager());
newNomConnData->initNominalIndividualIndirectConnectionData(prevNomConnData);
nomConnData = newNomConnData;
newNomConnData->setLastChangeId(mNextNomConnUpdateId++);
for (CXLinker* indirectlyConnectedIndividualIdLinkerIt = indirectlyConnectedIndividualIdLinker; indirectlyConnectedIndividualIdLinkerIt; indirectlyConnectedIndividualIdLinkerIt = indirectlyConnectedIndividualIdLinkerIt->getNext()) {
cint64 indiId = indirectlyConnectedIndividualIdLinkerIt->getData();
// avoid duplicates
bool alreadyPresent = false;
CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData = nullptr;
if (ontologyData->getIndividualIdAssoiationDataVector() && indiId < ontologyData->getIndividualIdAssoiationDataVectorSize()) {
associationData = ontologyData->getIndividualIdAssoiationDataVector()[indiId];
}
CBackendRepresentativeMemoryLabelCacheItem* indiConnNomIndiIdSetLabel = nullptr;
if (associationData) {
if (associationData->getCacheUpdateId() < mTmpIndiAssocPrevUpdateId) {
indiConnNomIndiIdSetLabel = associationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::INDIRECTLY_CONNECTED_NOMINAL_INDIVIDUAL_SET_LABEL);
} else {
indiConnNomIndiIdSetLabel = mTmpIndiIndirectlyConnNomLabelItemHash.value(indiId);
}
}
if (indiConnNomIndiIdSetLabel && indiConnNomIndiIdSetLabel->hasCachedTagValue(nomIndiId)) {
alreadyPresent = true;
}
if (!alreadyPresent) {
CXLinker* indiLinker = CObjectAllocator< CXLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
indiLinker->initLinker(indiId);
nomConnData->addIndirectlyConnectedIndividualIdLinker(indiLinker);
connectionDataUpdated = true;
}
}
}
return connectionDataUpdated;
}
void CBackendRepresentativeMemoryCache::setUpdatedIndividualAssociationData(cint64 individualID, CBackendRepresentativeMemoryCacheIndividualAssociationData* locAssociationData, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
CBackendRepresentativeMemoryCacheIndividualAssociationData** indiIdAssoDataVector = ontologyData->getIndividualIdAssoiationDataVector();
cint64 indiIdAssoDataVectorSize = ontologyData->getIndividualIdAssoiationDataVectorSize();
if (individualID >= indiIdAssoDataVectorSize) {
cint64 prevSize = indiIdAssoDataVectorSize;
CBackendRepresentativeMemoryCacheIndividualAssociationData** prevIndiIdAssoDataVector = indiIdAssoDataVector;
indiIdAssoDataVectorSize = qMax(indiIdAssoDataVectorSize * 10, individualID);
indiIdAssoDataVector = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualAssociationData* >::allocateAndConstructArray(context->getMemoryAllocationManager(), indiIdAssoDataVectorSize);
for (cint64 i = 0; i < prevSize; ++i) {
indiIdAssoDataVector[i] = prevIndiIdAssoDataVector[i];
}
for (cint64 i = prevSize; i < indiIdAssoDataVectorSize; ++i) {
indiIdAssoDataVector[i] = nullptr;
}
ontologyData->setIndividualIdAssoiationDataVector(indiIdAssoDataVectorSize, indiIdAssoDataVector);
}
ontologyData->updateMaxStoredIndvidualiId(individualID);
if (!indiIdAssoDataVector[individualID]) {
ontologyData->incIndividualAssociationsCount();
}
if (locAssociationData->getPreviousData() && !locAssociationData->getPreviousData()->getPreviousData()) {
ontologyData->incIndividualAssociationDataUpdateCount();
}
indiIdAssoDataVector[individualID] = locAssociationData;
//mIndiIdAssoDataHash->insert(individualID, locAssociationData);
DEBUG_HANDLING_STRING(QString("Stored update data for %1").arg(locAssociationData->getAssociatedIndividualId()))
}
bool CBackendRepresentativeMemoryCache::getIncompletlyAssociationCachedIndividuals(cint64 ontologyIdentifier, CBackendIndividualRetrievalComputationUpdateCoordinationHash* prevCoordHash, CBackendIndividualRetrievalComputationUpdateCoordinationHash* newCoordHash, bool allIndividualsAdded, bool refillRetrievalCoordHash, cint64 limit, CCallbackData* callbackData) {
if (!callbackData) {
CBlockingCallbackData blockingCallbackData;
if (mConfDirectUpdateSynchronization) {
mDirectUpdateSyncMutex.lock();
CRetrieveIncompletelyAssociationCachedEvent* procEvent = new CRetrieveIncompletelyAssociationCachedEvent(&blockingCallbackData, ontologyIdentifier, prevCoordHash, newCoordHash, allIndividualsAdded, refillRetrievalCoordHash, limit);
processCustomsEvents(CRetrieveIncompletelyAssociationCachedEvent::EVENTTYPE, procEvent);
mDirectUpdateSyncMutex.unlock();
} else {
postEvent(new CRetrieveIncompletelyAssociationCachedEvent(&blockingCallbackData, ontologyIdentifier, prevCoordHash, newCoordHash, allIndividualsAdded, refillRetrievalCoordHash, limit));
}
blockingCallbackData.waitForCallback();
} else {
if (mConfDirectUpdateSynchronization) {
mDirectUpdateSyncMutex.lock();
CRetrieveIncompletelyAssociationCachedEvent* procEvent = new CRetrieveIncompletelyAssociationCachedEvent(callbackData, ontologyIdentifier, prevCoordHash, newCoordHash, allIndividualsAdded, refillRetrievalCoordHash, limit);
processCustomsEvents(CRetrieveIncompletelyAssociationCachedEvent::EVENTTYPE, procEvent);
mDirectUpdateSyncMutex.unlock();
} else {
postEvent(new CRetrieveIncompletelyAssociationCachedEvent(callbackData, ontologyIdentifier, prevCoordHash, newCoordHash, allIndividualsAdded, refillRetrievalCoordHash, limit));
}
}
return true;
}
bool CBackendRepresentativeMemoryCache::initializeIndividualsAssociationCaching(cint64 ontologyIdentifier, cint64 individualCount) {
if (mConfDirectUpdateSynchronization) {
mDirectUpdateSyncMutex.lock();
CInitializeIndividualAssociationsCacheEvent* procEvent = new CInitializeIndividualAssociationsCacheEvent(ontologyIdentifier, individualCount);
processCustomsEvents(CInitializeIndividualAssociationsCacheEvent::EVENTTYPE, procEvent);
mDirectUpdateSyncMutex.unlock();
} else {
postEvent(new CInitializeIndividualAssociationsCacheEvent(ontologyIdentifier, individualCount));
}
return true;
}
bool CBackendRepresentativeMemoryCache::reportMaximumHandledRecomputationId(cint64 ontologyIdentifier, cint64 maximumRecomputationId) {
if (mConfDirectUpdateSynchronization) {
mDirectUpdateSyncMutex.lock();
CReportMaximumHandledRecomputationIdsEvent* procEvent = new CReportMaximumHandledRecomputationIdsEvent(ontologyIdentifier, maximumRecomputationId);
processCustomsEvents(CReportMaximumHandledRecomputationIdsEvent::EVENTTYPE, procEvent);
mDirectUpdateSyncMutex.unlock();
} else {
postEvent(new CReportMaximumHandledRecomputationIdsEvent(ontologyIdentifier, maximumRecomputationId));
}
return true;
}
bool CBackendRepresentativeMemoryCache::requiresIndividualAssociations(cint64 labelType) {
if (labelType == CBackendRepresentativeMemoryLabelCacheItem::FULL_CONCEPT_SET_LABEL || labelType == CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL || labelType == CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_COMBINED_EXISTENTIAL_INSTANTIATED_ROLE_SET_LABEL ||
labelType == CBackendRepresentativeMemoryLabelCacheItem::NONDETERMINISTIC_COMBINED_EXISTENTIAL_INSTANTIATED_ROLE_SET_LABEL || labelType == CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL || labelType == CBackendRepresentativeMemoryLabelCacheItem::NONDETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL ||
labelType == CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_COMBINED_DATA_INSTANTIATED_ROLE_SET_LABEL || labelType == CBackendRepresentativeMemoryLabelCacheItem::NONDETERMINISTIC_COMBINED_DATA_INSTANTIATED_ROLE_SET_LABEL) {
return true;
}
return false;
}
CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData* CBackendRepresentativeMemoryCache::getIndividualAssociationsExtensionData(CBackendRepresentativeMemoryLabelCacheItem* labelItem, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData* extensionData = (CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData*)labelItem->getExtensionData(CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData::INDIVIDUAL_ASSOCIATION_MAP);
if (!extensionData) {
extensionData = CObjectParameterizingAllocator< CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
labelItem->setExtensionData(CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData::INDIVIDUAL_ASSOCIATION_MAP, extensionData);
}
return extensionData;
}
CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* CBackendRepresentativeMemoryCache::getIndividualNeighbourArrayIndexExtensionData(CBackendRepresentativeMemoryLabelCacheItem* labelItem, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* extensionData = (CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData*)labelItem->getExtensionData(CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData::INDIVIDUAL_NEIGHBOUR_ARRAY_INDEX);
if (!extensionData) {
extensionData = CObjectParameterizingAllocator< CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
labelItem->setExtensionData(CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData::INDIVIDUAL_NEIGHBOUR_ARRAY_INDEX, extensionData);
extensionData->initNeighbourArrayIndexData(labelItem);
}
return extensionData;
}
CBackendRepresentativeMemoryLabelCacheItemTagLabelResolvingExtensionData* CBackendRepresentativeMemoryCache::getNeighbourArrayRoleTagResolvingLabelExtensionData(CBackendRepresentativeMemoryLabelCacheItem* labelItem, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
CBackendRepresentativeMemoryLabelCacheItemTagLabelResolvingExtensionData* extensionData = (CBackendRepresentativeMemoryLabelCacheItemTagLabelResolvingExtensionData*)labelItem->getExtensionData(CBackendRepresentativeMemoryLabelCacheItemTagLabelResolvingExtensionData::TAG_RESOLVING_HASH);
if (!extensionData) {
extensionData = CObjectParameterizingAllocator< CBackendRepresentativeMemoryLabelCacheItemTagLabelResolvingExtensionData, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
labelItem->setExtensionData(CBackendRepresentativeMemoryLabelCacheItemTagLabelResolvingExtensionData::TAG_RESOLVING_HASH, extensionData);
extensionData->initTagLabelResolvingExtensionData();
CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* indexExtensionData = getIndividualNeighbourArrayIndexExtensionData(labelItem, ontologyData);
for (cint64 i = 0; i < indexExtensionData->getArraySize(); ++i) {
CBackendRepresentativeMemoryLabelCacheItem* neighbourRoleSetLabelItem = indexExtensionData->getNeighbourRoleSetLabel(i);
for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = neighbourRoleSetLabelItem->getCacheValueLinker(); labelValueLinkerIt; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
CCacheValue cacheValue = labelValueLinkerIt->getData();
CCacheValue::CACHEVALUEIDENTIFIER identifier = cacheValue.getCacheValueIdentifier();
bool nondeterministc = identifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_ROLE || identifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_INVERSEDROLE || identifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_ASSERTEDROLE ||
identifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_INVERSEDASSERTEDROLE || identifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_NOMINAL_CONNECTED_ROLE || identifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_INVERSED_NOMINAL_CONNECTED_ROLE;
CBackendRepresentativeMemoryLabelCacheItemTagLabelResolvingDataLinker* linker = CObjectAllocator< CBackendRepresentativeMemoryLabelCacheItemTagLabelResolvingDataLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
linker->initTagLabelResolvingData(neighbourRoleSetLabelItem, i, !nondeterministc);
extensionData->appendTagLabelResolvingDataLinker(cacheValue.getTag(), linker);
}
}
}
return extensionData;
}
cint64 CBackendRepresentativeMemoryCache::getMinimumSlotReferreringInstalledValidRecomputationId(CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
cint64 minInstalledValidRecId = CINT64_MAX;
cint64 ontologyIdentifier = ontologyData->getOntologyIdentifer();
for (CBackendRepresentativeMemoryCacheSlotItem* slotLinkerIt = mSlotLinker; slotLinkerIt; slotLinkerIt = slotLinkerIt->getNext()) {
CBackendRepresentativeMemoryCacheOntologyData* referredOntologyData = slotLinkerIt->getOntologyData(ontologyIdentifier);
cint64 minValidRecId = referredOntologyData->getMinimumValidRecomputationId();
minInstalledValidRecId = qMin(minInstalledValidRecId, minValidRecId);
}
return minInstalledValidRecId;
}
bool CBackendRepresentativeMemoryCache::processCustomsEvents(QEvent::Type type, CCustomEvent *event) {
if (CThread::processCustomsEvents(type, event)) {
return true;
}
else if (type == CReportMaximumHandledRecomputationIdsEvent::EVENTTYPE) {
CReportMaximumHandledRecomputationIdsEvent* rmhrie = (CReportMaximumHandledRecomputationIdsEvent*)event;
cint64 maxHandledRecomId = rmhrie->getMaximumHandledRecomputationId();
cint64 ontologyIdentifier = rmhrie->getOntologyIdentifier();
CBackendRepresentativeMemoryCacheOntologyData* ontologyData = mOntologyIdentifierDataHash->value(ontologyIdentifier);
if (ontologyData) {
ontologyData->setNextUpdateMinimumValidRecomputationId(maxHandledRecomId+1);
LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Received report of handled recomputation id %1.").arg(maxHandledRecomId), this);
}
++mStatReportedMaximumHandledRecomputationIdCount;
return true;
} else if (type == CInitializeIndividualAssociationsCacheEvent::EVENTTYPE) {
CInitializeIndividualAssociationsCacheEvent* iiace = (CInitializeIndividualAssociationsCacheEvent*)event;
cint64 indiCount = iiace->getIndividualCount();
cint64 ontologyIdentifier = iiace->getOntologyIdentifier();
prepareOntologyDataUpdate(ontologyIdentifier, indiCount);
return true;
} else if (type == CRetrieveIncompletelyAssociationCachedEvent::EVENTTYPE) {
CRetrieveIncompletelyAssociationCachedEvent* iace = (CRetrieveIncompletelyAssociationCachedEvent*)event;
cint64 limit = iace->getRetrievalLimit();
cint64 ontologyIdentifier = iace->getOntologyIdentifier();
bool allIndividualsAdded = iace->hasAllIndividualsAdded();
bool refillRetrievalCoordHash = iace->hasRefillRetrievalCoordHashOrdered();
++mCheckingRemainingIncompletelyHandledCount;
bool updateSlot = false;
CBackendRepresentativeMemoryCacheOntologyData* ontologyData = mOntologyIdentifierDataHash->value(ontologyIdentifier);
if (ontologyData && !ontologyData->isAssociationCompleted()) {
ontologyData->incIncompletelyHandledIndividualsRetrievalCount();
cint64 count = 0;
cint64 prevLastMinIncompletelyHandledIndiId = ontologyData->getLastMinIncompletelyHandledIndvidualiId();
CBackendIndividualRetrievalComputationUpdateCoordinationHash* lastRetrievalHash = iace->getLastIndividualsRetrievalCoordinationHash();
CBackendIndividualRetrievalComputationUpdateCoordinationHash* newRetrievalHash = iace->getNewIndividualsRetrievalCoordinationHash();
newRetrievalHash->setApproximateTotalIncompletelyHandledCount(ontologyData->getIncompletelyHandledIndividualIdCount());
if (lastRetrievalHash) {
cint64 corr = lastRetrievalHash->getApproximateCorrectionIncompletelyHandledCount();
//LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("last corr %1").arg(corr), this);
corr += ontologyData->getIncompletelyHandledIndividualIdCount() - lastRetrievalHash->getApproximateRemainingIncompletelyHandledCount();
//LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("remain diff %1").arg(ontologyData->getIncompletelyHandledIndividualIdCount() - lastRetrievalHash->getApproximateRemainingIncompletelyHandledCount()), this);
corr += lastRetrievalHash->getApproximateTotalIncompletelyHandledCount() - ontologyData->getIncompletelyHandledIndividualIdCount();
//LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("last diff %1").arg(lastRetrievalHash->getApproximateTotalIncompletelyHandledCount() - ontologyData->getIncompletelyHandledIndividualIdCount()), this);
newRetrievalHash->setApproximateCorrectionIncompletelyHandledCount(corr);
//LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("new corr %1").arg(corr), this);
}
if (lastRetrievalHash && !refillRetrievalCoordHash) {
for (CBackendIndividualRetrievalComputationUpdateCoordinationHash::const_iterator it = lastRetrievalHash->constBegin(), itEnd = lastRetrievalHash->constEnd(); it != itEnd; ++it) {
cint64 indiId = it.key();
CBackendIndividualRetrievalComputationUpdateCoordinationHashData* refData = it.value();
bool keepIndi = false;
CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssData = ontologyData->getIndividualIdAssoiationDataVector()[indiId];
if (!refData->isComputationIntegrated() && !refData->isComputationOrdered()) {
keepIndi = true;
//newRetrievalHash->incHashRemainingCount();
//if (!indiAssData->isIncompletelyMarked()) {
// newRetrievalHash->incApproximateCorrectionIncompletelyHandledCount();
//}
} else if (!refData->isProcessed()) {
//newRetrievalHash->incHashComputationCount();
keepIndi = true;
} else {
if (indiAssData->getCacheTouchId() == refData->getAssociationUpdateId()) {
//newRetrievalHash->incHashProcessedCount();
keepIndi = true;
}
}
if (keepIndi) {
refData->incUsageCount();
newRetrievalHash->insert(indiId, refData);
} else {
//newRetrievalHash->incHashRemovedCount();
}
}
}
if (ontologyData->isBasicPrecomputationMode()) {
if (newRetrievalHash) {
cint64 pos = ontologyData->getBasicPrecompuationRetrievalIndividualIdPosition();
for (; pos < ontologyData->getBasicPrecomputationIndividualIdAssoiationDataVectorSize() && count < ontologyData->getIncompletelyHandledIndividualIdCount() && (count < limit || limit < 0); ++pos) {
CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssData = ontologyData->getIndividualIdAssoiationDataVector()[pos];
if (indiAssData && !indiAssData->isCompletelyHandled() && indiAssData->getAssociationDataUpdateId() <= 1) {
CBackendIndividualRetrievalComputationUpdateCoordinationHashData*& refData = (*newRetrievalHash)[pos];
if (!refData) {
refData = newRetrievalHash->createCoordinationData();
refData->setAssociationUpdateId(indiAssData->getCacheTouchId());
refData->incUsageCount();
newRetrievalHash->incHashRemainingCount();
count++;
}
}
}
ontologyData->setBasicPrecompuationRetrievalIndividualIdPosition(pos);
}
if (newRetrievalHash->isEmpty()) {
// deactivate basic precompuation mode
ontologyData->setBasicPrecomputationMode(false);
updateSlot = true;
newRetrievalHash->setBasicPrecomputationFinished(true);
} else {
newRetrievalHash->setBasicPrecomputationMode(true);
}
}
if (!ontologyData->isBasicPrecomputationMode()) {
CCACHINGSET* propIndiSet = ontologyData->getProblematicIncompletelyHandledIndividualSet();
if (propIndiSet) {
for (CCACHINGSET::const_iterator it = propIndiSet->constBegin(), itEnd = propIndiSet->constEnd(); it != itEnd && count < ontologyData->getIncompletelyHandledIndividualIdCount() && (count < limit || limit < 0); ++it) {
cint64 indiId = *it;
CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssData = ontologyData->getIndividualIdAssoiationDataVector()[indiId];
CBackendIndividualRetrievalComputationUpdateCoordinationHashData*& refData = (*newRetrievalHash)[indiId];
if (!refData) {
refData = newRetrievalHash->createCoordinationData();
refData->setAssociationUpdateId(indiAssData->getCacheTouchId());
refData->incUsageCount();
newRetrievalHash->incHashRemainingCount();
count++;
}
}
}
if (newRetrievalHash) {
for (cint64 i = ontologyData->getLastMinIncompletelyHandledIndvidualiId(); i < ontologyData->getIndividualIdAssoiationDataVectorSize() && count < ontologyData->getIncompletelyHandledIndividualIdCount() && (count < limit || limit < 0); ++i) {
CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssData = ontologyData->getIndividualIdAssoiationDataVector()[i];
if (indiAssData && !indiAssData->isCompletelyHandled()) {
CBackendIndividualRetrievalComputationUpdateCoordinationHashData*& refData = (*newRetrievalHash)[i];
if (!refData) {
refData = newRetrievalHash->createCoordinationData();
refData->setAssociationUpdateId(indiAssData->getCacheTouchId());
refData->incUsageCount();
newRetrievalHash->incHashRemainingCount();
count++;
}
} else if (i == ontologyData->getLastMinIncompletelyHandledIndvidualiId()) {
ontologyData->incLastMinIncompletelyHandledIndvidualiId();
}
}
/*if (limit < 0 || limit > mIncompletelyAssociatedIndividualSet.size()) {
*individualSet = mIncompletelyAssociatedIndividualSet;
} else {
cint64 count = 0;
for (QSet::const_iterator it = mIncompletelyAssociatedIndividualSet.constBegin(), itEnd = mIncompletelyAssociatedIndividualSet.constEnd(); it != itEnd && count < limit; ++it) {
individualSet->insert(*it);
++count;
}
}*/
if (newRetrievalHash->size() <= 0 && ontologyData->getIncompletelyHandledIndividualIdCount() > 0) {
cint64 foundIncompleteCount = 0;
cint64 prevIncompleteCount = ontologyData->getIncompletelyHandledIndividualIdCount();
for (cint64 i = 0; i < ontologyData->getIndividualIdAssoiationDataVectorSize() && count < ontologyData->getIncompletelyHandledIndividualIdCount() && (count < limit || limit < 0); ++i) {
CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssData = ontologyData->getIndividualIdAssoiationDataVector()[i];
if (indiAssData && !indiAssData->isCompletelyHandled()) {
if (foundIncompleteCount <= 0) {
LOG(WARN, "::Konclude::RepresentativeBackendCache", logTr("Corrected first insufficiently handled individual from %2 to %1.").arg(ontologyData->getLastMinIncompletelyHandledIndvidualiId()).arg(i), this);
ontologyData->setLastMinIncompletelyHandledIndvidualiId(i);
}
foundIncompleteCount++;
CBackendIndividualRetrievalComputationUpdateCoordinationHashData*& refData = (*newRetrievalHash)[i];
if (!refData) {
refData = newRetrievalHash->createCoordinationData();
refData->setAssociationUpdateId(indiAssData->getCacheTouchId());
refData->incUsageCount();
newRetrievalHash->incHashRemainingCount();
count++;
}
}
}
if (foundIncompleteCount <= 0) {
ontologyData->setIncompletelyHandledIndividualIdCount(0);
}
LOG(WARN, "::Konclude::RepresentativeBackendCache", logTr("Corrected insufficiently handled individuals from %2 to %1.").arg(foundIncompleteCount).arg(prevIncompleteCount), this);
}
}
}
if (allIndividualsAdded) {
if (!ontologyData->isFirstIncompletelyHandledIndividualsRetrieved()) {
if (!ontologyData->isSlotUpdateIntegrated()) {
updateSlot = true;
}
}
ontologyData->setFirstIncompletelyHandledIndividualsRetrieved(true);
}
bool forceCompletion = false;
if (mConfMaxIncompletelyHandledIndividualsRetrievalCount > 0 && ontologyData->getIncompletelyHandledIndividualsRetrievalCount() > mConfMaxIncompletelyHandledIndividualsRetrievalCount) {
LOG(WARN, "::Konclude::RepresentativeBackendCache", logTr("Forcing association completion with reached maximum of %1 retrievals.").arg(mConfMaxIncompletelyHandledIndividualsRetrievalCount), this);
forceCompletion = true;
count = 0;
if (newRetrievalHash) {
newRetrievalHash->clear();
}
}
checkAssociationComplete(ontologyData, forceCompletion);
LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Retrieval %3 returning %1 new of %2 insufficiently handled individuals for reprocessing.").arg(count).arg(ontologyData->getIncompletelyHandledIndividualIdCount()).arg(ontologyData->getIncompletelyHandledIndividualsRetrievalCount()), this);
//if (mIncompletelyHandledIndiIdCount != mIncompletelyAssociatedIndividualSet.size()) {
// bool debug = true;
//}
//if (mIncompletelyHandledIndiIdCount > 0 && individualSet->isEmpty()) {
// cint64 count = 0;
// for (cint64 i = 0; i < mIndiIdAssoDataVectorSize && count < mIncompletelyHandledIndiIdCount && count < limit; ++i) {
// CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssData = mIndiIdAssoDataVector[i];
// if (indiAssData && !indiAssData->isCompletelyHandled()) {
// individualSet->insert(i);
// count++;
// }
// }
// count = count;
// bool debug = true;
//}
}
CCallbackData* callbackData = iace->getCallback();
if (callbackData) {
callbackData->doCallback();
}
if (updateSlot) {
ontologyData->setNextSlotUpdateWaitingCount(mSlotUpdateWaitingIncreaseCount);
createReaderSlotUpdate(ontologyData, &mContext);
cleanUnusedSlots(&mContext);
}
return true;
}
else if (type == CWriteBackendAssociationCachedEvent::EVENTTYPE) {
CWriteBackendAssociationCachedEvent* wcde = (CWriteBackendAssociationCachedEvent*)event;
CMemoryPool* memoryPools = wcde->getMemoryPools();
CBackendRepresentativeMemoryCacheWriteData* newWriteData = (CBackendRepresentativeMemoryCacheWriteData*)wcde->getWriteData();
cint64 dataWriteCount = newWriteData->getCount();
CBackendRepresentativeMemoryCacheWriteData* dataLinkerIt = newWriteData;
cint64 ontologyIdentifier = dataLinkerIt->getOntologyIdentifier();
CBackendRepresentativeMemoryCacheOntologyData* ontologyData = mOntologyIdentifierDataHash->value(ontologyIdentifier);
mLastHandledOntologyContext = ontologyData;
if (mStatCollectStatistics) {
mPendingUpdateCount.deref();
}
mCurrentUpdateHandlingRecomputationId = -1;
if (!ontologyData || !ontologyData->isAssociationCompleted()) {
ontologyData->incCacheDataUpdateWritingCount();
++mWriteDataCount;
ontologyData = prepareOntologyDataUpdate(ontologyIdentifier);
cint64 prevUpdatedIndiCount = mUpdatedIndiCount;
cint64 prevAssociationUpdatedIndiCount = mAssociationUpdatedIndiCount;
bool allCachingSuccess = true;
bool oneCachingSuccess = false;
bool oneCachingExpSuccess = false;
while (dataLinkerIt) {
mCurrentUpdateHandlingRecomputationId = -1;
if (dataLinkerIt->getCacheWriteDataType() == CBackendRepresentativeMemoryCacheWriteData::BACKENDASSOCIATIONWRITEDATATYPE) {
CBackendRepresentativeMemoryCacheLabelAssociationWriteData* baAsWrDa = (CBackendRepresentativeMemoryCacheLabelAssociationWriteData*)dataLinkerIt;
mCurrentUpdateHandlingRecomputationId = baAsWrDa->getRecompuationId();
CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker* tempAssWriteDataLinker = baAsWrDa->getTemporaryAssociationWriteDataLinker();
CBackendRepresentativeMemoryCacheTemporaryLabelWriteDataLinker* tempLabelWriteDataLinker = baAsWrDa->getTemporaryLabelWriteDataLinker();
CBackendRepresentativeMemoryCacheTemporaryCardinalityWriteDataLinker* tempCardWriteDataLinker = baAsWrDa->getTemporaryCardinaltyWriteDataLinker();
CBackendRepresentativeMemoryCacheTemporaryAssociationUseDataLinker* tempAssUseDataLinker = baAsWrDa->getTemporaryAssociationUseDataLinker();
CBackendRepresentativeMemoryCacheTemporaryNominalIndirectConnectionDataLinker* tempNomIndirectConnDataLinker = baAsWrDa->getTemporaryNominalIndirectConnectionDataLinker();
CBackendRepresentativeMemoryCacheTemporaryInvolvedIndividualDataLinker* tmpInvolvedIndiDataLinker = baAsWrDa->getTemporaryInvolvedIndividualIdDataLinker();
CBackendRepresentativeMemoryCacheTemporaryPropagationCutDataLinker* tmpPropCutDataLinker = baAsWrDa->getTemporaryPropagationCutDataLinker();
installTemporaryLabels(tempLabelWriteDataLinker, ontologyData);
installTemporaryCardinalities(tempCardWriteDataLinker, ontologyData);
mTmpIndiAssocPrevUpdateId = mNextIndiUpdateId;
mTmpIndiIndirectlyConnNomLabelItemHash.clear();
bool cached = false;
if (checkUpdateRejection(tempAssWriteDataLinker, ontologyData)) {
cached |= handleUpdateRejection(tempAssWriteDataLinker, ontologyData);
LOG(WARN, "::Konclude::RepresentativeBackendCache", logTr("Recjecting installation of update from recomputation id %1.").arg(mCurrentUpdateHandlingRecomputationId), this);
} else {
cached |= integratePropagationCut(tmpPropCutDataLinker, ontologyData);
analyseDeterministicSameAsAssociationInstallation(tempAssWriteDataLinker, ontologyData);
cached |= installAssociationUpdates(tempAssWriteDataLinker, ontologyData);
cached |= installNominalIndirectConncetionUpdates(tempNomIndirectConnDataLinker, ontologyData);
cached |= checkAssociationUsage(tempAssUseDataLinker, ontologyData);
cached |= updateInvolvedIndividuals(tmpInvolvedIndiDataLinker, ontologyData);
cached |= installDeterministicSameAsAssociationUpdates(tempAssWriteDataLinker, ontologyData);
mDeterministicSameHandlingInstallationDataHash.clear();
mPropagationCutIndiSet.clear();
if (!mTmpDetSameMergingCompletionReferenceHash.isEmpty()) {
cached |= completeDeterministicSameAsMergingInformation(mTmpDetSameMergingCompletionReferenceHash, ontologyData);
mTmpDetSameMergingCompletionReferenceHash.clear();
}
if (!mTmpCompleteNeighbourSameIndiMergingSet.isEmpty()) {
cached |= completeNeighboursForSameAsMerging(mTmpCompleteNeighbourSameIndiMergingSet, ontologyData);
mTmpCompleteNeighbourSameIndiMergingSet.clear();
}
if (mTmpPropCutIndiArrayNeighboursHandlingDataHash) {
for (QHash::const_iterator it = mTmpPropCutIndiArrayNeighboursHandlingDataHash->constBegin(), itEnd = mTmpPropCutIndiArrayNeighboursHandlingDataHash->constBegin(); it != itEnd; ++it) {
delete it.value();
}
delete mTmpPropCutIndiArrayNeighboursHandlingDataHash;
mTmpPropCutIndiArrayNeighboursHandlingDataHash = nullptr;
}
}
allCachingSuccess &= cached;
oneCachingSuccess |= cached;
oneCachingExpSuccess |= cached;
}
dataLinkerIt = (CBackendRepresentativeMemoryCacheWriteData*)dataLinkerIt->getNext();
}
if (!oneCachingSuccess) {
++mEmptyWriteDataCount;
}
bool forceCompletion = false;
if (mConfMaxCacheDataUpdateWritingCount > 0 && ontologyData->getCacheDataUpdateWritingCount() > mConfMaxCacheDataUpdateWritingCount) {
LOG(WARN, "::Konclude::RepresentativeBackendCache", logTr("Forcing association completion with reached maximum of %1 updates.").arg(mConfMaxCacheDataUpdateWritingCount), this);
forceCompletion = true;
}
checkAssociationComplete(ontologyData, forceCompletion);
bool basicPrecomputationModeActivated = false;
if (checkBasicPrecompuationModeActivation(ontologyData)) {
if (activateBasicPrecompuationMode(ontologyData)) {
basicPrecomputationModeActivated = true;
}
}
bool slotUpdate = false;
if (ontologyData->getNextSlotUpdateWaitingCount() <= 0 || ontologyData->isAssociationCompleted() || basicPrecomputationModeActivated) {
deleteExpiredIndividualAssociationMemoryContexts(ontologyData, &mContext);
createReaderSlotUpdate(ontologyData, &mContext);
cleanUnusedSlots(&mContext);
ontologyData->setNextSlotUpdateWaitingCount(mSlotUpdateWaitingIncreaseCount++);
if (mSlotUpdateWaitingMaxCount >= 0 && mSlotUpdateWaitingIncreaseCount > mSlotUpdateWaitingMaxCount) {
mSlotUpdateWaitingIncreaseCount = mSlotUpdateWaitingMaxCount;
}
slotUpdate = true;
} else {
ontologyData->setNextSlotUpdateWaitingCount(ontologyData->getNextSlotUpdateWaitingCount() - 1);
}
cint64 updatedIndiCount = mUpdatedIndiCount - prevUpdatedIndiCount;
cint64 directUpdatedIndiCount = mAssociationUpdatedIndiCount - prevAssociationUpdatedIndiCount;
QString updateType = QString("intern");
if (slotUpdate) {
updateType = QString("published");
}
QString pendingString;
if (mStatCollectStatistics) {
pendingString = QString(", %1 pending").arg(mPendingUpdateCount);
}
LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Update %1 (%5%6) for representative backend cache, updated %2 (%7 direct) of overall %3 individual associations, %4 remaining insufficiently handled individuals.")
.arg(mWriteDataCount).arg(updatedIndiCount).arg(ontologyData->getIndividualAssociationsCount()).arg(ontologyData->getIncompletelyHandledIndividualIdCount()).arg(updateType).arg(pendingString).arg(directUpdatedIndiCount), this);
if (mStatCollectStatistics) {
LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Overall %1 updated invididual associations (max %5), %2 created neighbour links (%3 only adding link updates, %4 changed/removed link udpates, %6 max links).")
.arg(mUpdatedIndiCount).arg(mStatCreatedNeighbourLinks).arg(mStatAddingNeighbourLinksAssociationUpdateCount).arg(mStatUpdatedOrRemovedNeighbourLinksAssociationUpdateCount).arg(mStatMaxAssociationUpdateCount).arg(mStatMaxNeighbourLinksCount), this);
LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Successfully installed %1 deterministic same association data (of %6 deterministic merges), failed installation for %2 (%3 due to incompletely handled destination, %5 due to merged destination, %4 due to different update id).")
.arg(mStatDetSameAssociationInstallCount).arg(mStatDetSameAssociationFailedCount).arg(mStatDetSameAssociationIncompleteHandledDestFailedCount).arg(mStatDetSameAssociationDifferentUpdateIdFailedCount).arg(mStatDetSameAssociationRepMergedDestFailedCount).arg(mStatDetSameRepresentativeMergingCount), this);
QStringList labelsStringList;
for (cint64 i = 0; i < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_TYPE_COUNT; ++i) {
QString labelName = getRepresentativeCacheLabelItemString(i);
QString labelString = QString("\t%2 of label type %1 (max size %3, all value %4)").arg(labelName).arg(mStatLabelTypeCount[i]).arg(mStatLabelTypeMaxValueCount[i]).arg(mStatLabelTypeAllValueCount[i]);
labelsStringList.append(labelString);
}
LOG(INFO, "::Konclude::RepresentativeBackendCache", QString("Containing %2 labels, %3 max label size, %4 max same merged individuals,\r\n%1").arg(labelsStringList.join(",\r\n"))
.arg(mStatLabelCount).arg(mStatMaxLabelValueCount).arg(mStatMaxSameAsMergedCount), this);
}
} else {
LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Ignoring update since individual label association is already complete."), this);
}
//mContext.releaseTemporaryMemoryPools(memoryPools);
mCacheStat.setMemoryConsumption(mContext.getMemoryConsumption());
//if (ontologyData->isFirstIncompletelyHandledIndividualsRetrieved() && (mWriteDataCount % 50 == 0)) {
// writeStringifiedRepresentativeCacheToFile();
//}
if (mLimitRemainingWritePending) {
mRemainingWritePendingSemaphore.release();
}
#ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
if (mConfDebugWriteRepresentativeCache && (mWriteDataCount <= 50 || mWriteDataCount % 500 == 0)) {
//writeStringifiedRepresentativeCacheToFile();
}
#endif
return true;
}
return false;
}
bool CBackendRepresentativeMemoryCache::checkBasicPrecompuationModeActivation(CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
if (!ontologyData->isAssociationCompleted() && !ontologyData->hasBasicPrecomputationModeActivation() && ontologyData->getIncompletelyHandledIndividualIdCount() > 0 && mConfBasicPrecomputationModeActivationUpdateMergesRatio > 0) {
double basicIndiUpdateCount = ontologyData->getIndividualAssociationDataDirectUpdateCount();
double indiMergesCount = ontologyData->getIndividualAssociationMergingCount();
if (indiMergesCount > 0) {
double ratio = indiMergesCount / basicIndiUpdateCount;
if (ratio > mConfBasicPrecomputationModeActivationUpdateMergesRatio) {
return true;
}
}
}
return false;
}
bool CBackendRepresentativeMemoryCache::activateBasicPrecompuationMode(CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
if (!ontologyData->isBasicPrecomputationMode() && !ontologyData->hasBasicPrecomputationModeActivation()) {
ontologyData->setBasicPrecomputationModeActivation(true);
ontologyData->setBasicPrecomputationMode(true);
CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
cint64 basicIndiUpdateCount = ontologyData->getIndividualAssociationDataDirectUpdateCount();
cint64 indiMergesCount = ontologyData->getIndividualAssociationMergingCount();
CBackendRepresentativeMemoryCacheIndividualAssociationData** indiIdAssoDataVector = ontologyData->getIndividualIdAssoiationDataVector();
cint64 indiIdAssoDataVectorSize = ontologyData->getIndividualIdAssoiationDataVectorSize();
CBackendRepresentativeMemoryCacheIndividualAssociationData** basicPrecompIndiIdAssoDataVector = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualAssociationData* >::allocateAndConstructArray(context->getMemoryAllocationManager(), indiIdAssoDataVectorSize);
for (cint64 i = 0; i < indiIdAssoDataVectorSize; ++i) {
basicPrecompIndiIdAssoDataVector[i] = indiIdAssoDataVector[i];
}
ontologyData->setBasicPrecomputationIndividualIdAssoiationDataVector(indiIdAssoDataVectorSize, basicPrecompIndiIdAssoDataVector);
LOG(INFO, "::Konclude::RepresentativeBackendCache", QString("Activated basic precompuation mode for %1 merges at %2 updates\r\n").arg(indiMergesCount).arg(basicIndiUpdateCount), this);
return true;
}
return false;
}
void CBackendRepresentativeMemoryCache::indexIndividualLabelAssociations(CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
cint64 indexingCount = 0;
for (cint64 i = 0; i < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_ASSOCIATABLE_TYPE_COUNT; ++i) {
if (requiresIndividualAssociations(i)) {
indexingCount++;
}
}
ontologyData->setIndividualLabelAssociationIndexingCount(indexingCount);
cint64 indiCount = ontologyData->getMaxStoredIndvidualiId();
cint64 vecSize = ontologyData->getIndividualIdAssoiationDataVectorSize();
LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Starting concurrent indexing of %1 label types for %2 individuals.").arg(indexingCount).arg(indiCount), this);
for (cint64 i = 0; i < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_ASSOCIATABLE_TYPE_COUNT; ++i) {
bool exactIndiAssocTracking = requiresIndividualAssociations(i);
if (exactIndiAssocTracking) {
CConcurrentTaskScheduler::run([=] {
CMemoryPoolNewAllocationIncreasingContext newAllocContext;
for (cint64 indiId = 0; indiId <= indiCount && indiId < vecSize; ++indiId) {
CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssData = ontologyData->getIndividualIdAssoiationDataVector()[indiId];
if (indiAssData) {
CBackendRepresentativeMemoryLabelCacheItem* labelItem = indiAssData->getLabelCacheEntry(i);
if (labelItem) {
CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData* indiAssoExtData = (CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData*)labelItem->getExtensionData(CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData::INDIVIDUAL_ASSOCIATION_MAP);
if (!indiAssoExtData) {
indiAssoExtData = CObjectParameterizingAllocator< CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData, CMemoryPoolNewAllocationIncreasingContext* >::allocateAndConstructAndParameterize(newAllocContext.getMemoryAllocationManager(), &newAllocContext);
labelItem->setExtensionData(CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData::INDIVIDUAL_ASSOCIATION_MAP, indiAssoExtData);
}
indiAssoExtData->addIndividualIdAssociation(indiAssData);
}
}
}
CMemoryPool* memoryPools = newAllocContext.takeMemoryPools();
cint64 remainingCount = ontologyData->updateIndividualLabelAssociationIndexed(true, memoryPools);
if (remainingCount <= 0) {
LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Finished indexing all label types for %1 individuals.").arg(indiCount), this);
} else {
LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Finished indexing %1 label types.").arg(indexingCount - remainingCount), this);
}
});
}
}
if (mConfWaitIndividualLabelAssociationIndexed) {
LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Waiting indexing finishing."), this);
ontologyData->waitIndividualLabelAssociationIndexed();
}
}
bool CBackendRepresentativeMemoryCache::integratePropagationCut(CBackendRepresentativeMemoryCacheTemporaryPropagationCutDataLinker* tmpPropCutDataLinker, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
bool updated = false;
if (tmpPropCutDataLinker) {
mLastHandledPropCutDataLinker = tmpPropCutDataLinker;
cint64 propCutIndiCount = 0;
cint64 propCutInfluencedIndiCount = 0;
cint64 propCutAllNeighbourRelationsInfluencingCount = 0;
cint64 propCutNeighbourNonDetermininisticRelationInfluencingCount = 0;
cint64 propCutNeighbourRelationPropagatedConceptMissingInfluencingCount = 0;
cint64 propCutNeighbourDirectConceptInfluenceUpdatingCount = 0;
cint64 propCutNeighbourRelationUnspecifiedInfluencingCount = 0;
CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
CBackendRepresentativeMemoryCacheIndividualAssociationData** indiAssDataVector = ontologyData->getIndividualIdAssoiationDataVector();
QSet allNeighbourRelationsConsideredIndiSet;
QHash incompletelyUpdatingNeighbourPropCutIndisHash;
QSet> incompletelyUpdatingNeighbourPropCutIndiPairSet;
QSet expandedIndiSet;
CXLinker* lastInvestigatedExpandedIndiLinker = nullptr;
QHash propagationCutIndiReferredNeighbourArrayIdHash;
for (CBackendRepresentativeMemoryCacheTemporaryPropagationCutDataLinker* tmpPropCutDataLinkerIt = tmpPropCutDataLinker; tmpPropCutDataLinkerIt; tmpPropCutDataLinkerIt = tmpPropCutDataLinkerIt->getNext()) {
CXLinker* expandedIndiLinker = tmpPropCutDataLinkerIt->getExpandedIndividualIdsLinker();
if (lastInvestigatedExpandedIndiLinker != expandedIndiLinker) {
lastInvestigatedExpandedIndiLinker = expandedIndiLinker;
expandedIndiSet.clear();
for (CXLinker* expandedIndiLinkerIt = expandedIndiLinker; expandedIndiLinkerIt; expandedIndiLinkerIt = expandedIndiLinkerIt->getNext()) {
cint64 expIndiId = expandedIndiLinkerIt->getData();
expandedIndiSet.insert(expIndiId);
}
}
cint64 propCutIndiId = tmpPropCutDataLinkerIt->getIndividualID();
mPropagationCutIndiSet.insert(propCutIndiId);
cint64 assocUpdateId = tmpPropCutDataLinkerIt->getAssociationUpdateId();
cint64 visitingCursor = tmpPropCutDataLinkerIt->getNeighbourPropagationCutCursor();
CCacheValue propConValue = tmpPropCutDataLinkerIt->getConceptPropagationValue();
bool missingNondeterministicExpansionPropagation = tmpPropCutDataLinkerIt->isMissingNondeterministicExpansionPropagation();
bool allNeighbourRelationsConsiderationRequired = false;
CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssocData = indiAssDataVector[propCutIndiId];
if (indiAssocData->getAssociationDataUpdateId() != assocUpdateId) {
allNeighbourRelationsConsiderationRequired = true;
}
if (propConValue.getIdentification() == 0) {
allNeighbourRelationsConsiderationRequired = true;
}
if (!allNeighbourRelationsConsiderationRequired || !allNeighbourRelationsConsideredIndiSet.contains(propCutIndiId)) {
if (allNeighbourRelationsConsiderationRequired) {
if (!missingNondeterministicExpansionPropagation) {
allNeighbourRelationsConsideredIndiSet.insert(propCutIndiId);
}
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* neighArray = indiAssocData->getRoleSetNeighbourArray();
for (cint64 i = 0; i < neighArray->getIndexData()->getArraySize(); ++i) {
bool hasNonExpanded = false;
neighArray->at(i).visitNeighbourIndividualIdsFromCursor([&](cint64 neighbourId, cint64 nextCursor)->bool {
if (!expandedIndiSet.contains(neighbourId)) {
hasNonExpanded = true;
CBackendRepresentativeMemoryCacheIndividualAssociationData* neighbourIndiAssocData = indiAssDataVector[neighbourId];
// set as incompletely handled and add non-deterministic top property link
bool neighbourPropagationAffected = false;
if (missingNondeterministicExpansionPropagation) {
CBackendRepresentativeMemoryLabelCacheItem* neighbourConSetLabel = neighbourIndiAssocData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::FULL_CONCEPT_SET_LABEL);
CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData* cardExtData = (CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData*)neighbourConSetLabel->getExtensionData(CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData::CARDINALITY_HASH);
if (cardExtData || neighbourConSetLabel->hasNondeterministicElements()) {
neighbourPropagationAffected = true;
}
} else {
neighbourPropagationAffected = true;
}
if (neighbourPropagationAffected && !incompletelyUpdatingNeighbourPropCutIndiPairSet.contains(QPair(neighbourId, propCutIndiId))) {
propCutAllNeighbourRelationsInfluencingCount++;
incompletelyUpdatingNeighbourPropCutIndiPairSet.insert(QPair(neighbourId, propCutIndiId));
incompletelyUpdatingNeighbourPropCutIndisHash.insertMulti(neighbourId, propCutIndiId);
}
}
return true;
}, 0);
if (hasNonExpanded) {
bool hasNonDetElements = neighArray->getIndexData()->getNeighbourRoleSetLabel(i)->hasNondeterministicElements();
if (!hasNonDetElements) {
propagationCutIndiReferredNeighbourArrayIdHash.insertMulti(propCutIndiId, i);
}
}
}
} else {
cint64 arrayIndexId = tmpPropCutDataLinkerIt->getArrayIndex();
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* neighArray = indiAssocData->getRoleSetNeighbourArray();
bool hasNonExpanded = false;
bool hasNonDetElements = neighArray->getIndexData()->getNeighbourRoleSetLabel(arrayIndexId)->hasNondeterministicElements();
neighArray->at(arrayIndexId).visitNeighbourIndividualIdsFromCursor([&](cint64 neighbourId, cint64 nextCursor)->bool {
if (!expandedIndiSet.contains(neighbourId)) {
hasNonExpanded = true;
CBackendRepresentativeMemoryCacheIndividualAssociationData* neighbourIndiAssocData = indiAssDataVector[neighbourId];
bool neighbourPropagationAffected = false;
if (hasNonDetElements) {
neighbourPropagationAffected = true;
++propCutNeighbourNonDetermininisticRelationInfluencingCount;
} else if (propConValue.getIdentification() != 0) {
CBackendRepresentativeMemoryLabelCacheItem* neighbourConSetLabel = neighbourIndiAssocData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::FULL_CONCEPT_SET_LABEL);
CBackendRepresentativeMemoryLabelValueLinker* cacheValueLinker = neighbourConSetLabel->getTagCacheValueHash(false)->value(propConValue.getTag());
if (!cacheValueLinker) {
// only for deterministically propagated concepts
if (mConfPropagationCutPropagatedConceptDirectInstallation && (propConValue.getCacheValueIdentifier() == CCacheValue::CACHEVALTAGANDCONCEPT || propConValue.getCacheValueIdentifier() == CCacheValue::CACHEVALTAGANDNEGATEDCONCEPT)) {
updatePropagationCutIndividualIncompletelyHandled(neighbourId, propConValue, ontologyData);
propCutNeighbourDirectConceptInfluenceUpdatingCount++;
++propCutInfluencedIndiCount;
updated = true;
} else {
neighbourPropagationAffected = true;
propCutNeighbourRelationPropagatedConceptMissingInfluencingCount++;
}
}
} else if (missingNondeterministicExpansionPropagation) {
CBackendRepresentativeMemoryLabelCacheItem* neighbourConSetLabel = neighbourIndiAssocData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::FULL_CONCEPT_SET_LABEL);
CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData* cardExtData = (CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData*)neighbourConSetLabel->getExtensionData(CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData::CARDINALITY_HASH);
if (cardExtData || neighbourConSetLabel->hasNondeterministicElements()) {
propCutNeighbourRelationUnspecifiedInfluencingCount++;
neighbourPropagationAffected = true;
}
} else {
propCutNeighbourRelationUnspecifiedInfluencingCount++;
neighbourPropagationAffected = true;
}
if (neighbourPropagationAffected) {
// set as incompletely handled and add non-deterministic top property link
if (!incompletelyUpdatingNeighbourPropCutIndiPairSet.contains(QPair(neighbourId, propCutIndiId))) {
incompletelyUpdatingNeighbourPropCutIndiPairSet.insert(QPair(neighbourId, propCutIndiId));
incompletelyUpdatingNeighbourPropCutIndisHash.insertMulti(neighbourId, propCutIndiId);
}
}
}
return true;
}, visitingCursor);
if (!hasNonDetElements && hasNonExpanded) {
propagationCutIndiReferredNeighbourArrayIdHash.insertMulti(propCutIndiId, arrayIndexId);
}
}
}
}
propCutIndiCount = mPropagationCutIndiSet.count();
cint64 lastPropCutIndiId = -1;
for (QHash::const_iterator it = propagationCutIndiReferredNeighbourArrayIdHash.constBegin(), itEnd = propagationCutIndiReferredNeighbourArrayIdHash.constEnd(); it != itEnd; ++it) {
cint64 propCutIndiId = it.key();
if (lastPropCutIndiId != propCutIndiId) {
lastPropCutIndiId = propCutIndiId;
QList referredNeighbourArrayIdList = propagationCutIndiReferredNeighbourArrayIdHash.values(propCutIndiId);
QSet referredNeighbourArrayIdSet = referredNeighbourArrayIdList.toSet();
CBackendRepresentativeMemoryCacheIndividualAssociationData* propCutIndiAssocData = indiAssDataVector[propCutIndiId];
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* neighArray = propCutIndiAssocData->getRoleSetNeighbourArray();
CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* arrayIndexData = neighArray->getIndexData();
CPropagationCutNeighbourArrayHandlingData* handlingData = nullptr;
for (cint64 i = 0; i < arrayIndexData->getArraySize(); ++i) {
bool hasExpandableNeighbours = referredNeighbourArrayIdSet.contains(i);
CBackendRepresentativeMemoryLabelCacheItem* neighbourLabelItem = neighArray->getIndexData()->getNeighbourRoleSetLabel(i);
bool hasNonDetElements = neighbourLabelItem->hasNondeterministicElements();
if (!hasExpandableNeighbours) {
neighArray->at(i).visitNeighbourIndividualIdsFromCursor([&](cint64 neighbourId, cint64 nextCursor)->bool {
if (!expandedIndiSet.contains(neighbourId)) {
hasExpandableNeighbours = true;
return false;
}
return true;
}, 0);
}
if (hasExpandableNeighbours) {
if (!mTmpPropCutIndiArrayNeighboursHandlingDataHash) {
mTmpPropCutIndiArrayNeighboursHandlingDataHash = new QHash();
}
if (!handlingData) {
CPropagationCutNeighbourArrayHandlingData*& tmpHandlingData = (*mTmpPropCutIndiArrayNeighboursHandlingDataHash)[propCutIndiId];
if (!tmpHandlingData) {
tmpHandlingData = new CPropagationCutNeighbourArrayHandlingData();
}
handlingData = tmpHandlingData;
}
if (handlingData) {
if (!hasNonDetElements) {
handlingData->mReaddingArrayPosSet.insert(i);
} else if (neighbourLabelItem->getCacheValueCount() > 1 || neighbourLabelItem->getCacheValueLinker()->getCacheValue().getTag() != 1) {
bool allNondeterministic = true;
for (CBackendRepresentativeMemoryLabelValueLinker* cacheValueLinkerIt = neighbourLabelItem->getCacheValueLinker(); cacheValueLinkerIt && allNondeterministic; cacheValueLinkerIt = cacheValueLinkerIt->getNext()) {
if (!isCacheValueRoleNondeterministic(cacheValueLinkerIt->getCacheValue())) {
allNondeterministic = false;
}
}
if (allNondeterministic) {
handlingData->mRemovalArrayPosSet.insert(i);
} else {
handlingData->mReductionArrayPosSet.insert(i);
}
}
}
}
}
}
}
if (!incompletelyUpdatingNeighbourPropCutIndisHash.isEmpty()) {
CBackendRepresentativeMemoryLabelCacheItem* propMarkLabelItem = ontologyData->getPrioritizedPropagationMarkedNeighbourLabelItem();
CCacheValue propMarkRoleCacheValue = tmpPropCutDataLinker->getPropagationMarkingRoleValue();
if (!propMarkLabelItem) {
CBackendRepresentativeMemoryLabelSignatureResolveCacheItem& sigResolCacheItem = (*ontologyData->getSignatureLabelItemHash(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_LABEL))[propMarkRoleCacheValue.getTag()];
for (CBackendRepresentativeMemoryLabelCacheItem* labelItemLinkerIt = sigResolCacheItem.getLabelItems(); labelItemLinkerIt && !propMarkLabelItem; labelItemLinkerIt = labelItemLinkerIt->getNext()) {
if (labelItemLinkerIt->getCacheValueCount() == 1) {
if (labelItemLinkerIt->getCacheValueLinker() && labelItemLinkerIt->getCacheValueLinker()->getCacheValue() == propMarkRoleCacheValue) {
propMarkLabelItem = labelItemLinkerIt;
}
}
}
if (!propMarkLabelItem) {
propMarkLabelItem = CObjectParameterizingAllocator< CBackendRepresentativeMemoryLabelCacheItem, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
propMarkLabelItem->initCacheEntry(propMarkRoleCacheValue.getTag(), ontologyData->getNextEntryID(), CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_LABEL);
CCACHINGHASH* labelValueHash = propMarkLabelItem->getTagCacheValueHash(true);
CBackendRepresentativeMemoryLabelValueLinker* newValueLinker = CObjectAllocator< CBackendRepresentativeMemoryLabelValueLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
newValueLinker->initLabelValueLinker(propMarkRoleCacheValue);
labelValueHash->insert(propMarkRoleCacheValue.getTag(), newValueLinker);
propMarkLabelItem->addCacheValueLinker(newValueLinker);
sigResolCacheItem.appendLabelItem(propMarkLabelItem);
ontologyData->setPrioritizedPropagationMarkedNeighbourLabelItem(propMarkLabelItem);
}
}
cint64 lastNeighbour = -1;
for (QHash::const_iterator it = incompletelyUpdatingNeighbourPropCutIndisHash.constBegin(), itEnd = incompletelyUpdatingNeighbourPropCutIndisHash.constEnd(); it != itEnd; ++it) {
cint64 neighbourId = it.key();
if (lastNeighbour != neighbourId) {
lastNeighbour = neighbourId;
QList propCutIndiIds = incompletelyUpdatingNeighbourPropCutIndisHash.values(neighbourId);
if (updatePropagationCutIndividualIncompletelyHandled(neighbourId, propCutIndiIds, propMarkLabelItem, ontologyData)) {
++propCutInfluencedIndiCount;
updated = true;
}
}
}
}
QString propCutIndisString = QString("%1 individuals").arg(propCutIndiCount);
bool debuggingIndi = false;
if (mStatCollectStatistics && propCutIndiCount < 5) {
QStringList propCutIndStringList;
for (cint64 propCutIndiId : mPropagationCutIndiSet) {
QString indiString = QString::number(propCutIndiId);
#ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssocData = indiAssDataVector[propCutIndiId];
if (indiAssocData->mDebugIndi) {
QString indiName = CIRIName::getRecentIRIName(indiAssocData->mDebugIndi->getIndividualNameLinker());
indiString = QString("%2 with id %1").arg(propCutIndiId).arg(indiName);
}
#endif
propCutIndStringList.append(indiString);
}
propCutIndisString += QString(" (%1)").arg(propCutIndStringList.join(", "));
}
#ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
if (debuggingIndi) {
QStringList modelStringList = getRepresentativeCacheString();
modelStringList.append(QString("\r\n\r\nReceived propagation cut for %1 influencing %2 neighbour individuals (%3 from all relations, %4 due to non deterministic relations, %5 due to propagated concept missing, %7 direct added propagated concept, %6 unspecified)\r\n")
.arg(propCutIndisString).arg(propCutInfluencedIndiCount).arg(propCutAllNeighbourRelationsInfluencingCount).arg(propCutNeighbourNonDetermininisticRelationInfluencingCount).arg(propCutNeighbourRelationPropagatedConceptMissingInfluencingCount).arg(propCutNeighbourRelationUnspecifiedInfluencingCount).arg(propCutNeighbourDirectConceptInfluenceUpdatingCount));
for (QHash::const_iterator it = incompletelyUpdatingNeighbourPropCutIndisHash.constBegin(), itEnd = incompletelyUpdatingNeighbourPropCutIndisHash.constEnd(); it != itEnd; ++it) {
modelStringList.append(QString("propagation cut individual %1 influenced %2\r\n").arg(it.key()).arg(it.value()));
}
QFile modelFile(QString("./Debugging/RepresentativeCache/individual-label-association-data-%1-propagation-cut.txt").arg(mWriteDataCount));
if (modelFile.open(QIODevice::WriteOnly)) {
for (const QString& modelString : modelStringList) {
QByteArray data = modelString.toUtf8();
modelFile.write(data);
}
modelFile.close();
}
}
#endif
LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Received propagation cut for %1 influencing %2 neighbour individuals (%3 from all relations, %4 due to non deterministic relations, %5 due to propagated concept missing, %7 direct added propagated concept, %6 unspecified).")
.arg(propCutIndisString).arg(propCutInfluencedIndiCount).arg(propCutAllNeighbourRelationsInfluencingCount).arg(propCutNeighbourNonDetermininisticRelationInfluencingCount).arg(propCutNeighbourRelationPropagatedConceptMissingInfluencingCount).arg(propCutNeighbourRelationUnspecifiedInfluencingCount).arg(propCutNeighbourDirectConceptInfluenceUpdatingCount), this);
}
return updated;
}
bool CBackendRepresentativeMemoryCache::updatePropagationCutIndividualIncompletelyHandled(cint64 neighbourId, const CCacheValue& propConValue, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
CBackendRepresentativeMemoryCacheIndividualAssociationData** indiAssDataVector = ontologyData->getIndividualIdAssoiationDataVector();
CBackendRepresentativeMemoryCacheIndividualAssociationData* neighbourIndiAssocData = indiAssDataVector[neighbourId];
CBackendRepresentativeMemoryCacheIndividualAssociationData* locNeighbourAssociationData = createLocalizedIndividualAssociationData(neighbourIndiAssocData->getAssociatedIndividualId(), neighbourIndiAssocData, ontologyData, false);
CBackendRepresentativeMemoryCacheContext* context = getIndividualAssociationDataMemoryContext(locNeighbourAssociationData, ontologyData);
#ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
locNeighbourAssociationData->mDebugUpdateFunction = QString("updatePropagationCutIndividualIncompletelyHandled");
locNeighbourAssociationData->mDebugUpdateRecomputationId = mCurrentUpdateHandlingRecomputationId;
#endif
locNeighbourAssociationData->setNeighbourRoleSetHash(neighbourIndiAssocData->getNeighbourRoleSetHash());
locNeighbourAssociationData->setRoleSetNeighbourArray(neighbourIndiAssocData->getRoleSetNeighbourArray());
CBackendRepresentativeMemoryLabelCacheItem* conSetLabel = neighbourIndiAssocData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::FULL_CONCEPT_SET_LABEL);
CBackendRepresentativeMemoryLabelCacheItem* newConSetLabel = getExtendedLabel(CBackendRepresentativeMemoryLabelCacheItem::FULL_CONCEPT_SET_LABEL, conSetLabel, propConValue, ontologyData);
locNeighbourAssociationData->setLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::FULL_CONCEPT_SET_LABEL, newConSetLabel);
locNeighbourAssociationData->setCacheUpdateId(mNextIndiUpdateId++);
ontologyData->setMaxIndividualAssociationDataUpdateCount(qMax(locNeighbourAssociationData->getAssociationDataUpdateId(), ontologyData->getMaxIndividualAssociationDataUpdateCount()));
++mUpdatedIndiCount;
if (locNeighbourAssociationData->isCompletelyHandled() && locNeighbourAssociationData->hasRepresentativeSameIndividualMerging()) {
markRepresentativeReferencedIndividualAssociationIncompletelyHandled(neighbourId, locNeighbourAssociationData, ontologyData);
} else if (locNeighbourAssociationData->isCompletelyHandled()) {
locNeighbourAssociationData->setCompletelyHandled(false);
storeIndividualIncompletelyMarked(locNeighbourAssociationData, true, ontologyData);
}
setUpdatedIndividualAssociationData(locNeighbourAssociationData->getAssociatedIndividualId(), locNeighbourAssociationData, ontologyData);
return true;
}
bool CBackendRepresentativeMemoryCache::updatePropagationCutIndividualIncompletelyHandled(cint64 neighbourId, const QList& propCutIndiIdList, CBackendRepresentativeMemoryLabelCacheItem* propMarkLabelItem, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
CBackendRepresentativeMemoryCacheIndividualAssociationData** indiAssDataVector = ontologyData->getIndividualIdAssoiationDataVector();
CBackendRepresentativeMemoryCacheIndividualAssociationData* neighbourIndiAssocData = indiAssDataVector[neighbourId];
if (!neighbourIndiAssocData->hasRepresentativeSameIndividualMerging()) {
CBackendRepresentativeMemoryCacheIndividualAssociationData* locNeighbourAssociationData = createLocalizedIndividualAssociationData(neighbourIndiAssocData->getAssociatedIndividualId(), neighbourIndiAssocData, ontologyData, false);
bool requiresDataCopying = false;
CBackendRepresentativeMemoryCacheContext* context = getIndividualAssociationDataMemoryContext(locNeighbourAssociationData, ontologyData, &requiresDataCopying);
#ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
locNeighbourAssociationData->mDebugUpdateFunction = QString("updatePropagationCutIndividualIncompletelyHandled");
locNeighbourAssociationData->mDebugUpdateRecomputationId = mCurrentUpdateHandlingRecomputationId;
#endif
CBackendRepresentativeMemoryLabelCacheItem* neighbourRoleSetCompLabel = neighbourIndiAssocData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL);
CBackendRepresentativeMemoryLabelValueLinker* propMarkTagCacheValueLinker = neighbourRoleSetCompLabel->getTagCacheValueHash()->value(propMarkLabelItem->getCacheValueLinker()->getData().getTag());
CBackendRepresentativeMemoryLabelCacheItem* newNeighbourRoleSetCompLabel = neighbourRoleSetCompLabel;
if (!propMarkTagCacheValueLinker || propMarkTagCacheValueLinker->getData() != propMarkLabelItem->getCacheValueLinker()->getData()) {
CCacheValue extendingCacheValue;
extendingCacheValue.initCacheValue(propMarkLabelItem->getCacheEntryID(), (cint64)propMarkLabelItem, CCacheValue::CACHE_VALUE_TAG_AND_ENTRY);
newNeighbourRoleSetCompLabel = getExtendedLabel(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL, neighbourRoleSetCompLabel, extendingCacheValue, ontologyData);
}
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* newArray = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash* newNeighbourRoleSetHash = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* prevArrayData = neighbourIndiAssocData->getRoleSetNeighbourArray();
CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* prevArrayIndexData = prevArrayData->getIndexData();
CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* newArrayIndexData = prevArrayIndexData;
if (newNeighbourRoleSetCompLabel == neighbourRoleSetCompLabel) {
newArray->initNeighbourArray(neighbourIndiAssocData->getRoleSetNeighbourArray());
} else {
locNeighbourAssociationData->setLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL, newNeighbourRoleSetCompLabel);
updateIndexedAssociationCount(locNeighbourAssociationData, neighbourIndiAssocData, CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL, ontologyData);
newArrayIndexData = getIndividualNeighbourArrayIndexExtensionData(newNeighbourRoleSetCompLabel, ontologyData);
newArray->initNeighbourArray(newArrayIndexData);
for (cint64 i = 0; i < newArrayIndexData->getArraySize(); ++i) {
CBackendRepresentativeMemoryLabelCacheItem* referredLabelCacheItem = newArrayIndexData->getNeighbourRoleSetLabel(i);
cint64 prevIndex = prevArrayIndexData->getIndex(referredLabelCacheItem);
if (prevIndex >= 0) {
newArray->at(i) = prevArrayData->at(prevIndex);
}
}
}
QSet existingPropCutIndiSet;
cint64 propMarkIndex = newArrayIndexData->getIndex(propMarkLabelItem);
newArray->at(propMarkIndex).visitNeighbourIndividualIds([&](cint64 indi)->bool {
existingPropCutIndiSet.insert(indi);
return true;
});
for (cint64 propCutIndi : propCutIndiIdList) {
if (!existingPropCutIndiSet.contains(propCutIndi)) {
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newLinker = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
++mStatCreatedNeighbourLinks;
newLinker->initIndividualIdLinker(propCutIndi);
newArray->at(propMarkIndex).addIndividualIdLinker(newLinker);
existingPropCutIndiSet.insert(propCutIndi);
}
}
newNeighbourRoleSetHash->initNeighbourRoleSetHash(neighbourIndiAssocData->getNeighbourRoleSetHash(), requiresDataCopying);
locNeighbourAssociationData->setNeighbourRoleSetHash(newNeighbourRoleSetHash);
locNeighbourAssociationData->setRoleSetNeighbourArray(newArray);
locNeighbourAssociationData->setCacheUpdateId(mNextIndiUpdateId++);
ontologyData->setMaxIndividualAssociationDataUpdateCount(qMax(locNeighbourAssociationData->getAssociationDataUpdateId(), ontologyData->getMaxIndividualAssociationDataUpdateCount()));
++mUpdatedIndiCount;
locNeighbourAssociationData->setCompletelyHandled(false);
storeIndividualIncompletelyMarked(locNeighbourAssociationData, true, ontologyData);
setUpdatedIndividualAssociationData(locNeighbourAssociationData->getAssociatedIndividualId(), locNeighbourAssociationData, ontologyData);
return true;
}
return false;
}
bool CBackendRepresentativeMemoryCache::updateInvolvedIndividuals(CBackendRepresentativeMemoryCacheTemporaryInvolvedIndividualDataLinker* tmpInvolvedIndiDataLinker, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
bool updated = false;
if (tmpInvolvedIndiDataLinker) {
QSet newInvolvedIndiSet;
QSet addedInvolvedIndiStatusSet;
cint64 involvedIndiCount = 0;
cint64 newInvolvedIndiCount = 0;
cint64 involvedIndiUpdatingCount = 0;
CBackendRepresentativeMemoryCacheIndividualAssociationData** indiAssDataVector = ontologyData->getIndividualIdAssoiationDataVector();
for (CXLinker* indiLinkerIt = tmpInvolvedIndiDataLinker->getInvolvedIndividualIdsLinker(); indiLinkerIt; indiLinkerIt = indiLinkerIt->getNext()) {
cint64 indiId = indiLinkerIt->getData();
newInvolvedIndiSet.insert(indiId);
CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssocData = indiAssDataVector[indiId];
bool addedInvolvedStatus = false;
if (!indiAssocData->hasProblematicLevel()) {
ontologyData->incInvolvedIndividualCount();
addedInvolvedStatus = true;
addedInvolvedIndiStatusSet.insert(indiId);
++newInvolvedIndiCount;
}
indiAssocData->incProblematicLevel();
++involvedIndiCount;
}
QSet neighbourInvolvedIndiArrayUpdateSet;
for (CXLinker* indiLinkerIt = tmpInvolvedIndiDataLinker->getInvolvedIndividualIdsLinker(); indiLinkerIt; indiLinkerIt = indiLinkerIt->getNext()) {
cint64 indiId = indiLinkerIt->getData();
if (addedInvolvedIndiStatusSet.contains(indiId)) {
CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssocData = indiAssDataVector[indiId];
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* roleAssNeighArray = indiAssocData->getRoleSetNeighbourArray();
CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* arrayIndexData = roleAssNeighArray->getIndexData();
if (arrayIndexData) {
for (cint64 i = 0; i < arrayIndexData->getArraySize(); ++i) {
for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* neighbourIndiLinkerIt = roleAssNeighArray->at(i).getIndividualIdLinker(); neighbourIndiLinkerIt; neighbourIndiLinkerIt = neighbourIndiLinkerIt->getNext()) {
cint64 neighbourIndiId = neighbourIndiLinkerIt->getIndividualId();
neighbourInvolvedIndiArrayUpdateSet.insert(neighbourIndiId);
}
}
}
for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* neighbourIndiLinkerIt = indiAssocData->getPropagationCutRemovedNeighbourIndividualLinker(); neighbourIndiLinkerIt; neighbourIndiLinkerIt = neighbourIndiLinkerIt->getNext()) {
cint64 neighbourIndiId = neighbourIndiLinkerIt->getIndividualId();
neighbourInvolvedIndiArrayUpdateSet.insert(neighbourIndiId);
}
}
}
for (QSet::const_iterator it = neighbourInvolvedIndiArrayUpdateSet.constBegin(), itEnd = neighbourInvolvedIndiArrayUpdateSet.constEnd(); it != itEnd; ++it) {
cint64 indiId = *it;
++involvedIndiUpdatingCount;
CBackendRepresentativeMemoryCacheIndividualAssociationData* prevIndiAssocData = indiAssDataVector[indiId];
CBackendRepresentativeMemoryCacheIndividualAssociationData* locAssociationData = createLocalizedIndividualAssociationData(prevIndiAssocData->getAssociatedIndividualId(), prevIndiAssocData, ontologyData, false);
bool requiresDataCopying = false;
CBackendRepresentativeMemoryCacheContext* context = getIndividualAssociationDataMemoryContext(locAssociationData, ontologyData, &requiresDataCopying);
#ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
locAssociationData->mDebugUpdateFunction = QString("updateInvolvedIndividuals");
locAssociationData->mDebugUpdateRecomputationId = mCurrentUpdateHandlingRecomputationId;
#endif
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* prevArrayData = prevIndiAssocData->getRoleSetNeighbourArray();
CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* prevArrayIndexData = prevArrayData->getIndexData();
CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* newArrayIndexData = prevArrayIndexData;
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* newArray = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash* newNeighbourRoleSetHash = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
newArray->initNeighbourArray(newArrayIndexData);
newNeighbourRoleSetHash->initNeighbourRoleSetHash(prevIndiAssocData->getNeighbourRoleSetHash(), requiresDataCopying);
for (cint64 i = 0; i < newArrayIndexData->getArraySize(); ++i) {
QSet updatingInolvedNeighbourIndiSet;
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* firstNewLinker = nullptr;
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* lastNewLinker = nullptr;
prevArrayData->at(i).visitNeighbourIndividualIds([&](cint64 neighbourIndiId)->bool {
if (!addedInvolvedIndiStatusSet.contains(neighbourIndiId)) {
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newLinker = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
++mStatCreatedNeighbourLinks;
newLinker->initIndividualIdLinker(neighbourIndiId);
if (lastNewLinker) {
lastNewLinker->append(newLinker);
lastNewLinker = newLinker;
} else {
firstNewLinker = newLinker;
lastNewLinker = newLinker;
}
} else {
updatingInolvedNeighbourIndiSet.insert(neighbourIndiId);
}
return true;
});
// add problematic/involved neighbour ids to the beginning
for (QSet::const_iterator updateIt = updatingInolvedNeighbourIndiSet.constBegin(), updateItEnd = updatingInolvedNeighbourIndiSet.constEnd(); updateIt != updateItEnd; ++updateIt) {
cint64 updatingIndiId = *updateIt;
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newLinker = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
++mStatCreatedNeighbourLinks;
newLinker->initIndividualIdLinker(updatingIndiId);
firstNewLinker = newLinker->append(firstNewLinker);
}
if (firstNewLinker) {
newArray->at(i).addIndividualIdLinker(firstNewLinker);
}
}
locAssociationData->setNeighbourRoleSetHash(newNeighbourRoleSetHash);
locAssociationData->setRoleSetNeighbourArray(newArray);
locAssociationData->setProblematicLeveledNeigbour(true);
locAssociationData->setCacheUpdateId(mNextIndiUpdateId++);
ontologyData->setMaxIndividualAssociationDataUpdateCount(qMax(locAssociationData->getAssociationDataUpdateId(), ontologyData->getMaxIndividualAssociationDataUpdateCount()));
++mUpdatedIndiCount;
storeIndividualIncompletelyMarked(locAssociationData, !locAssociationData->isCompletelyHandled(), ontologyData);
setUpdatedIndividualAssociationData(locAssociationData->getAssociatedIndividualId(), locAssociationData, ontologyData);
updated = true;
}
LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Received clash influence for %1 individuals (new for %2 individuals), reordered neighbours for %3 individuals.").arg(involvedIndiCount).arg(newInvolvedIndiCount).arg(involvedIndiUpdatingCount), this);
}
return updated;
}
void CBackendRepresentativeMemoryCache::writeStringifiedRepresentativeCacheToFile() {
QStringList modelStringList = getRepresentativeCacheString();
QFile modelFileLatest("./Debugging/RepresentativeCache/individual-label-association-data.txt");
if (modelFileLatest.open(QIODevice::WriteOnly)) {
for (const QString& modelString : modelStringList) {
QByteArray data = modelString.toUtf8();
modelFileLatest.write(data);
}
modelFileLatest.close();
}
QFile modelFile(QString("./Debugging/RepresentativeCache/individual-label-association-data-%1.txt").arg(mWriteDataCount));
if (modelFile.open(QIODevice::WriteOnly)) {
for (const QString& modelString : modelStringList) {
QByteArray data = modelString.toUtf8();
modelFile.write(data);
}
modelFile.close();
}
#ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
QFile handlingFile(QString("./Debugging/RepresentativeCache/handling-data.txt"));
if (handlingFile.open(QIODevice::WriteOnly)) {
for (const QString& modelString : mDebugHandlingStrings) {
QByteArray data = modelString.toUtf8();
handlingFile.write(data);
handlingFile.write(QString("\r\n").toUtf8());
}
handlingFile.close();
}
#endif
}
QString CBackendRepresentativeMemoryCache::getRepresentativeCacheLabelItemString(cint64 labelType) {
QString labelTypeString;
if (labelType == CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_CONCEPT_SET_LABEL) {
labelTypeString = "DETERMINISTIC_CONCEPT_SET_LABEL";
}
else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::NONDETERMINISTIC_CONCEPT_SET_LABEL) {
labelTypeString = "NONDETERMINISTIC_CONCEPT_SET_LABEL";
}
else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::FULL_CONCEPT_SET_LABEL) {
labelTypeString = "FULL_CONCEPT_SET_LABEL";
}
else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_COMBINED_EXISTENTIAL_INSTANTIATED_ROLE_SET_LABEL) {
labelTypeString = "DETERMINISTIC_COMBINED_EXISTENTIAL_INSTANTIATED_ROLE_SET_LABEL";
}
else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::NONDETERMINISTIC_COMBINED_EXISTENTIAL_INSTANTIATED_ROLE_SET_LABEL) {
labelTypeString = "NONDETERMINISTIC_COMBINED_EXISTENTIAL_INSTANTIATED_ROLE_SET_LABEL";
}
else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_COMBINED_NEIGHBOUR_INSTANTIATED_ROLE_SET_LABEL) {
labelTypeString = "DETERMINISTIC_COMBINED_NEIGHBOUR_INSTANTIATED_ROLE_SET_LABEL";
}
else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::NONDETERMINISTIC_COMBINED_NEIGHBOUR_INSTANTIATED_ROLE_SET_LABEL) {
labelTypeString = "NONDETERMINISTIC_COMBINED_NEIGHBOUR_INSTANTIATED_ROLE_SET_LABEL";
}
else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL) {
labelTypeString = "NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL";
}
else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL) {
labelTypeString = "DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL";
}
else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::NONDETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL) {
labelTypeString = "NONDETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL";
}
else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_DIFFRENT_INDIVIDUAL_SET_LABEL) {
labelTypeString = "DETERMINISTIC_DIFFRENT_INDIVIDUAL_SET_LABEL";
}
else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::NONDETERMINISTIC_DIFFRENT_INDIVIDUAL_SET_LABEL) {
labelTypeString = "NONDETERMINISTIC_DIFFRENT_INDIVIDUAL_SET_LABEL";
}
else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::INDIRECTLY_CONNECTED_NOMINAL_INDIVIDUAL_SET_LABEL) {
labelTypeString = "INDIRECTLY_CONNECTED_NOMINAL_INDIVIDUAL_SET_LABEL";
}
else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_LABEL) {
labelTypeString = "NEIGHBOUR_INSTANTIATED_ROLE_SET_LABEL";
}
else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_COMBINED_DATA_INSTANTIATED_ROLE_SET_LABEL) {
labelTypeString = "DETERMINISTIC_COMBINED_DATA_INSTANTIATED_ROLE_SET_LABEL";
}
else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::NONDETERMINISTIC_COMBINED_DATA_INSTANTIATED_ROLE_SET_LABEL) {
labelTypeString = "NONDETERMINISTIC_COMBINED_DATA_INSTANTIATED_ROLE_SET_LABEL";
}
return labelTypeString;
}
QStringList CBackendRepresentativeMemoryCache::getRepresentativeCacheString() {
QStringList cacheStringList;
for (CCACHINGHASH::const_iterator it = mOntologyIdentifierDataHash->constBegin(), itEnd = mOntologyIdentifierDataHash->constEnd(); it != itEnd; ++it) {
cint64 ontId = it.key();
CBackendRepresentativeMemoryCacheOntologyData* ontologyData = it.value();
cacheStringList.append(QString("Ontology: %1\r\n\r\n\r\n").arg(ontId));
QStringList ontologyIdDataStringList = getRepresentativeCacheString(ontologyData);
cacheStringList += ontologyIdDataStringList;
cacheStringList.append(QString("\r\n\r\n\r\n\r\n\r\n\r\n"));
}
return cacheStringList;
}
void CBackendRepresentativeMemoryCache::debugCheckWriteMaxSameMergedIndividualsToFile(CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
CBackendRepresentativeMemoryLabelCacheItem* maxSameMergedItem = nullptr;
cint64 maxSameMergedCount = 0;
CCACHINGHASH* sigLabelItemHash = ontologyData->getSignatureLabelItemHash(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL);
for (CCACHINGHASH::const_iterator it = sigLabelItemHash->constBegin(), itEnd = sigLabelItemHash->constEnd(); it != itEnd; ++it) {
CBackendRepresentativeMemoryLabelSignatureResolveCacheItem sigResolveItem = it.value();
for (CBackendRepresentativeMemoryLabelCacheItem* labelItemLinkerIt = sigResolveItem.getLabelItems(); labelItemLinkerIt; labelItemLinkerIt = labelItemLinkerIt->getNext()) {
cint64 itemId = labelItemLinkerIt->getCacheEntryID();
if (labelItemLinkerIt->getCacheValueCount() > maxSameMergedCount) {
maxSameMergedCount = labelItemLinkerIt->getCacheValueCount();
maxSameMergedItem = labelItemLinkerIt;
}
}
}
if (maxSameMergedItem) {
QSet indiSet;
CBackendRepresentativeMemoryLabelValueLinker* labelValueLinker = maxSameMergedItem->getCacheValueLinker();
for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = labelValueLinker; labelValueLinkerIt; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
CCacheValue cacheValue = labelValueLinkerIt->getCacheValue();
cint64 tag = cacheValue.getTag();
CIndividual* individual = (CIndividual*)mDebugOntology->getABox()->getIndividualVector()->getData(tag);
QString indiName = CIRIName::getRecentIRIName(individual->getNameLinker());
indiSet.insert(indiName);
}
QSet indi1Set = indiSet;
QSet indi2Set;
QFile file2("./Debugging/RepresentativeCache/max-deterministically-merged-data.txt");
cint64 count2 = 0;
if (file2.open(QIODevice::ReadOnly)) {
while (!file2.atEnd()) {
QString line = file2.readLine().trimmed();
indi2Set.insert(line);
}
}
file2.close();
for (QString indi2 : indi2Set) {
if (!indi1Set.contains(indi2)) {
bool debug = true;
}
}
for (QString indi1 : indi1Set) {
if (!indi2Set.contains(indi1)) {
bool debug = true;
}
}
bool writeFile = false;
if (writeFile) {
QFile sameAsMergedFile("./Debugging/RepresentativeCache/max-deterministically-merged-data.txt");
if (sameAsMergedFile.open(QIODevice::WriteOnly)) {
for (const QString& indiString : indiSet) {
QByteArray data = indiString.toUtf8();
sameAsMergedFile.write(data);
sameAsMergedFile.write("\r\n");
}
}
sameAsMergedFile.close();
}
}
}
QStringList CBackendRepresentativeMemoryCache::getRepresentativeCacheString(CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
QStringList cacheStringList;
QStringList labelStringList;
for (cint64 labelType = 0; labelType < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_TYPE_COUNT; ++labelType) {
QString labelTypeString = getRepresentativeCacheLabelItemString(labelType);
labelTypeString = QString("\r\n\r\n%1:\r\n").arg(labelTypeString);
CCACHINGHASH* sigLabelItemHash = ontologyData->getSignatureLabelItemHash(labelType);
for (CCACHINGHASH::const_iterator it = sigLabelItemHash->constBegin(), itEnd = sigLabelItemHash->constEnd(); it != itEnd; ++it) {
cint64 signature = it.key();
CBackendRepresentativeMemoryLabelSignatureResolveCacheItem sigResolveItem = it.value();
for (CBackendRepresentativeMemoryLabelCacheItem* labelItemLinkerIt = sigResolveItem.getLabelItems(); labelItemLinkerIt; labelItemLinkerIt = labelItemLinkerIt->getNext()) {
cint64 itemId = labelItemLinkerIt->getCacheEntryID();
QString labelString = QString("Label %1 (with signature %2, association count %3): ").arg(itemId).arg(signature).arg(labelItemLinkerIt->getIndividualAssociationCount());
CBackendRepresentativeMemoryLabelValueLinker* labelValueLinker = labelItemLinkerIt->getCacheValueLinker();
for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = labelValueLinker; labelValueLinkerIt; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
CCacheValue cacheValue = labelValueLinkerIt->getCacheValue();
if (labelValueLinkerIt != labelValueLinker) {
labelString += ", ";
}
labelString += QString("%1(%2)").arg(cacheValue.getTag()).arg((cint64)cacheValue.getCacheValueIdentifier());
}
if (labelType == CBackendRepresentativeMemoryLabelCacheItem::FULL_CONCEPT_SET_LABEL) {
CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData* cardExtData = (CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData*)labelItemLinkerIt->getExtensionData(CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData::CARDINALITY_HASH);
if (cardExtData) {
QString cardDataString;
CCACHINGHASH* roleTagCardDataHash = cardExtData->getRoleCardinalityDataHash();
for (CCACHINGHASH::const_iterator it = roleTagCardDataHash->constBegin(), itEnd = roleTagCardDataHash->constEnd(); it != itEnd; ++it) {
cint64 roleTag = it.key();
CBackendRepresentativeMemoryLabelCacheItemCardinalityData* cardData = it.value();
if (!cardDataString.isEmpty()) {
cardDataString += "; ";
}
cardDataString += QString("%1 -> # %2 with <= %3").arg(roleTag).arg(cardData->getExistentialMaxUsedCardinality()).arg(cardData->getMinimumRestrictingCardinality());
}
labelString += QString("\r\n\t\t%1").arg(cardDataString);
}
}
labelTypeString += QString("%1\r\n").arg(labelString);
}
}
labelStringList.append(labelTypeString);
}
QStringList indiStringList;
for (cint64 i = 0; i <= ontologyData->getMaxStoredIndvidualiId() && i < ontologyData->getIndividualIdAssoiationDataVectorSize(); ++i) {
//for (CCACHINGHASH::const_iterator it = mIndiIdAssoDataHash->constBegin(), itEnd = mIndiIdAssoDataHash->constEnd(); it != itEnd; ++it) {
cint64 indiId = i;
CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssData = ontologyData->getIndividualIdAssoiationDataVector()[indiId];
if (indiAssData) {
//QString indiName = CIRIName::getRecentIRIName(indiAssData->mDebugIndi->getNameLinker());
//if (i < 100 || indiName == "http://semantics.crl.ibm.com/univ-bench-dl.owl#Tennis" || indiName == "http://semantics.crl.ibm.com/univ-bench-dl.owl#BasketBall" || indiName == "http://semantics.crl.ibm.com/univ-bench-dl.owl#Baseball"
// || indiName == "http://www.Department1.University0.edu/StudentWhoLikesTennisButDislikesBasketBallAndBaseBall" || indiName == "http://www.Department1.University0.edu/StudentWhoLikesBasketBallButDislikesBaseBall"
// || indiName == "http://www.Department1.University0.edu/StudentWhoLikesTennisAndBasketBallAndBaseBallAndShouldHaveManyHobbies" || mDebugIndiId == i) {
if (indiAssData && indiAssData->getAssociationDataUpdateId() >= 0) {
QString indiAssString = getIndividualAssociationDebugString(indiId, indiAssData, ontologyData, 0);
indiStringList.append(indiAssString);
cint64 prevCount = 0;
CBackendRepresentativeMemoryCacheIndividualAssociationData* prevIndiAssData = indiAssData->getPreviousData();
while (prevIndiAssData && !indiAssData->getIndividualAssociationMemoryContext() && prevCount <= 10) {
QString prevIndiAssString = getIndividualAssociationDebugString(indiId, prevIndiAssData, ontologyData, ++prevCount);
indiStringList.append(prevIndiAssString);
prevIndiAssData = prevIndiAssData->getPreviousData();
}
}
//}
}
}
QString statusString = QString("\tcache writes: %1,\r\n\ttotal association changes: %2,\r\n\tincompletely handled individuals: %3,\r\n\tincompatible association changes: %4,\r\n\tincompatible association checks: %5,\r\n\tmax individual association changes: %6\r\n")
.arg(mWriteDataCount).arg(mNextIndiUpdateId).arg(ontologyData->getIncompletelyHandledIndividualIdCount()).arg(mUpdateIncompatibleIndiCount).arg(mCheckIncompatibleIndiCount).arg(ontologyData->getMaxIndividualAssociationDataUpdateCount());
cacheStringList.append(QString("STATISTICS:\r\n\r\n%1\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n").arg(statusString));
cacheStringList.append(QString("REPRESENATIVE LABELS:\r\n\r\n"));
for (const QString& stringLine : labelStringList) {
cacheStringList.append(QString("%1\r\n").arg(stringLine));
}
cacheStringList.append(QString("\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\nINDIVIDUAL LABEL ASSOCIATIONS:\r\n"));
for (const QString& stringLine : indiStringList) {
cacheStringList.append(QString("%1\r\n").arg(stringLine));
}
cacheStringList.append(QString("\r\n"));
return cacheStringList;
}
QString CBackendRepresentativeMemoryCache::getIndividualAssociationDebugString(cint64 indiId, CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssData, CBackendRepresentativeMemoryCacheOntologyData* ontologyData, cint64 prevTab) {
QString prevTabString;
for (cint64 i = 0; i < prevTab; ++i) {
prevTabString += QString("\t");
}
QString indiAssString = QString("\r\nIndividual %1:\r\n").arg(indiId);
#ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
if (indiAssData->mDebugIndi) {
QString indiName = CIRIName::getRecentIRIName(indiAssData->mDebugIndi->getIndividualNameLinker());
indiAssString = QString("\r\nIndividual (%2) %1:\r\n").arg(indiId).arg(indiName);
}
#endif
if (prevTab > 0) {
indiAssString = QString("\r\n%3Individual %1 previous entry %2:\r\n").arg(indiId).arg(prevTab).arg(prevTabString);
}
prevTabString += QString("\t");
QString updateString = QString("%5update ids: individual association change: %1, total association change: %2, total touch: %3, \r\n%5\t\tlast indirectly connected individuals integration: %4").arg(indiAssData->getAssociationDataUpdateId()).arg(indiAssData->getCacheUpdateId()).arg(indiAssData->getCacheTouchId()).arg(indiAssData->getLastIntegratedIndirectlyConnectedIndividualsChangeId()).arg(prevTabString);
#ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
updateString += QString(", update function: %2, depending recomputation id: %1").arg(indiAssData->mDebugUpdateRecomputationId).arg(indiAssData->mDebugUpdateFunction);
#endif
updateString += QString("\r\n");
indiAssString += updateString;
QStringList statusStrings;
if (indiAssData->isCompletelySaturated()) {
statusStrings += "completely saturated";
} else {
statusStrings += "incompletely saturated";
}
if (indiAssData->isCompletelyPropagated()) {
statusStrings += "completely propagated";
} else {
statusStrings += "incompletely propagated";
}
if (indiAssData->isCompletelyHandled()) {
statusStrings += "completely handled";
} else {
statusStrings += "incompletely handled";
}
if (indiAssData->hasIndirectlyConnectedIndividualIntegration()) {
statusStrings += "indirectly connected individuals integration";
}
if (indiAssData->isIndirectlyConnectedNominalIndividual()) {
statusStrings += "indirectly connected nominal";
}
if (indiAssData->hasProblematicLevel()) {
statusStrings += "self clashing involved";
}
if (indiAssData->hasProblematicLeveledNeigbour()) {
statusStrings += "neighbour clashing involved";
}
#ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
if (indiAssData->mDebugSameIndividualsCompletion) {
statusStrings += "deterministic same individuals completion";
}
if (indiAssData->mDebugNeighbourCompletionPerformed) {
statusStrings += "neighbours same individuals completed";
}
if (indiAssData->mDebugNeighbourCompletionRequested) {
statusStrings += "neighbours same individuals completion requested";
}
if (indiAssData->mDebugNeighbourCompletionExtended) {
statusStrings += "neighbours same individuals extended";
}
#endif
QString statusString = QString("%2status: %1\r\n").arg(statusStrings.join(", ")).arg(prevTabString);
indiAssString += statusString;
if (indiAssData->getRepresentativeSameIndividualId() != indiAssData->getAssociatedIndividualId()) {
indiAssString += QString("%2representative same individual id: %1\r\n").arg(indiAssData->getRepresentativeSameIndividualId()).arg(prevTabString);
}
if (indiAssData->getDeterministicSameIndividualId() != indiAssData->getAssociatedIndividualId()) {
indiAssString += QString("%2deterministic merged same individual id: %1\r\n").arg(indiAssData->getDeterministicSameIndividualId()).arg(prevTabString);
}
if (indiAssData->getDeterministicMergedSameConsideredLabelCacheEntry()) {
indiAssString += QString("%2deterministic merged same considered individuals label: %1\r\n").arg(indiAssData->getDeterministicMergedSameConsideredLabelCacheEntry()->getCacheEntryID()).arg(prevTabString);
}
for (cint64 labelType = 0; labelType < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_ASSOCIATABLE_TYPE_COUNT; ++labelType) {
CBackendRepresentativeMemoryLabelCacheItem* labelItem = indiAssData->getLabelCacheEntry(labelType);
if (labelItem) {
cint64 itemId = labelItem->getCacheEntryID();
QString labelTypeString = getRepresentativeCacheLabelItemString(labelType);
QString indiLabelAssString = QString("%3associated %1 label: %2\r\n").arg(labelTypeString).arg(itemId).arg(prevTabString);
indiAssString += indiLabelAssString;
}
}
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* roleSetNeighbourArray = indiAssData->getRoleSetNeighbourArray();
if (roleSetNeighbourArray) {
CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* indexData = roleSetNeighbourArray->getIndexData();
for (cint64 i = 0; i < indexData->getArraySize(); ++i) {
CBackendRepresentativeMemoryLabelCacheItem* neighbourRoleSetLabelItem = indexData->getNeighbourRoleSetLabel(i);
QString roleSetNeighbourIndiIdsString;
roleSetNeighbourArray->at(i).visitNeighbourIndividualIds([&](cint64 id)->bool {
if (!roleSetNeighbourIndiIdsString.isEmpty()) {
roleSetNeighbourIndiIdsString += ", ";
}
roleSetNeighbourIndiIdsString += QString::number(id);
CBackendRepresentativeMemoryCacheIndividualAssociationData* neighIndiAssData = ontologyData->getIndividualIdAssoiationDataVector()[id];
if (neighIndiAssData && neighIndiAssData->hasProblematicLevel()) {
roleSetNeighbourIndiIdsString += "(ci)";
}
if (neighbourRoleSetLabelItem->getCacheValueCount() == 1 && neighbourRoleSetLabelItem->getCacheValueLinker()->getData().getTag() == 1) {
roleSetNeighbourIndiIdsString += "(pc)";
}
return true;
});
QString indiArrayAssString = QString("%4neighbouring ids for label %1 (%3): %2\r\n").arg(neighbourRoleSetLabelItem->getCacheEntryID()).arg(roleSetNeighbourIndiIdsString).arg(roleSetNeighbourArray->at(i).getIndividualCount()).arg(prevTabString);
indiAssString += indiArrayAssString;
}
}
CBackendRepresentativeMemoryCacheNominalIndividualIndirectConnectionData* indConnData = ontologyData->getNominaIIndividualdIndirectConnectionDataHash()->value(indiId);
if (indConnData) {
QStringList indConnIndiIds;
for (CXLinker* indiLinkerIt = indConnData->getIndirectlyConnectedIndividualIdLinker(); indiLinkerIt; indiLinkerIt = indiLinkerIt->getNext()) {
cint64 indiId = indiLinkerIt->getData();
indConnIndiIds.append(QString::number(indiId));
}
QString indiIndConnDatatring = QString("%3indirectly connected individuals with change id %1: %2\r\n").arg(indConnData->getLastChangeId()).arg(indConnIndiIds.join(", ")).arg(prevTabString);
indiAssString += indiIndConnDatatring;
}
CBackendRepresentativeMemoryCacheIndividualAssociationContext* memManCon = indiAssData->getIndividualAssociationMemoryContext();
if (memManCon) {
QString indiIndMemManCon = QString("memory management context: %1\r\n").arg((cint64)memManCon);
indiAssString += indiIndMemManCon;
}
return indiAssString;
}
bool CBackendRepresentativeMemoryCache::debugCheckPropagationCutRemainingNeighbours(CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssData, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
bool error = false;
#ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
if (indiAssData && indiAssData->mDebugIndi && ontologyData->isFirstIncompletelyHandledIndividualsRetrieved()) {
QHash prevNeighbourIndiArrayPosHash;
CBackendRepresentativeMemoryCacheIndividualAssociationData* prevIndiAssData = indiAssData->getPreviousData();
if (prevIndiAssData) {
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* roleSetNeighbourArray = prevIndiAssData->getRoleSetNeighbourArray();
if (roleSetNeighbourArray) {
CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* indexData = roleSetNeighbourArray->getIndexData();
for (cint64 i = 0; i < indexData->getArraySize(); ++i) {
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiLinker = roleSetNeighbourArray->at(i).getIndividualIdLinker();
bool firstArrayPosNeighbour = true;
for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiLinkerIt = indiLinker; indiLinkerIt; indiLinkerIt = indiLinkerIt->getNext()) {
cint64 neighbourIndiId = indiLinkerIt->getIndividualId();
prevNeighbourIndiArrayPosHash.insert(neighbourIndiId, i);
}
}
}
}
QHash newNeighbourIndiArrayPosHash;
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* roleSetNeighbourArray = indiAssData->getRoleSetNeighbourArray();
if (roleSetNeighbourArray) {
CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* indexData = roleSetNeighbourArray->getIndexData();
for (cint64 i = 0; i < indexData->getArraySize(); ++i) {
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiLinker = roleSetNeighbourArray->at(i).getIndividualIdLinker();
for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiLinkerIt = indiLinker; indiLinkerIt; indiLinkerIt = indiLinkerIt->getNext()) {
cint64 neighbourIndiId = indiLinkerIt->getIndividualId();
newNeighbourIndiArrayPosHash.insert(neighbourIndiId, i);
if (!prevNeighbourIndiArrayPosHash.contains(neighbourIndiId)) {
bool debug = true;
}
}
}
for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiLinkerIt = indiAssData->getPropagationCutRemovedNeighbourIndividualLinker(); indiLinkerIt; indiLinkerIt = indiLinkerIt->getNext()) {
cint64 neighbourIndiId = indiLinkerIt->getIndividualId();
newNeighbourIndiArrayPosHash.insert(neighbourIndiId, -1);
if (!prevNeighbourIndiArrayPosHash.contains(neighbourIndiId)) {
bool debug = true;
error = true;
}
}
}
for (QHash::const_iterator it = prevNeighbourIndiArrayPosHash.constBegin(), itEnd = prevNeighbourIndiArrayPosHash.constEnd(); it != itEnd; ++it) {
cint64 prevIndiId = it.key();
cint64 prevIndiArrayPos = it.value();
if (!newNeighbourIndiArrayPosHash.contains(prevIndiId)) {
bool debug = true;
error = true;
}
}
}
#endif
return error;
}
bool CBackendRepresentativeMemoryCache::debugCheckSingleNeighbourInvolvedOccurrences(CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssData, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
bool error = false;
#ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
if (indiAssData && indiAssData->mDebugIndi && ontologyData->isFirstIncompletelyHandledIndividualsRetrieved()) {
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* roleSetNeighbourArray = indiAssData->getRoleSetNeighbourArray();
auto hash = indiAssData->getNeighbourRoleSetHash();
QSet neighbourIndiSet;
if (roleSetNeighbourArray) {
CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* indexData = roleSetNeighbourArray->getIndexData();
for (cint64 i = 0; i < indexData->getArraySize(); ++i) {
auto label = indexData->getNeighbourRoleSetLabel(i);
if (label->getCacheValueCount() > 1 || label->getCacheValueLinker()->getCacheValue().getTag() != 1) {
auto& data = roleSetNeighbourArray->at(i);
CBackendRepresentativeMemoryLabelCacheItem* neighbourRoleSetLabelItem = indexData->getNeighbourRoleSetLabel(i);
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiLinker = roleSetNeighbourArray->at(i).getIndividualIdLinker();
bool firstArrayPosNeighbour = true;
for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiLinkerIt = indiLinker; indiLinkerIt; indiLinkerIt = indiLinkerIt->getNext()) {
cint64 neighbourIndiId = indiLinkerIt->getIndividualId();
auto neighbourLabel = hash->getNeighbourRoleSetLabel(neighbourIndiId);
if (neighbourLabel != label) {
bool debug = true;
mDebugIndiId = indiAssData->getAssociatedIndividualId();
error = true;
}
if (neighbourIndiSet.contains(neighbourIndiId)) {
bool debug = true;
mDebugIndiId = indiAssData->getAssociatedIndividualId();
error = true;
}
neighbourIndiSet.insert(neighbourIndiId);
CBackendRepresentativeMemoryCacheIndividualAssociationData* neighIndiAssData = ontologyData->getIndividualIdAssoiationDataVector()[neighbourIndiId];
if (neighIndiAssData && neighIndiAssData->hasProblematicLevel()) {
if (!firstArrayPosNeighbour) {
bool debug = true;
mDebugIndiId = indiAssData->getAssociatedIndividualId();
error = true;
}
} else {
firstArrayPosNeighbour = false;
}
}
}
}
}
}
#endif
return error;
}
bool CBackendRepresentativeMemoryCache::debugCheckCorrectNeighbourOccurrence(CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssData, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
bool missingOccurrence = false;
#ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
if (indiAssData && indiAssData->mDebugIndi && ontologyData->isFirstIncompletelyHandledIndividualsRetrieved()) {
QString indiName = CIRIName::getRecentIRIName(indiAssData->mDebugIndi->getNameLinker());
if (indiName == "http://www.Department1.University0.edu/StudentWhoLikesTennisButDislikesBasketBallAndBaseBall") {
s1 = true;
}
if (indiName == "http://www.Department1.University0.edu/StudentWhoLikesTennisAndBasketBallAndBaseBallAndShouldHaveManyHobbies") {
s2 = true;
}
if (indiName == "http://www.Department1.University0.edu/StudentWhoLikesBasketBallButDislikesBaseBall") {
s3 = true;
}
if (indiName == "http://semantics.crl.ibm.com/univ-bench-dl.owl#Tennis" || indiName == "http://semantics.crl.ibm.com/univ-bench-dl.owl#BasketBall" || indiName == "http://semantics.crl.ibm.com/univ-bench-dl.owl#Baseball") {
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* roleSetNeighbourArray = indiAssData->getRoleSetNeighbourArray();
QSet requiredNeighbourNamesSet;
if (indiName == "http://semantics.crl.ibm.com/univ-bench-dl.owl#Tennis") {
if (s1) {
requiredNeighbourNamesSet.insert("http://www.Department1.University0.edu/StudentWhoLikesTennisButDislikesBasketBallAndBaseBall");
}
if (s2) {
requiredNeighbourNamesSet.insert("http://www.Department1.University0.edu/StudentWhoLikesTennisAndBasketBallAndBaseBallAndShouldHaveManyHobbies");
}
} else if (indiName == "http://semantics.crl.ibm.com/univ-bench-dl.owl#BasketBall") {
if (s2) {
requiredNeighbourNamesSet.insert("http://www.Department1.University0.edu/StudentWhoLikesTennisAndBasketBallAndBaseBallAndShouldHaveManyHobbies");
}
if (s3) {
requiredNeighbourNamesSet.insert("http://www.Department1.University0.edu/StudentWhoLikesBasketBallButDislikesBaseBall");
}
} else if (indiName == "http://semantics.crl.ibm.com/univ-bench-dl.owl#Baseball") {
if (s2) {
requiredNeighbourNamesSet.insert("http://www.Department1.University0.edu/StudentWhoLikesTennisAndBasketBallAndBaseBallAndShouldHaveManyHobbies");
}
}
if (roleSetNeighbourArray && !requiredNeighbourNamesSet.isEmpty()) {
CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* indexData = roleSetNeighbourArray->getIndexData();
for (cint64 i = 0; i < indexData->getArraySize(); ++i) {
auto& data = roleSetNeighbourArray->at(i);
CBackendRepresentativeMemoryLabelCacheItem* neighbourRoleSetLabelItem = indexData->getNeighbourRoleSetLabel(i);
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiLinker = roleSetNeighbourArray->at(i).getIndividualIdLinker();
for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiLinkerIt = indiLinker; indiLinkerIt; indiLinkerIt = indiLinkerIt->getNext()) {
cint64 neighbourIndiId = indiLinkerIt->getIndividualId();
CBackendRepresentativeMemoryCacheIndividualAssociationData* neighIndiAssData = ontologyData->getIndividualIdAssoiationDataVector()[neighbourIndiId];
if (neighIndiAssData && neighIndiAssData->mDebugIndi) {
requiredNeighbourNamesSet.remove(CIRIName::getRecentIRIName(neighIndiAssData->mDebugIndi->getNameLinker()));
}
}
}
if (!requiredNeighbourNamesSet.isEmpty()) {
bool debug = true;
missingOccurrence = true;
if (debug) {
writeStringifiedRepresentativeCacheToFile();
}
}
}
}
}
#endif
return missingOccurrence;
}
bool CBackendRepresentativeMemoryCache::debugCheckNeighoursCorrectlyCounted(CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssData, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
if (indiAssData) {
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* roleSetNeighbourArray = indiAssData->getRoleSetNeighbourArray();
if (roleSetNeighbourArray) {
CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* indexData = roleSetNeighbourArray->getIndexData();
for (cint64 i = 0; i < indexData->getArraySize(); ++i) {
auto& data = roleSetNeighbourArray->at(i);
CBackendRepresentativeMemoryLabelCacheItem* neighbourRoleSetLabelItem = indexData->getNeighbourRoleSetLabel(i);
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiLinker = roleSetNeighbourArray->at(i).getIndividualIdLinker();
cint64 count = 0;
if (!indiLinker) {
bool debug = false;
if (debug) {
writeStringifiedRepresentativeCacheToFile();
}
}
if (indiLinker) {
count = indiLinker->getCount();
}
if (count != roleSetNeighbourArray->at(i).getIndividualCount()) {
bool debug = false;
if (debug) {
writeStringifiedRepresentativeCacheToFile();
}
return false;
}
}
}
}
return true;
}
void CBackendRepresentativeMemoryCache::checkAssociationComplete(CBackendRepresentativeMemoryCacheOntologyData* ontologyData, bool forceCompletion) {
if ((ontologyData->getIncompletelyHandledIndividualIdCount() <= 0 || forceCompletion) && !ontologyData->isAssociationCompleted() && ontologyData->isFirstIncompletelyHandledIndividualsRetrieved()) {
cint64 ontologyIdentifier = ontologyData->getOntologyIdentifer();
ontologyData->setAssociationCompleted(true);
mFixedOntologyIdentifierDataHashLock.lockForWrite();
ontologyData->incUsageCount();
mFixedOntologyIdentifierDataHash.insert(ontologyIdentifier, ontologyData);
mFixedOntologyIdentifierDataHashLock.unlock();
LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Associating %1 individuals with %2 labels completed.").arg(ontologyData->getMaxStoredIndvidualiId()).arg(ontologyData->getNextEntryID(false)), this);
if (mConfLateIndividualLabelAssociationIndexing) {
indexIndividualLabelAssociations(ontologyData);
}
if (mConfDebugWriteRepresentativeCache) {
writeStringifiedRepresentativeCacheToFile();
}
}
}
bool CBackendRepresentativeMemoryCache::debugTestingPrioritizedExpansionLinkDuplicates(CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* testingArray, CBackendRepresentativeMemoryLabelCacheItem* propMarkLabelItem) {
if (testingArray) {
CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* testingArrayIndexData = testingArray->getIndexData();
if (testingArrayIndexData) {
cint64 testingPrioPropMarkIndex = testingArrayIndexData->getIndex(propMarkLabelItem);
if (testingPrioPropMarkIndex >= 0) {
QSet neighbourIndiIdSet;
testingArray->at(testingPrioPropMarkIndex).visitNeighbourIndividualIds([&](cint64 id)->bool {
if (neighbourIndiIdSet.contains(id)) {
bool debug = true;
} else {
neighbourIndiIdSet.insert(id);
}
return true;
});
}
}
}
return true;
}
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* CBackendRepresentativeMemoryCache::copyNeighbourIndividualIdLinkers(CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiIdLinker, CBackendRepresentativeMemoryCacheContext* context, CBackendRepresentativeMemoryCacheIndividualAssociationData* locAssociationData, cint64 pos) {
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* firstNewIndiIdLinker = nullptr;
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* lastNewIndiIdLinker = nullptr;
cint64 copiedLinkerCount = 0;
for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiIdLinkerIt = indiIdLinker; indiIdLinkerIt; indiIdLinkerIt = indiIdLinkerIt->getNext()) {
cint64 neighbourId = indiIdLinkerIt->getIndividualId();
CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newLinker = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
newLinker->initIndividualIdLinker(neighbourId);
++mStatIndividualAssociationSeparateMemoryManagmentNeighbourLinkCopyingCount;
++copiedLinkerCount;
if (!lastNewIndiIdLinker) {
firstNewIndiIdLinker = lastNewIndiIdLinker = newLinker;
} else {
lastNewIndiIdLinker->setNext(newLinker);
lastNewIndiIdLinker = newLinker;
}
}
return firstNewIndiIdLinker;
}
}; // end namespace Cache
}; // end namespace Kernel
}; // end namespace Reasoner
}; // end namespace Konclude