Debian Codegraph @

Source/Reasoner/Kernel/Cache/CBackendRepresentativeMemoryCache.cpp

raw
1 /*
2 * Copyright (C) 2013-2015, 2019 by the Konclude Developer Team.
3 *
4 * This file is part of the reasoning system Konclude.
5 * For details and support, see <http://konclude.com/>.
6 *
7 * Konclude is free software: you can redistribute it and/or modify
8 * it under the terms of version 3 of the GNU Lesser General Public
9 * License (LGPLv3) as published by the Free Software Foundation.
10 *
11 * Konclude is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU (Lesser) General Public License for more details.
15 *
16 * You should have received a copy of the GNU (Lesser) General Public
17 * License along with Konclude. If not, see <http://www.gnu.org/licenses/>.
18 *
19 */
20
21 #include "CBackendRepresentativeMemoryCache.h"
22 #include "CBackendRepresentativeMemoryCacheWriter.h"
23
24
25 namespace Konclude {
26
27 namespace Reasoner {
28
29 namespace Kernel {
30
31 namespace Cache {
32
33
34 CBackendRepresentativeMemoryCache::CBackendRepresentativeMemoryCache(CConfiguration* config, QString threadIdentifierName, CWatchDog *watchDogThread) : CThread(threadIdentifierName, watchDogThread) {
35
36 mConfig = config;
37
38 mSlotLinker = nullptr;
39 mLastUpdatedSlotLinker = nullptr;
40 mReaderLinker = nullptr;
41
42 mNextIndiUpdateId = 1;
43 mNextNomConnUpdateId = 1;
44
45 mReaderSlotUpdateCount = 0;
46 mOntologyDataUpdateCount = 0;
47 mOntologyDataReleasedCount = 0;
48 mOntologyDataReleasedWhileNewCreationCount = 0;
49 mOntologyDataReleasedWhileSlotUpdateCount = 0;
50 mReaderSlotReleasedCount = 0;
51 mCheckingRemainingIncompletelyHandledCount = 0;
52 mEmptyWriteDataCount = 0;
53 mWriteDataCount = 0;
54 mStartWriteCollectCount = 0;
55 mNextWriteCollectCount = 0;
56 mTmpPropCutIndiArrayNeighboursHandlingDataHash = nullptr;
57
58 mConfInstallingDeterministicSameHandlingLargeDifference = 50;
59 mConfInstallingDeterministicSameHandlingLargeDifferenceReached = false;
60
61 mConfWaitIndividualLabelAssociationIndexed = CConfigDataReader::readConfigBoolean(config, "Konclude.Cache.RepresentativeBackendCache.WaitIndividualLabelAssociationIndexed", true);
62 mConfLateIndividualLabelAssociationIndexing = CConfigDataReader::readConfigBoolean(config, "Konclude.Cache.RepresentativeBackendCache.LateIndividualLabelAssociationIndexing", true);
63 mConfDebugWriteRepresentativeCache = CConfigDataReader::readConfigBoolean(config, "Konclude.Debugging.WriteDebuggingRepresentativeCacheContent", false);
64 mStatCollectStatistics = CConfigDataReader::readConfigBoolean(config, "Konclude.Cache.RepresentativeBackendCache.LogStorageStatistics", false);
65 mConfDirectUpdateSynchronization = !CConfigDataReader::readConfigBoolean(config, "Konclude.Cache.RepresentativeBackendCache.ThreadedUpdateSynchronization", true);
66
67
68 cint64 limitCount = CConfigDataReader::readConfigInteger(config, "Konclude.Cache.RepresentativeBackendCache.PendingWriteDataLimit", -1);
69 if (limitCount >= 0) {
70 mLimitRemainingWritePending = true;
71 mRemainingWritePendingSemaphore.release(limitCount);
72 } else {
73 mLimitRemainingWritePending = false;
74 }
75
76
77
78 mConfIncrementUpdateIdForDeterministicSameAsCompletion = CConfigDataReader::readConfigBoolean(config, "Konclude.Cache.RepresentativeBackendCache.IncrementUpdateIdForDeterministicSameAsCompletion", true);
79 mConfMinRequiredDeterministicSameMergedHandledInstallationPossiblitiesForNeighbourCompletion = CConfigDataReader::readConfigInteger(config, "Konclude.Cache.RepresentativeBackendCache.MinRequiredDeterministicSameMergedHandledInstallationPossiblitiesForNeighbourCompletion", 1);
80 mConfUnchangedDeterministicSameMergeUpdatesForDeterministicSameNeighbourCompletion = CConfigDataReader::readConfigInteger(config, "Konclude.Cache.RepresentativeBackendCache.UnchangedDeterministicSameMergeUpdatesForDeterministicSameNeighbourCompletion", 1);
81
82 mConfInstallingDeterministicSameHandlingLargeDifference = CConfigDataReader::readConfigInteger(config, "Konclude.Cache.RepresentativeBackendCache.TryingDeterministicSameMergedHandlingInstallationDifferenceCount", -1);
83 if (mConfInstallingDeterministicSameHandlingLargeDifference != -1) {
84 mConfInstallingDeterministicSameHandlingLargeDifferenceReached = true;
85 }
86
87
88 mConfBasicPrecomputationModeActivationUpdateMergesRatio = CConfigDataReader::readConfigInteger(config, "Konclude.Cache.RepresentativeBackendCache.BasicPrecomputationModeActivationUpdateMergesPercentRatio", 5) / 100.;
89
90
91 mConfMaxIncompletelyHandledIndividualsRetrievalCount = CConfigDataReader::readConfigInteger(config, "Konclude.Cache.RepresentativeBackendCache.MaxIncompletelyHandledIndividualsRetrievalCount", -1);
92 mConfMaxCacheDataUpdateWritingCount = CConfigDataReader::readConfigInteger(config, "Konclude.Cache.RepresentativeBackendCache.MaxCacheDataUpdateWritingCount", -1);
93
94
95 mConfUpdateRejectingIncompatibleIndividualAssociationsRatio = CConfigDataReader::readConfigDouble(config, "Konclude.Cache.RepresentativeBackendCache.RejectingUpdateIncompatibleIndividualAssociationsRatio", 1.0);
96 mConfUpdateRejectingIncompatiblePropagationCuttedIndividualLinkedNeighbourRatio = CConfigDataReader::readConfigDouble(config, "Konclude.Cache.RepresentativeBackendCache.RejectingUpdateIncompatiblePropagationCuttedIndividualLinkedNeighbourRatio", 1.0);
97 mConfUpdateRejectingIncompatiblePropagationCuttedIndividualLinkedNeighbourCount = CConfigDataReader::readConfigInteger(config, "Konclude.Cache.RepresentativeBackendCache.RejectingUpdateIncompatiblePropagationCuttedIndividualLinkedNeighbourCount", -1);
98
99 mConfInterpretUnchangedLabelsAsCompatible = CConfigDataReader::readConfigBoolean(config, "Konclude.Cache.RepresentativeBackendCache.InterpretUnchangedLabelsAsCompatibleIndividualAssociationUpdates", false);
100
101
102 mSlotUpdateWaitingMaxCount = CConfigDataReader::readConfigInteger(config, "Konclude.Cache.RepresentativeBackendCache.SlotUpdateWaitingIncreaseMaximumCount", 20);
103
104 mStatAddingNeighbourLinksAssociationUpdateCount = 0;
105 mStatUpdatedOrRemovedNeighbourLinksAssociationUpdateCount = 0;
106 mStatMaxSameAsMergedCount = 0;
107 mStatMaxLabelValueCount = 0;
108 for (cint64 i = 0; i < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_TYPE_COUNT; ++i) {
109 mStatLabelTypeCount[i] = 0;
110 mStatLabelTypeMaxValueCount[i] = 0;
111 mStatLabelTypeAllValueCount[i] = 0;
112 }
113 mStatLabelCount = 0;
114 mStatMaxAssociationUpdateCount = 0;
115 mStatCreatedNeighbourLinks = 0;
116 mStatMaxNeighbourLinksCount = 0;
117 mStatIncompatibleLabelNeighbourCompletionCount = 0;
118 mStatChangedLabelNeighbourCompletionCount = 0;
119 mStatNeighbourCompletionDetSameSuccededCount = 0;
120 mStatNeighbourCompletionDetSameChangedCount = 0;
121 mStatNeighbourCompletionDetSameIncompatibleCount = 0;
122 mStatNeighbourCompletionDetSameUnchangedCount = 0;
123
124 mStatDetSameAssociationInstallCount = 0;
125 mStatDetSameAssociationFailedCount = 0;
126 mStatDetSameAssociationDifferentUpdateIdFailedCount = 0;
127 mStatDetSameAssociationDifferentDestIdFailedCount = 0;
128 mStatDetSameAssociationIncompleteHandledDestFailedCount = 0;
129 mStatDetSameAssociationRepMergedDestFailedCount = 0;
130 mStatDetSameRepresentativeMergingCount = 0;
131
132 mSlotUpdateWaitingIncreaseCount = 1;
133
134 CMemoryAllocationManager* memMan = mContext.getMemoryAllocationManager();
135 //mIndiIdAssoDataHash = nullptr;
136
137 mOntologyIdentifierDataHash = CObjectParameterizingAllocator< CCACHINGHASH<cint64, CBackendRepresentativeMemoryCacheOntologyData*>, CBackendRepresentativeMemoryCacheBaseContext* >::allocateAndConstructAndParameterize(memMan, &mContext);
138
139 startThread(QThread::HighestPriority);
140 }
141
142
143 CBackendRepresentativeMemoryCache::~CBackendRepresentativeMemoryCache() {
144 }
145
146
147
148 CCacheStatistics* CBackendRepresentativeMemoryCache::getCacheStatistics() {
149 return &mCacheStat;
150 }
151
152
153
154 void CBackendRepresentativeMemoryCache::deleteExpiredIndividualAssociationMemoryContexts(CBackendRepresentativeMemoryCacheOntologyData* ontologyData, CBackendRepresentativeMemoryCacheBaseContext* context) {
155 CBackendRepresentativeMemoryCacheContext* ontologyContext = ontologyData->getOntologyContext();
156 if (ontologyData->isBasicPrecomputationMode()) {
157 return;
158 }
159
160 cint64 statIndividualAssociationSeparateMemoryManagmentDeletionCount = 0;
161 cint64 statIndividualAssociationSeparateMemoryManagmentSlotReferredCheckingCount = 0;
162 CCACHINGMAP<cint64, CBackendRepresentativeMemoryCacheIndividualAssociationContext*>* recomputationIdReleasingIndividualAssociationContextMap = ontologyData->getRecomputationIdReleasingIndividualAssociationContextMap();
163 bool recomputationIdReleasingIndividualAssociationContextMapLocated = false;
164 CBackendRepresentativeMemoryCacheIndividualAssociationContext* releaseQueuedIndividualAssociationContextLinker = ontologyData->getReleaseQueuedIndividualAssociationContextLinker();
165 if (releaseQueuedIndividualAssociationContextLinker) {
166 cint64 ontologyIdentifier = ontologyData->getOntologyIdentifer();
167
168 CBackendRepresentativeMemoryCacheIndividualAssociationContext* prevReleaseQueuedIndividualAssociationContextLinker = nullptr;
169 for (CBackendRepresentativeMemoryCacheIndividualAssociationContext* releaseQueuedIndividualAssociationContextLinkerIt = releaseQueuedIndividualAssociationContextLinker; releaseQueuedIndividualAssociationContextLinkerIt; ) {
170 CBackendRepresentativeMemoryCacheIndividualAssociationContext* nextReleaseQueuedIndividualAssociationContextLinker = releaseQueuedIndividualAssociationContextLinkerIt->getNext();
171 CBackendRepresentativeMemoryCacheOntologyDataRecomputationReferenceLinker* firstRecomputationReferenceLinker = releaseQueuedIndividualAssociationContextLinkerIt->getFirstRecomputationReferenceLinker();
172 CBackendRepresentativeMemoryCacheOntologyDataRecomputationReferenceLinker* firstNextRecomputationReferenceLinker = firstRecomputationReferenceLinker->getNext();
173 CBackendRepresentativeMemoryCacheOntologyDataRecomputationReferenceLinker* lastRecomputationReferenceLinker = releaseQueuedIndividualAssociationContextLinkerIt->getLastRecomputationReferenceLinker();
174
175 bool noneOntologyDataUpdateIdSlotReferred = true;
176 cint64 maxReferedRecomputationId = CINT64_MIN;
177 cint64 minReferedRecomputationId = CINT64_MAX;
178 for (CBackendRepresentativeMemoryCacheOntologyDataRecomputationReferenceLinker* recomputationReferenceLinkerIt = lastRecomputationReferenceLinker; recomputationReferenceLinkerIt && noneOntologyDataUpdateIdSlotReferred && recomputationReferenceLinkerIt != firstNextRecomputationReferenceLinker; recomputationReferenceLinkerIt = recomputationReferenceLinkerIt->getNext()) {
179 cint64 ontologyDataUpdateId = recomputationReferenceLinkerIt->getOntologyDataUpdateId();
180
181 if (!recomputationReferenceLinkerIt->isOntologyDataActive()) {
182 mStatIndividualAssociationSeparateMemoryManagmentSlotReferredCheckingCount++;
183 statIndividualAssociationSeparateMemoryManagmentSlotReferredCheckingCount++;
184
185 bool ontologyDataUpdateIdSlotReferred = false;
186 for (CBackendRepresentativeMemoryCacheSlotItem* slotLinkerIt = mSlotLinker; slotLinkerIt && !ontologyDataUpdateIdSlotReferred; slotLinkerIt = slotLinkerIt->getNext()) {
187 CBackendRepresentativeMemoryCacheOntologyData* referredOntologyData = slotLinkerIt->getOntologyData(ontologyIdentifier);
188 cint64 ontologyUpdateId = referredOntologyData->getOntologyDataUpdateId();
189 if (ontologyUpdateId == ontologyDataUpdateId) {
190 ontologyDataUpdateIdSlotReferred = true;
191 }
192 }
193
194 if (ontologyDataUpdateIdSlotReferred) {
195 noneOntologyDataUpdateIdSlotReferred = false;
196 }
197 maxReferedRecomputationId = qMax(recomputationReferenceLinkerIt->getMaxUsedRecomputationId(), maxReferedRecomputationId);
198 minReferedRecomputationId = qMin(recomputationReferenceLinkerIt->getMinUsedRecomputationId(), minReferedRecomputationId);
199 } else {
200 noneOntologyDataUpdateIdSlotReferred = false;
201 }
202 }
203
204 if (noneOntologyDataUpdateIdSlotReferred) {
205 --mStatMemoryManagmentQueuedCheckingCount;
206 ++mStatMemoryManagmentScheduledReleasingCount;
207 mStatIndividualAssociationSeparateMemoryManagmentUnreferredSlotCount++;
208 // move to recomputation id releasing map
209 if (!recomputationIdReleasingIndividualAssociationContextMap || !recomputationIdReleasingIndividualAssociationContextMapLocated) {
210 CCACHINGMAP<cint64, CBackendRepresentativeMemoryCacheIndividualAssociationContext*>* tmpMap = CObjectParameterizingAllocator< CCACHINGMAP<cint64, CBackendRepresentativeMemoryCacheIndividualAssociationContext*> , CContext* >::allocateAndConstructAndParameterize(ontologyContext->getMemoryAllocationManager(), ontologyContext);
211 if (recomputationIdReleasingIndividualAssociationContextMap) {
212 *tmpMap = *recomputationIdReleasingIndividualAssociationContextMap;
213 }
214 recomputationIdReleasingIndividualAssociationContextMap = tmpMap;
215 recomputationIdReleasingIndividualAssociationContextMapLocated = true;
216 ontologyData->setRecomputationIdReleasingIndividualAssociationContextMap(recomputationIdReleasingIndividualAssociationContextMap);
217 }
218
219 if (!prevReleaseQueuedIndividualAssociationContextLinker) {
220 ontologyData->setReleaseQueuedIndividualAssociationContextLinker(nextReleaseQueuedIndividualAssociationContextLinker);
221 } else {
222 prevReleaseQueuedIndividualAssociationContextLinker->setNext(nextReleaseQueuedIndividualAssociationContextLinker);
223 }
224 releaseQueuedIndividualAssociationContextLinkerIt->setNext(nullptr);
225
226 CBackendRepresentativeMemoryCacheIndividualAssociationContext* prevReleasingIndividualAssociationContextLinker = recomputationIdReleasingIndividualAssociationContextMap->value(maxReferedRecomputationId);
227 releaseQueuedIndividualAssociationContextLinkerIt = releaseQueuedIndividualAssociationContextLinkerIt->append(prevReleasingIndividualAssociationContextLinker);
228
229 recomputationIdReleasingIndividualAssociationContextMap->insert(maxReferedRecomputationId, releaseQueuedIndividualAssociationContextLinkerIt);
230 } else {
231 prevReleaseQueuedIndividualAssociationContextLinker = releaseQueuedIndividualAssociationContextLinkerIt;
232 }
233 releaseQueuedIndividualAssociationContextLinkerIt = nextReleaseQueuedIndividualAssociationContextLinker;
234 }
235 }
236
237
238 if (recomputationIdReleasingIndividualAssociationContextMap && !recomputationIdReleasingIndividualAssociationContextMap->isEmpty()) {
239 cint64 minValidRecompId = getMinimumSlotReferreringInstalledValidRecomputationId(ontologyData);
240 mLastMemoryContextDeletionMinValidRecompId = minValidRecompId;
241
242 cint64 statDeletedMemoryPoolsCount = 0;
243 cint64 statDeletedMemoryPoolSize = 0;
244
245 CMemoryPoolAllocationManager* memMan = context->getMemoryPoolAllocationManager();
246 while (!recomputationIdReleasingIndividualAssociationContextMap->isEmpty() && recomputationIdReleasingIndividualAssociationContextMap->constBegin().key() < minValidRecompId) {
247 CBackendRepresentativeMemoryCacheIndividualAssociationContext* unusedIndiAssContextLinker = recomputationIdReleasingIndividualAssociationContextMap->begin().value();
248 for (CBackendRepresentativeMemoryCacheIndividualAssociationContext* unusedIndiAssContextLinkerIt = unusedIndiAssContextLinker; unusedIndiAssContextLinkerIt; ) {
249 CBackendRepresentativeMemoryCacheIndividualAssociationContext* nextUnusedIndiAssContextLinker = unusedIndiAssContextLinkerIt->getNext();
250 CMemoryPoolContainer* memPoolContainer = unusedIndiAssContextLinkerIt->getMemoryPoolContainer();
251 CMemoryPool* memoryPools = memPoolContainer->getMemoryPools();
252 statDeletedMemoryPoolsCount += memPoolContainer->getMemoryPoolCount();
253 statDeletedMemoryPoolSize += memPoolContainer->getMemorySize();
254
255 if (mStatCollectStatistics) {
256 #ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
257 mIndiContextCountHash[unusedIndiAssContextLinkerIt->mIndiId]--;
258 #endif
259 }
260 --mStatMemoryManagmentScheduledReleasingCount;
261
262 //#ifndef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
263 memMan->releaseTemporaryMemoryPools(memoryPools);
264 //#endif
265
266 mStatIndividualAssociationSeparateMemoryManagmentDeletionCount++;
267 statIndividualAssociationSeparateMemoryManagmentDeletionCount++;
268 unusedIndiAssContextLinkerIt = nextUnusedIndiAssContextLinker;
269 }
270
271 if (!recomputationIdReleasingIndividualAssociationContextMapLocated) {
272 CCACHINGMAP<cint64, CBackendRepresentativeMemoryCacheIndividualAssociationContext*>* tmpMap = CObjectParameterizingAllocator< CCACHINGMAP<cint64, CBackendRepresentativeMemoryCacheIndividualAssociationContext*>, CContext* >::allocateAndConstructAndParameterize(ontologyContext->getMemoryAllocationManager(), ontologyContext);
273 if (recomputationIdReleasingIndividualAssociationContextMap) {
274 *tmpMap = *recomputationIdReleasingIndividualAssociationContextMap;
275 }
276 recomputationIdReleasingIndividualAssociationContextMap = tmpMap;
277 recomputationIdReleasingIndividualAssociationContextMapLocated = true;
278 ontologyData->setRecomputationIdReleasingIndividualAssociationContextMap(recomputationIdReleasingIndividualAssociationContextMap);
279 }
280 recomputationIdReleasingIndividualAssociationContextMap->erase(recomputationIdReleasingIndividualAssociationContextMap->begin());
281 }
282
283 if (statDeletedMemoryPoolsCount > 0) {
284 LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Deleted %1 memory pools for %2 individual associations, %5 with separate memory management remaining, checked %3 (overall %4) slot reference usages.")
285 .arg(statDeletedMemoryPoolsCount).arg(statIndividualAssociationSeparateMemoryManagmentDeletionCount).arg(statIndividualAssociationSeparateMemoryManagmentSlotReferredCheckingCount).arg(mStatIndividualAssociationSeparateMemoryManagmentSlotReferredCheckingCount).arg(mStatIndividualAssociationSeparateMemoryManagmentContextCreationCount - mStatIndividualAssociationSeparateMemoryManagmentDeletionCount), this);
286 }
287
288
289 if (mStatCollectStatistics) {
290 cint64 maxIndi = -1;
291 cint64 maxCount = -1;
292 for (QHash<cint64, cint64>::const_iterator it = mIndiContextCountHash.constBegin(), itEnd = mIndiContextCountHash.constEnd(); it != itEnd; ++it) {
293 if (it.value() > maxCount) {
294 maxCount = it.value();
295 maxIndi = it.key();
296 }
297 }
298
299 LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("%1 individuals with separate memory management, max memory contexts (%2) for individual %3, %4 remaining deletion queued, %5 remaining deletion scheduled.")
300 .arg(mIndiContextCountHash.size()).arg(maxCount).arg(maxIndi).arg(mStatMemoryManagmentQueuedCheckingCount).arg(mStatMemoryManagmentScheduledReleasingCount), this);
301 }
302
303
304 }
305 }
306
307
308
309 void CBackendRepresentativeMemoryCache::queueIndividualAssociationMemoryContextDeletion(CBackendRepresentativeMemoryCacheIndividualAssociationContext* indiAssMemContext, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
310 ontologyData->addReleaseQueuedIndividualAssociationContextLinker(indiAssMemContext);
311 ++mStatIndividualAssociationSeparateMemoryManagmentSlotReferredCheckingQueuingCount;
312 ++mStatMemoryManagmentQueuedCheckingCount;
313 }
314
315
316
317 void CBackendRepresentativeMemoryCache::createReaderSlotUpdate(CBackendRepresentativeMemoryCacheOntologyData* ontologyData, CBackendRepresentativeMemoryCacheBaseContext* context) {
318 ontologyData->setMinimumValidRecomputationId(ontologyData->getNextUpdateMinimumValidRecomputationId());
319 ontologyData->setSlotUpdateIntegrated(true);
320 CMemoryPoolProvider* memProv = context->getMemoryPoolAllocationManager();
321 CBackendRepresentativeMemoryCacheSlotItem* slot = CObjectMemoryPoolAllocator<CBackendRepresentativeMemoryCacheSlotItem>::allocateAndConstructWithMemroyPool(memProv);
322 CMemoryPoolContainerAllocationManager slotMemMan(slot, memProv);
323 CMemoryAllocationManager* memMan = &slotMemMan;
324 CContextBase tmpContext(memMan);
325
326
327 CCACHINGHASH<cint64, CBackendRepresentativeMemoryCacheOntologyData*>* ontologyIdentifierDataHash = CObjectParameterizingAllocator< CCACHINGHASH<cint64, CBackendRepresentativeMemoryCacheOntologyData*>, CContext* >::allocateAndConstructAndParameterize(memMan, &tmpContext);
328 *ontologyIdentifierDataHash = *mOntologyIdentifierDataHash;
329 ontologyIdentifierDataHash->detach();
330 slot->setOntologyIdentifierDataHash(ontologyIdentifierDataHash);
331
332 mReaderSlotUpdateCount++;
333
334 mLastUpdatedSlotLinker = slot;
335 if (mSlotLinker) {
336 mSlotLinker->append(slot);
337 } else {
338 mSlotLinker = slot;
339 }
340
341
342 for (CCACHINGHASH<cint64, CBackendRepresentativeMemoryCacheOntologyData*>::const_iterator it = ontologyIdentifierDataHash->constBegin(), itEnd = ontologyIdentifierDataHash->constEnd(); it != itEnd; ++it) {
343 CBackendRepresentativeMemoryCacheOntologyData* ontologyData = it.value();
344 ontologyData->incUsageCount();
345 }
346
347 CBackendRepresentativeMemoryCacheReader* readerLinkerIt = mReaderLinker;
348 while (readerLinkerIt) {
349 slot->incReader();
350 readerLinkerIt->updateSlot(slot);
351 readerLinkerIt = readerLinkerIt->getNext();
352 }
353 }
354
355
356
357
358
359
360 CBackendRepresentativeMemoryCacheOntologyData* CBackendRepresentativeMemoryCache::prepareOntologyDataUpdate(cint64 ontologyIdentifier, cint64 minIndiCount) {
361 CBackendRepresentativeMemoryCacheOntologyData*& ontologyData = (*mOntologyIdentifierDataHash)[ontologyIdentifier];
362 if (!ontologyData || ontologyData->isSlotUpdateIntegrated()) {
363 CBackendRepresentativeMemoryCacheOntologyData* prevOntologyData = ontologyData;
364
365 ontologyData = new CBackendRepresentativeMemoryCacheOntologyData(&mContext);
366 ontologyData->initOntologyData(ontologyIdentifier, !mConfLateIndividualLabelAssociationIndexing);
367
368
369 mOntologyDataUpdateCount++;
370
371 CBackendRepresentativeMemoryCacheContext* tmpContext = ontologyData->getTemporaryContext();
372 CMemoryAllocationManager* tmpMemMan = tmpContext->getMemoryAllocationManager();
373 CCACHINGHASH<cint64, CBackendRepresentativeMemoryLabelSignatureResolveCacheItem>* slotSigLabelItemHashArray[CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_TYPE_COUNT];
374 for (cint64 i = 0; i < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_TYPE_COUNT; ++i) {
375 CCACHINGHASH<cint64, CBackendRepresentativeMemoryLabelSignatureResolveCacheItem>* ontoSigLabelItemHash = CObjectParameterizingAllocator< CCACHINGHASH<cint64, CBackendRepresentativeMemoryLabelSignatureResolveCacheItem>, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(tmpMemMan, tmpContext);
376 slotSigLabelItemHashArray[i] = ontoSigLabelItemHash;
377 ontologyData->setSignatureLabelItemHash(i, ontoSigLabelItemHash);
378
379 if (prevOntologyData) {
380 *ontoSigLabelItemHash = *prevOntologyData->getSignatureLabelItemHash(i);
381 ontoSigLabelItemHash->detach();
382 }
383 }
384
385 CCACHINGHASH<cint64, CBackendRepresentativeMemoryCacheNominalIndividualIndirectConnectionData* >* ontologyNominalIndiIdIndirectConnectionDataHash = CObjectParameterizingAllocator< CCACHINGHASH<cint64, CBackendRepresentativeMemoryCacheNominalIndividualIndirectConnectionData* >, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(tmpMemMan, tmpContext);
386 if (prevOntologyData) {
387 *ontologyNominalIndiIdIndirectConnectionDataHash = *prevOntologyData->getNominaIIndividualdIndirectConnectionDataHash();
388 ontologyNominalIndiIdIndirectConnectionDataHash->detach();
389 }
390 ontologyData->setNominaIIndividualdIndirectConnectionDataHash(ontologyNominalIndiIdIndirectConnectionDataHash);
391
392
393
394 if (!prevOntologyData) {
395 CBackendRepresentativeMemoryCacheOntologyContext* ontologyContext = new CBackendRepresentativeMemoryCacheOntologyContext(&mContext);
396 ontologyData->setOntologyContext(ontologyContext);
397 } else {
398 ontologyData->setOntologyContext(prevOntologyData->getOntologyContext());
399 }
400
401
402
403 if (!prevOntologyData) {
404 cint64 indiIdAssoDataVectorSize = qMax((cint64)1000, minIndiCount+1);
405 CBackendRepresentativeMemoryCacheIndividualAssociationData** indiIdAssoDataVector = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualAssociationData* >::allocateAndConstructArray(ontologyData->getOntologyContext()->getMemoryAllocationManager(), indiIdAssoDataVectorSize);
406 for (cint64 i = 0; i < indiIdAssoDataVectorSize; ++i) {
407 indiIdAssoDataVector[i] = nullptr;
408 }
409 ontologyData->setIndividualIdAssoiationDataVector(indiIdAssoDataVectorSize, indiIdAssoDataVector);
410 } else {
411 cint64 indiIdAssoDataVectorSize = prevOntologyData->getIndividualIdAssoiationDataVectorSize();
412 CBackendRepresentativeMemoryCacheIndividualAssociationData** indiIdAssoDataVector = prevOntologyData->getIndividualIdAssoiationDataVector();
413 ontologyData->setIndividualIdAssoiationDataVector(indiIdAssoDataVectorSize, indiIdAssoDataVector);
414 }
415
416 cint64 indiIdAssoDataVectorSize = ontologyData->getIndividualIdAssoiationDataVectorSize();
417 if (minIndiCount > indiIdAssoDataVectorSize) {
418 cint64 prevSize = indiIdAssoDataVectorSize;
419 CBackendRepresentativeMemoryCacheIndividualAssociationData** indiIdAssoDataVector = ontologyData->getIndividualIdAssoiationDataVector();
420 CBackendRepresentativeMemoryCacheIndividualAssociationData** prevIndiIdAssoDataVector = indiIdAssoDataVector;
421 indiIdAssoDataVectorSize = minIndiCount+1;
422 indiIdAssoDataVector = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualAssociationData* >::allocateAndConstructArray(ontologyData->getOntologyContext()->getMemoryAllocationManager(), indiIdAssoDataVectorSize);
423 for (cint64 i = 0; i < prevSize; ++i) {
424 indiIdAssoDataVector[i] = prevIndiIdAssoDataVector[i];
425 }
426 for (cint64 i = prevSize; i < indiIdAssoDataVectorSize; ++i) {
427 indiIdAssoDataVector[i] = nullptr;
428 }
429 ontologyData->setIndividualIdAssoiationDataVector(indiIdAssoDataVectorSize, indiIdAssoDataVector);
430 }
431
432
433
434 if (prevOntologyData) {
435 ontologyData->copyOntologyData(prevOntologyData);
436 }
437
438 CBackendRepresentativeMemoryCacheOntologyDataRecomputationReferenceLinker* linker = new CBackendRepresentativeMemoryCacheOntologyDataRecomputationReferenceLinker(ontologyData->getOntologyDataUpdateId());
439 ontologyData->setRecomputationReferenceLinker(linker);
440
441
442 ontologyData->incUsageCount();
443 if (prevOntologyData) {
444 prevOntologyData->decUsageCount();
445 if (prevOntologyData->getUsageCount() <= 0) {
446 ++mOntologyDataReleasedCount;
447 ++mOntologyDataReleasedWhileNewCreationCount;
448 CMemoryPoolAllocationManager* memMan = mContext.getMemoryPoolAllocationManager();
449 CMemoryPool* memoryPools = prevOntologyData->getTemporaryContext()->getMemoryPoolContainer()->getMemoryPools();
450 memMan->releaseTemporaryMemoryPools(memoryPools);
451 delete prevOntologyData;
452 }
453
454 }
455 }
456
457 return ontologyData;
458 }
459
460
461
462
463
464
465 void CBackendRepresentativeMemoryCache::cleanUnusedSlots(CBackendRepresentativeMemoryCacheBaseContext* context) {
466 CMemoryPoolAllocationManager* memMan = context->getMemoryPoolAllocationManager();
467 CBackendRepresentativeMemoryCacheSlotItem* slotLinkerIt = mSlotLinker;
468 CBackendRepresentativeMemoryCacheSlotItem* lastSlotLinker = nullptr;
469 while (slotLinkerIt) {
470 CBackendRepresentativeMemoryCacheSlotItem* tmpSlotLinker = nullptr;
471 if (!slotLinkerIt->hasCacheReaders()) {
472 tmpSlotLinker = slotLinkerIt;
473 }
474 CBackendRepresentativeMemoryCacheSlotItem* nextSlotLinker = slotLinkerIt->getNext();
475 if (tmpSlotLinker) {
476 if (!lastSlotLinker) {
477 mSlotLinker = nextSlotLinker;
478 } else {
479 lastSlotLinker->setNext(nextSlotLinker);
480 }
481
482 CCACHINGHASH<cint64, CBackendRepresentativeMemoryCacheOntologyData*>* ontologyIdentifierDataHash = slotLinkerIt->getOntologyIdentifierDataHash();
483 for (CCACHINGHASH<cint64, CBackendRepresentativeMemoryCacheOntologyData*>::const_iterator it = ontologyIdentifierDataHash->constBegin(), itEnd = ontologyIdentifierDataHash->constEnd(); it != itEnd; ++it) {
484 CBackendRepresentativeMemoryCacheOntologyData* ontologyData = it.value();
485 ontologyData->decUsageCount();
486 if (ontologyData->getUsageCount() <= 0) {
487 ++mOntologyDataReleasedCount;
488 ++mOntologyDataReleasedWhileSlotUpdateCount;
489
490 ontologyData->getRecomputationReferenceLinker()->setOntologyDataInactive();
491
492 CMemoryPool* memoryPools = ontologyData->getTemporaryContext()->getMemoryPoolContainer()->getMemoryPools();
493 memMan->releaseTemporaryMemoryPools(memoryPools);
494 delete ontologyData;
495 }
496 }
497
498 CMemoryPool* memoryPools = tmpSlotLinker->getMemoryPools();
499 memMan->releaseTemporaryMemoryPools(memoryPools);
500 ++mReaderSlotReleasedCount;
501 }
502 else {
503 lastSlotLinker = slotLinkerIt;
504 }
505 slotLinkerIt = nextSlotLinker;
506 }
507 }
508
509
510
511 CBackendRepresentativeMemoryCacheReader* CBackendRepresentativeMemoryCache::createCacheReader() {
512 CBackendRepresentativeMemoryCacheReader* reader = new CBackendRepresentativeMemoryCacheReader();
513 mReaderSyncMutex.lock();
514 mReaderLinker = reader->append(mReaderLinker);
515 mReaderSyncMutex.unlock();
516 return reader;
517 }
518
519
520 CBackendRepresentativeMemoryCacheReader* CBackendRepresentativeMemoryCache::createOntologyFixedCacheReader(cint64 ontologyIdentifier) {
521 CBackendRepresentativeMemoryCacheReader* reader = new CBackendRepresentativeMemoryCacheReader();
522 mFixedOntologyIdentifierDataHashLock.lockForRead();
523 CBackendRepresentativeMemoryCacheOntologyData* ontologyData = mFixedOntologyIdentifierDataHash.value(ontologyIdentifier);
524 if (ontologyData) {
525 ontologyData->incUsageCount();
526 }
527 reader->fixOntologyData(ontologyData);
528 mFixedOntologyIdentifierDataHashLock.unlock();
529 ontologyData->waitIndividualLabelAssociationIndexed();
530 return reader;
531 }
532
533
534 CBackendRepresentativeMemoryCacheWriter* CBackendRepresentativeMemoryCache::createCacheWriter() {
535 CBackendRepresentativeMemoryCacheWriter* writer = new CBackendRepresentativeMemoryCacheWriter(this);
536 return writer;
537 }
538
539
540
541 CBackendRepresentativeMemoryCache* CBackendRepresentativeMemoryCache::writeCachedData(CBackendRepresentativeMemoryCacheWriteData* writeData, CMemoryPool* memoryPools) {
542 if (mStatCollectStatistics) {
543 mPendingUpdateCount.ref();
544 }
545 if (mLimitRemainingWritePending) {
546 mRemainingWritePendingSemaphore.acquire();
547 }
548 if (mConfDirectUpdateSynchronization) {
549 mDirectUpdateSyncMutex.lock();
550 CWriteBackendAssociationCachedEvent* procEvent = new CWriteBackendAssociationCachedEvent(writeData, memoryPools);
551 processCustomsEvents(CWriteBackendAssociationCachedEvent::EVENTTYPE, procEvent);
552 mDirectUpdateSyncMutex.unlock();
553 } else {
554 postEvent(new CWriteBackendAssociationCachedEvent(writeData, memoryPools));
555 }
556 return this;
557 }
558
559
560
561
562 void CBackendRepresentativeMemoryCache::installTemporaryCardinalities(CBackendRepresentativeMemoryCacheTemporaryCardinalityWriteDataLinker* tempCardWriteDataLinker, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
563 CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
564 for (CBackendRepresentativeMemoryCacheTemporaryCardinalityWriteDataLinker* tempCardWriteDataLinkerIt = tempCardWriteDataLinker; tempCardWriteDataLinkerIt; tempCardWriteDataLinkerIt = tempCardWriteDataLinkerIt->getNext()) {
565
566 CBackendRepresentativeMemoryCacheTemporaryLabelWriteDataLinker* labelWriteDataLinker = tempCardWriteDataLinkerIt->getLabelWriteDataLinker();
567 CBackendRepresentativeMemoryLabelCacheItem* labelCacheItem = (CBackendRepresentativeMemoryLabelCacheItem*)labelWriteDataLinker->getTemporaryData();
568
569 CBackendRepresentativeMemoryCardinalityValueLinker* cardValueLinker = tempCardWriteDataLinkerIt->getCardinalityCacheValueLinker();
570
571 bool handled = false;
572 CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData* extensionData = (CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData*)labelCacheItem->getExtensionData(CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData::CARDINALITY_HASH);
573 if (extensionData) {
574 handled = true;
575 for (CBackendRepresentativeMemoryCardinalityValueLinker* cardValueLinkerIt = cardValueLinker; cardValueLinkerIt; cardValueLinkerIt = cardValueLinkerIt->getNext()) {
576 cint64 roleTag = cardValueLinkerIt->getRoleTag();
577 CBackendRepresentativeMemoryLabelCacheItemCardinalityData* roleCardData = extensionData->getRoleCardinalityData(roleTag);
578 if (roleCardData) {
579 roleCardData->updateExistentialMaxUsedCardinality(cardValueLinkerIt->getExistentialMaxUsedCardinality());
580 roleCardData->updateMinimumRestrictingCardinality(cardValueLinkerIt->getMinimalRestrictingCardinality());
581 } else {
582 handled = false;
583 }
584 }
585 }
586
587 if (!handled) {
588 CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData* newExtensionData = CObjectParameterizingAllocator< CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
589 newExtensionData->initCardinalityExtensionData();
590 for (CBackendRepresentativeMemoryCardinalityValueLinker* cardValueLinkerIt = cardValueLinker; cardValueLinkerIt; cardValueLinkerIt = cardValueLinkerIt->getNext()) {
591 cint64 roleTag = cardValueLinkerIt->getRoleTag();
592 cint64 cardCount = cardValueLinkerIt->getExistentialMaxUsedCardinality();
593 cint64 minRestCount = cardValueLinkerIt->getMinimalRestrictingCardinality();
594 CBackendRepresentativeMemoryLabelCacheItemCardinalityData* cardData = CObjectAllocator< CBackendRepresentativeMemoryLabelCacheItemCardinalityData >::allocateAndConstruct(context->getMemoryAllocationManager());
595 cardData->initCardinalityData(cardCount, minRestCount);
596 if (extensionData) {
597 CBackendRepresentativeMemoryLabelCacheItemCardinalityData* roleCardData = extensionData->getRoleCardinalityData(roleTag);
598 cardData->updateExistentialMaxUsedCardinality(roleCardData->getExistentialMaxUsedCardinality());
599 cardData->updateMinimumRestrictingCardinality(roleCardData->getMinimumRestrictingCardinality());
600 }
601 newExtensionData->setRoleCardinalityData(roleTag, cardData);
602 }
603
604 labelCacheItem->setExtensionData(CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData::CARDINALITY_HASH, newExtensionData);
605 }
606 }
607 }
608
609
610
611
612
613
614 void CBackendRepresentativeMemoryCache::installTemporaryLabels(CBackendRepresentativeMemoryCacheTemporaryLabelWriteDataLinker* tempLabelWriteDataLinker, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
615 CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
616 for (CBackendRepresentativeMemoryCacheTemporaryLabelWriteDataLinker* tempLabelWriteDataLinkerIt = tempLabelWriteDataLinker; tempLabelWriteDataLinkerIt; tempLabelWriteDataLinkerIt = tempLabelWriteDataLinkerIt->getNext()) {
617
618 cint64 signature = tempLabelWriteDataLinkerIt->getSignature();
619 if (tempLabelWriteDataLinkerIt->getLabelType() == CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL) {
620 signature = 0;
621 for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = tempLabelWriteDataLinkerIt->getCacheValueLinker(); labelValueLinkerIt; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
622 CCacheValue cacheValue = labelValueLinkerIt->getCacheValue();
623 if (cacheValue.getCacheValueIdentifier() == CCacheValue::CACHE_VALUE_TAG_AND_TEMPORARY_ENTRY) {
624 CBackendRepresentativeMemoryCacheTemporaryLabelWriteDataLinker* tmpLabelWriteData = (CBackendRepresentativeMemoryCacheTemporaryLabelWriteDataLinker*)cacheValue.getIdentification();
625 CBackendRepresentativeMemoryLabelCacheItem* labelCacheItem = (CBackendRepresentativeMemoryLabelCacheItem*)tmpLabelWriteData->getTemporaryData();
626 cacheValue.initCacheValue(labelCacheItem->getCacheEntryID(), (cint64)labelCacheItem, CCacheValue::CACHE_VALUE_TAG_AND_ENTRY);
627 labelValueLinkerIt->setCacheValue(cacheValue);
628 }
629 signature = CBackendRepresentativeMemoryCacheUtilities::getSignatureExtensionByCacheValue(signature, cacheValue);
630 }
631 // update signature
632 tempLabelWriteDataLinkerIt->setSignature(signature);
633 }
634
635 CBackendRepresentativeMemoryLabelValueLinker* labelValueLinker = tempLabelWriteDataLinkerIt->getCacheValueLinker();
636 cint64 labelValueCount = tempLabelWriteDataLinkerIt->getCacheValueCount();
637 CBackendRepresentativeMemoryLabelSignatureResolveCacheItem& sigResolveItem = (*ontologyData->getSignatureLabelItemHash(tempLabelWriteDataLinkerIt->getLabelType()))[signature];
638
639 CBackendRepresentativeMemoryLabelCacheItem* refLabelItem = nullptr;
640 for (CBackendRepresentativeMemoryLabelCacheItem* labelItemLinkerIt = sigResolveItem.getLabelItems(); labelItemLinkerIt && !refLabelItem; labelItemLinkerIt = labelItemLinkerIt->getNext()) {
641 if (labelItemLinkerIt->getCacheValueCount() == labelValueCount) {
642 CCACHINGHASH<cint64, CBackendRepresentativeMemoryLabelValueLinker*>* labelValueHash = labelItemLinkerIt->getTagCacheValueHash(false);
643 bool identical = true;
644 for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = labelValueLinker; labelValueLinkerIt && identical; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
645 const CCacheValue& cacheValue = labelValueLinkerIt->getCacheValue();
646 cint64 valueTag = cacheValue.getTag();
647 CBackendRepresentativeMemoryLabelValueLinker* labelValue = labelValueHash->value(valueTag);
648 if (!labelValue || labelValue->getCacheValue() != cacheValue) {
649 identical = false;
650 }
651 }
652
653 if (identical) {
654 refLabelItem = labelItemLinkerIt;
655 }
656 }
657 }
658
659 if (!refLabelItem) {
660 refLabelItem = CObjectParameterizingAllocator< CBackendRepresentativeMemoryLabelCacheItem, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
661 refLabelItem->initCacheEntry(signature, ontologyData->getNextEntryID(), (CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_TYPE)tempLabelWriteDataLinkerIt->getLabelType());
662
663 CCACHINGHASH<cint64, CBackendRepresentativeMemoryLabelValueLinker*>* labelValueHash = refLabelItem->getTagCacheValueHash(true);
664 for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = labelValueLinker; labelValueLinkerIt; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
665
666 CBackendRepresentativeMemoryLabelValueLinker*& newValueLinker = (*labelValueHash)[labelValueLinkerIt->getCacheValue().getTag()];
667 if (!newValueLinker) {
668 CBackendRepresentativeMemoryLabelValueLinker* newValueLinker = CObjectAllocator< CBackendRepresentativeMemoryLabelValueLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
669 newValueLinker->initLabelValueLinker(labelValueLinkerIt->getCacheValue());
670 labelValueHash->insert(labelValueLinkerIt->getCacheValue().getTag(), newValueLinker);
671 refLabelItem->addCacheValueLinker(newValueLinker);
672 }
673 }
674 addCreatedLabelStatistics(tempLabelWriteDataLinkerIt->getLabelType(), refLabelItem, ontologyData);
675 sigResolveItem.appendLabelItem(refLabelItem);
676 }
677
678 if (!refLabelItem->isCompletelyHandled() && tempLabelWriteDataLinkerIt->isCompletelyHandled()) {
679 refLabelItem->setCompletelyHandled(true);
680 }
681 if (!refLabelItem->isCompletelySaturated() && tempLabelWriteDataLinkerIt->isCompletelySaturated()) {
682 refLabelItem->setCompletelySaturated(true);
683 }
684 if (!refLabelItem->hasNondeterministicElements() && tempLabelWriteDataLinkerIt->hasNondeterministicElements()) {
685 refLabelItem->setNondeterministicElements(true);
686 }
687
688
689 if (tempLabelWriteDataLinkerIt->getLabelType() == CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL) {
690 getNeighbourArrayRoleTagResolvingLabelExtensionData(refLabelItem, ontologyData);
691 }
692
693 tempLabelWriteDataLinkerIt->setTemporaryData(refLabelItem);
694 }
695
696 }
697
698
699 CBackendRepresentativeMemoryCache* CBackendRepresentativeMemoryCache::addCreatedLabelStatistics(cint64 labelType, CBackendRepresentativeMemoryLabelCacheItem* label, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
700 ++mStatLabelCount;
701 mStatMaxLabelValueCount = qMax(mStatMaxLabelValueCount, label->getCacheValueCount());
702 mStatLabelTypeCount[labelType]++;
703 mStatLabelTypeMaxValueCount[labelType] = qMax(mStatLabelTypeMaxValueCount[labelType], label->getCacheValueCount());
704 mStatLabelTypeAllValueCount[labelType] += label->getCacheValueCount();
705 return this;
706 }
707
708
709
710 CBackendRepresentativeMemoryLabelCacheItem* CBackendRepresentativeMemoryCache::getExtendedLabel(cint64 labelType, CBackendRepresentativeMemoryLabelCacheItem* label, CCacheValue extendingCacheValue, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
711 CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
712 if (label->getTagCacheValueHash(false)->contains(extendingCacheValue.getTag())) {
713 return label;
714 }
715 CBackendRepresentativeMemoryLabelValueLinker* labelValueLinker = label->getCacheValueLinker();
716 cint64 signature = label->getSignature() + qHash((qint64)extendingCacheValue.getTag());
717 cint64 newCount = label->getCacheValueCount() + 1;
718
719 // test with signature whether extended label already exists
720 CBackendRepresentativeMemoryLabelSignatureResolveCacheItem& sigResolveItem = (*ontologyData->getSignatureLabelItemHash(labelType))[signature];
721
722 for (CBackendRepresentativeMemoryLabelCacheItem* labelItemLinkerIt = sigResolveItem.getLabelItems(); labelItemLinkerIt; labelItemLinkerIt = labelItemLinkerIt->getNext()) {
723 if (labelItemLinkerIt->getCacheValueCount() == newCount) {
724 CCACHINGHASH<cint64, CBackendRepresentativeMemoryLabelValueLinker*>* labelValueHash = labelItemLinkerIt->getTagCacheValueHash(false);
725 bool identical = true;
726
727 CBackendRepresentativeMemoryLabelValueLinker* labelValue = labelValueHash->value(extendingCacheValue.getTag());
728 if (!labelValue || labelValue->getCacheValue() != extendingCacheValue) {
729 identical = false;
730 }
731
732 for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = labelValueLinker; labelValueLinkerIt && identical; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
733 const CCacheValue& cacheValue = labelValueLinkerIt->getCacheValue();
734 cint64 valueTag = cacheValue.getTag();
735 CBackendRepresentativeMemoryLabelValueLinker* labelValue = labelValueHash->value(valueTag);
736 if (!labelValue || labelValue->getCacheValue() != cacheValue) {
737 identical = false;
738 }
739 }
740
741 if (identical) {
742 return labelItemLinkerIt;
743 }
744 }
745 }
746
747
748 // create new label item
749 CBackendRepresentativeMemoryLabelCacheItem* refLabelItem = CObjectParameterizingAllocator< CBackendRepresentativeMemoryLabelCacheItem, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
750 refLabelItem->initCacheEntry(signature, ontologyData->getNextEntryID(), (CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_TYPE)labelType);
751
752 CCACHINGHASH<cint64, CBackendRepresentativeMemoryLabelValueLinker*>* labelValueHash = refLabelItem->getTagCacheValueHash(true);
753 for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = labelValueLinker; labelValueLinkerIt; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
754
755 const CCacheValue& cacheValue = labelValueLinkerIt->getCacheValue();
756 CBackendRepresentativeMemoryLabelValueLinker* newValueLinker = CObjectAllocator< CBackendRepresentativeMemoryLabelValueLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
757 newValueLinker->initLabelValueLinker(cacheValue);
758 labelValueHash->insert(cacheValue.getTag(), newValueLinker);
759 refLabelItem->addCacheValueLinker(newValueLinker);
760 }
761
762 CBackendRepresentativeMemoryLabelValueLinker* newValueLinker = CObjectAllocator< CBackendRepresentativeMemoryLabelValueLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
763 newValueLinker->initLabelValueLinker(extendingCacheValue);
764 labelValueHash->insert(extendingCacheValue.getTag(), newValueLinker);
765 refLabelItem->addCacheValueLinker(newValueLinker);
766
767 if (refLabelItem->getLabelType() == CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL) {
768 getNeighbourArrayRoleTagResolvingLabelExtensionData(refLabelItem, ontologyData);
769 }
770 if (refLabelItem->getLabelType() == CBackendRepresentativeMemoryLabelCacheItem::FULL_CONCEPT_SET_LABEL) {
771
772
773 CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData* extensionData = (CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData*)label->getExtensionData(CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData::CARDINALITY_HASH);
774 if (extensionData) {
775
776 CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData* newExtensionData = CObjectParameterizingAllocator< CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
777 newExtensionData->initCardinalityExtensionData();
778 CCACHINGHASH<cint64, CBackendRepresentativeMemoryLabelCacheItemCardinalityData*>* newExtensionDataHash = newExtensionData->getRoleCardinalityDataHash();
779
780 for (CCACHINGHASH<cint64, CBackendRepresentativeMemoryLabelCacheItemCardinalityData*>::const_iterator it = extensionData->getRoleCardinalityDataHash()->constBegin(), itEnd = extensionData->getRoleCardinalityDataHash()->constEnd(); it != itEnd; ++it) {
781 cint64 roleTag = it.key();
782 CBackendRepresentativeMemoryLabelCacheItemCardinalityData* cardData = it.value();
783 cint64 cardCount = cardData->getExistentialMaxUsedCardinality();
784 cint64 minRestCount = cardData->getMinimumRestrictingCardinality();
785
786 CBackendRepresentativeMemoryLabelCacheItemCardinalityData*& roleCardData = (*newExtensionDataHash)[roleTag];
787 if (!roleCardData) {
788 roleCardData = CObjectAllocator< CBackendRepresentativeMemoryLabelCacheItemCardinalityData >::allocateAndConstruct(context->getMemoryAllocationManager());
789 roleCardData->initCardinalityData(cardCount, minRestCount);
790 } else {
791 roleCardData->updateExistentialMaxUsedCardinality(cardCount);
792 roleCardData->updateMinimumRestrictingCardinality(minRestCount);
793 }
794 }
795
796 refLabelItem->setExtensionData(CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData::CARDINALITY_HASH, newExtensionData);
797
798
799 }
800
801 }
802
803 addCreatedLabelStatistics(labelType, refLabelItem, ontologyData);
804 sigResolveItem.appendLabelItem(refLabelItem);
805 return refLabelItem;
806 }
807
808
809
810
811 CBackendRepresentativeMemoryLabelCacheItem* CBackendRepresentativeMemoryCache::getReducedLabel(cint64 labelType, CBackendRepresentativeMemoryLabelCacheItem* label, function<bool(const CCacheValue& cacheValue)> reduceCheckFunction, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
812 CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
813 CBackendRepresentativeMemoryLabelValueLinker* labelValueLinker = label->getCacheValueLinker();
814 cint64 signature = 0;
815 cint64 newCount = 0;
816 for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = labelValueLinker; labelValueLinkerIt; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
817 CCacheValue cacheValue = labelValueLinkerIt->getCacheValue();
818 if (!reduceCheckFunction(cacheValue)) {
819 // extend signature/hash value
820 signature += qHash((qint64)cacheValue.getTag());
821 newCount++;
822 }
823 }
824
825
826 if (newCount == label->getCacheValueCount()) {
827 return label;
828 }
829
830 // test with signature whether reduced label already exists
831 CBackendRepresentativeMemoryLabelSignatureResolveCacheItem& sigResolveItem = (*ontologyData->getSignatureLabelItemHash(labelType))[signature];
832
833 for (CBackendRepresentativeMemoryLabelCacheItem* labelItemLinkerIt = sigResolveItem.getLabelItems(); labelItemLinkerIt; labelItemLinkerIt = labelItemLinkerIt->getNext()) {
834 if (labelItemLinkerIt->getCacheValueCount() == newCount) {
835 CCACHINGHASH<cint64, CBackendRepresentativeMemoryLabelValueLinker*>* labelValueHash = labelItemLinkerIt->getTagCacheValueHash(false);
836 bool identical = true;
837 for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = labelValueLinker; labelValueLinkerIt && identical; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
838 const CCacheValue& cacheValue = labelValueLinkerIt->getCacheValue();
839 if (!reduceCheckFunction(cacheValue)) {
840 cint64 valueTag = cacheValue.getTag();
841 CBackendRepresentativeMemoryLabelValueLinker* labelValue = labelValueHash->value(valueTag);
842 if (!labelValue || labelValue->getCacheValue() != cacheValue) {
843 identical = false;
844 }
845 }
846 }
847
848 if (identical) {
849 return labelItemLinkerIt;
850 }
851 }
852 }
853
854
855 // create new label item
856 CBackendRepresentativeMemoryLabelCacheItem* refLabelItem = CObjectParameterizingAllocator< CBackendRepresentativeMemoryLabelCacheItem, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
857 refLabelItem->initCacheEntry(signature, ontologyData->getNextEntryID(), (CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_TYPE)labelType);
858
859 CCACHINGHASH<cint64, CBackendRepresentativeMemoryLabelValueLinker*>* labelValueHash = refLabelItem->getTagCacheValueHash(true);
860 for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = labelValueLinker; labelValueLinkerIt; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
861
862 const CCacheValue& cacheValue = labelValueLinkerIt->getCacheValue();
863 if (!reduceCheckFunction(cacheValue)) {
864 CBackendRepresentativeMemoryLabelValueLinker* newValueLinker = CObjectAllocator< CBackendRepresentativeMemoryLabelValueLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
865 newValueLinker->initLabelValueLinker(cacheValue);
866 labelValueHash->insert(cacheValue.getTag(), newValueLinker);
867 refLabelItem->addCacheValueLinker(newValueLinker);
868 }
869 }
870
871 if (refLabelItem->getLabelType() == CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL) {
872 getNeighbourArrayRoleTagResolvingLabelExtensionData(refLabelItem, ontologyData);
873 }
874
875
876 addCreatedLabelStatistics(labelType, refLabelItem, ontologyData);
877 sigResolveItem.appendLabelItem(refLabelItem);
878 return refLabelItem;
879 }
880
881
882
883 CBackendRepresentativeMemoryLabelCacheItem* CBackendRepresentativeMemoryCache::getAdditionMergedLabel(cint64 labelType, CBackendRepresentativeMemoryLabelCacheItem* additionLabel, CBackendRepresentativeMemoryLabelCacheItem* associatedlabel, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
884 CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
885 if (!additionLabel) {
886 return associatedlabel;
887 }
888 if (!associatedlabel) {
889 return additionLabel;
890 }
891 if (additionLabel == associatedlabel) {
892 return associatedlabel;
893 }
894 CBackendRepresentativeMemoryLabelValueLinker* additionLabelValueLinker = additionLabel->getCacheValueLinker();
895 CBackendRepresentativeMemoryLabelValueLinker* associatedLabelValueLinker = associatedlabel->getCacheValueLinker();
896 CCACHINGHASH<cint64, CBackendRepresentativeMemoryLabelValueLinker*>* associatedLabelValueHash = associatedlabel->getTagCacheValueHash(false);
897 // check whether all values are already included
898 bool valuesAlreadyIncluded = true;
899 cint64 signature = associatedlabel->getSignature();
900 cint64 newCount = associatedlabel->getCacheValueCount();
901 for (CBackendRepresentativeMemoryLabelValueLinker* additionLabelValueLinkerIt = additionLabelValueLinker; additionLabelValueLinkerIt; additionLabelValueLinkerIt = additionLabelValueLinkerIt->getNext()) {
902 CCacheValue cacheValue = additionLabelValueLinkerIt->getCacheValue();
903 if (!associatedLabelValueHash || !associatedLabelValueHash->contains(cacheValue.getTag())) {
904 valuesAlreadyIncluded = false;
905 // extend signature/hash value
906 signature += qHash((qint64)cacheValue.getTag());
907 newCount++;
908 }
909 }
910 if (valuesAlreadyIncluded) {
911 return associatedlabel;
912 }
913 // test with signature whether extended label already exists
914 CBackendRepresentativeMemoryLabelSignatureResolveCacheItem& sigResolveItem = (*ontologyData->getSignatureLabelItemHash(labelType))[signature];
915
916 for (CBackendRepresentativeMemoryLabelCacheItem* labelItemLinkerIt = sigResolveItem.getLabelItems(); labelItemLinkerIt; labelItemLinkerIt = labelItemLinkerIt->getNext()) {
917 if (labelItemLinkerIt->getCacheValueCount() == newCount) {
918 CCACHINGHASH<cint64, CBackendRepresentativeMemoryLabelValueLinker*>* labelValueHash = labelItemLinkerIt->getTagCacheValueHash(false);
919 bool identical = true;
920 for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = associatedLabelValueLinker; labelValueLinkerIt && identical; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
921 const CCacheValue& cacheValue = labelValueLinkerIt->getCacheValue();
922 cint64 valueTag = cacheValue.getTag();
923 CBackendRepresentativeMemoryLabelValueLinker* labelValue = labelValueHash->value(valueTag);
924 if (!labelValue || labelValue->getCacheValue() != cacheValue) {
925 identical = false;
926 }
927 }
928 for (CBackendRepresentativeMemoryLabelValueLinker* additionLabelValueLinkerIt = additionLabelValueLinker; additionLabelValueLinkerIt && identical; additionLabelValueLinkerIt = additionLabelValueLinkerIt->getNext()) {
929 const CCacheValue& cacheValue = additionLabelValueLinkerIt->getCacheValue();
930 cint64 valueTag = cacheValue.getTag();
931 CBackendRepresentativeMemoryLabelValueLinker* labelValue = labelValueHash->value(valueTag);
932 if (!labelValue || labelValue->getCacheValue() != cacheValue) {
933 identical = false;
934 }
935 }
936
937 if (identical) {
938 return labelItemLinkerIt;
939 }
940 }
941 }
942
943
944 // create new label item
945 CBackendRepresentativeMemoryLabelCacheItem* refLabelItem = CObjectParameterizingAllocator< CBackendRepresentativeMemoryLabelCacheItem, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
946 refLabelItem->initCacheEntry(signature, ontologyData->getNextEntryID(), (CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_TYPE)labelType);
947
948 CCACHINGHASH<cint64, CBackendRepresentativeMemoryLabelValueLinker*>* labelValueHash = refLabelItem->getTagCacheValueHash(true);
949 for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = associatedLabelValueLinker; labelValueLinkerIt; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
950
951 CBackendRepresentativeMemoryLabelValueLinker* newValueLinker = CObjectAllocator< CBackendRepresentativeMemoryLabelValueLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
952 newValueLinker->initLabelValueLinker(labelValueLinkerIt->getCacheValue());
953 labelValueHash->insert(labelValueLinkerIt->getCacheValue().getTag(), newValueLinker);
954 refLabelItem->addCacheValueLinker(newValueLinker);
955 }
956
957 for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = additionLabelValueLinker; labelValueLinkerIt; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
958 CBackendRepresentativeMemoryLabelValueLinker*& newValueLinker = (*labelValueHash)[labelValueLinkerIt->getCacheValue().getTag()];
959 if (!newValueLinker) {
960 newValueLinker = CObjectAllocator< CBackendRepresentativeMemoryLabelValueLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
961 newValueLinker->initLabelValueLinker(labelValueLinkerIt->getCacheValue());
962 labelValueHash->insert(labelValueLinkerIt->getCacheValue().getTag(), newValueLinker);
963 refLabelItem->addCacheValueLinker(newValueLinker);
964 }
965 }
966
967 if (refLabelItem->getLabelType() == CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL) {
968 getNeighbourArrayRoleTagResolvingLabelExtensionData(refLabelItem, ontologyData);
969 }
970 if (refLabelItem->getLabelType() == CBackendRepresentativeMemoryLabelCacheItem::FULL_CONCEPT_SET_LABEL) {
971
972
973 CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData* extensionData1 = (CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData*)additionLabel->getExtensionData(CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData::CARDINALITY_HASH);
974 CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData* extensionData2 = (CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData*)associatedlabel->getExtensionData(CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData::CARDINALITY_HASH);
975 if (extensionData1 || extensionData2) {
976
977
978 CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData* newExtensionData = CObjectParameterizingAllocator< CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
979 newExtensionData->initCardinalityExtensionData();
980 CCACHINGHASH<cint64, CBackendRepresentativeMemoryLabelCacheItemCardinalityData*>* newExtensionDataHash = newExtensionData->getRoleCardinalityDataHash();
981
982 for (CCACHINGHASH<cint64, CBackendRepresentativeMemoryLabelCacheItemCardinalityData*>::const_iterator it = extensionData1->getRoleCardinalityDataHash()->constBegin(), itEnd = extensionData1->getRoleCardinalityDataHash()->constEnd(); it != itEnd; ++it) {
983 cint64 roleTag = it.key();
984 CBackendRepresentativeMemoryLabelCacheItemCardinalityData* cardData = it.value();
985 cint64 cardCount = cardData->getExistentialMaxUsedCardinality();
986 cint64 minRestCount = cardData->getMinimumRestrictingCardinality();
987
988 CBackendRepresentativeMemoryLabelCacheItemCardinalityData*& roleCardData = (*newExtensionDataHash)[roleTag];
989 if (!roleCardData) {
990 roleCardData = CObjectAllocator< CBackendRepresentativeMemoryLabelCacheItemCardinalityData >::allocateAndConstruct(context->getMemoryAllocationManager());
991 roleCardData->initCardinalityData(cardCount, minRestCount);
992 } else {
993 roleCardData->updateExistentialMaxUsedCardinality(cardCount);
994 roleCardData->updateMinimumRestrictingCardinality(minRestCount);
995 }
996 }
997 for (CCACHINGHASH<cint64, CBackendRepresentativeMemoryLabelCacheItemCardinalityData*>::const_iterator it = extensionData2->getRoleCardinalityDataHash()->constBegin(), itEnd = extensionData2->getRoleCardinalityDataHash()->constEnd(); it != itEnd; ++it) {
998 cint64 roleTag = it.key();
999 CBackendRepresentativeMemoryLabelCacheItemCardinalityData* cardData = it.value();
1000 cint64 cardCount = cardData->getExistentialMaxUsedCardinality();
1001 cint64 minRestCount = cardData->getMinimumRestrictingCardinality();
1002
1003 CBackendRepresentativeMemoryLabelCacheItemCardinalityData*& roleCardData = (*newExtensionDataHash)[roleTag];
1004 if (!roleCardData) {
1005 roleCardData = CObjectAllocator< CBackendRepresentativeMemoryLabelCacheItemCardinalityData >::allocateAndConstruct(context->getMemoryAllocationManager());
1006 roleCardData->initCardinalityData(cardCount, minRestCount);
1007 } else {
1008 roleCardData->updateExistentialMaxUsedCardinality(cardCount);
1009 roleCardData->updateMinimumRestrictingCardinality(minRestCount);
1010 }
1011 }
1012
1013 refLabelItem->setExtensionData(CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData::CARDINALITY_HASH, newExtensionData);
1014
1015
1016 }
1017
1018 }
1019
1020 addCreatedLabelStatistics(labelType, refLabelItem, ontologyData);
1021 sigResolveItem.appendLabelItem(refLabelItem);
1022 return refLabelItem;
1023
1024 }
1025
1026
1027
1028
1029
1030 bool CBackendRepresentativeMemoryCache::checkAssociationUsage(CBackendRepresentativeMemoryCacheTemporaryAssociationUseDataLinker* tempAssUseDataLinker, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
1031 CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
1032 bool associationsUpdated = false;
1033 for (CBackendRepresentativeMemoryCacheTemporaryAssociationUseDataLinker* tempAssUseDataLinkerIt = tempAssUseDataLinker; tempAssUseDataLinkerIt; tempAssUseDataLinkerIt = tempAssUseDataLinkerIt->getNext()) {
1034 cint64 individualID = tempAssUseDataLinkerIt->getIndividualID();
1035 cint64 usedUpateID = tempAssUseDataLinkerIt->getUsedAssociationUpdateId();
1036
1037 CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData = nullptr;
1038 if (ontologyData->getIndividualIdAssoiationDataVector() && individualID < ontologyData->getIndividualIdAssoiationDataVectorSize()) {
1039 associationData = ontologyData->getIndividualIdAssoiationDataVector()[individualID];
1040 }
1041
1042 mCheckedIndiCount++;
1043
1044 if (!associationData || (associationData->getAssociationDataUpdateId() != usedUpateID && associationData->isCompletelyHandled())) {
1045
1046 associationsUpdated |= markRepresentativeReferencedIndividualAssociationIncompletelyHandled(individualID, associationData, ontologyData);
1047
1048 mCheckIncompatibleIndiCount++;
1049
1050
1051 }
1052 }
1053 return associationsUpdated;
1054 }
1055
1056
1057
1058
1059
1060
1061 bool CBackendRepresentativeMemoryCache::markRepresentativeReferencedIndividualAssociationIncompletelyHandled(cint64 indiId, CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
1062 bool associationsUpdated = false;
1063 if (!associationData || !associationData->hasRepresentativeSameIndividualMerging()) {
1064 DEBUG_HANDLING_STRING(QString("Marking %1 incompletly handled").arg(associationData->getAssociatedIndividualId()))
1065 associationsUpdated = markIndividualAssociationIncompletelyHandled(indiId, associationData, ontologyData);
1066 } else {
1067 DEBUG_HANDLING_STRING(QString("Marking same representative individual %2 incompletly handled for %1").arg(associationData->getAssociatedIndividualId()).arg(associationData->getRepresentativeSameIndividualId()))
1068 CBackendRepresentativeMemoryCacheIndividualAssociationData* repAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[associationData->getRepresentativeSameIndividualId()];
1069 associationsUpdated = markIndividualAssociationIncompletelyHandled(repAssociationData->getRepresentativeSameIndividualId(), repAssociationData, ontologyData);
1070 }
1071 return associationsUpdated;
1072 }
1073
1074
1075
1076 bool CBackendRepresentativeMemoryCache::markIndividualAssociationIncompletelyHandled(cint64 individualID, CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
1077 bool associationsUpdated = false;
1078
1079 if (!associationData || associationData->isCompletelyHandled()) {
1080
1081 CBackendRepresentativeMemoryCacheIndividualAssociationData* locAssociationData = nullptr;
1082 locAssociationData = createLocalizedIndividualAssociationData(individualID, associationData, ontologyData, false);
1083 CBackendRepresentativeMemoryCacheContext* context = getIndividualAssociationDataMemoryContext(locAssociationData, ontologyData);
1084 locAssociationData->setCacheUpdateId(mNextIndiUpdateId++);
1085 #ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
1086 locAssociationData->mDebugUpdateFunction = QString("markIndividualAssociationIncompletelyHandled");
1087 locAssociationData->mDebugUpdateRecomputationId = mCurrentUpdateHandlingRecomputationId;
1088 #endif
1089
1090
1091 if (associationData && associationData->getNeighbourRoleSetHash()) {
1092
1093 locAssociationData->setNeighbourRoleSetHash(associationData->getNeighbourRoleSetHash());
1094 locAssociationData->setRoleSetNeighbourArray(associationData->getRoleSetNeighbourArray());
1095 }
1096
1097 locAssociationData->setCompletelyHandled(false);
1098
1099 associationsUpdated = true;
1100
1101 storeIndividualIncompletelyMarked(locAssociationData, !locAssociationData->isCompletelyHandled(), ontologyData);
1102 setUpdatedIndividualAssociationData(individualID, locAssociationData, ontologyData);
1103
1104 }
1105 return associationsUpdated;
1106 }
1107
1108
1109
1110 bool CBackendRepresentativeMemoryCache::markIndividualAssociationCompletelyHandled(cint64 individualID, CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
1111 bool associationsUpdated = false;
1112
1113 if (!associationData || !associationData->isCompletelyHandled()) {
1114
1115 CBackendRepresentativeMemoryCacheIndividualAssociationData* locAssociationData = nullptr;
1116 locAssociationData = createLocalizedIndividualAssociationData(individualID, associationData, ontologyData, false);
1117 CBackendRepresentativeMemoryCacheContext* context = getIndividualAssociationDataMemoryContext(locAssociationData, ontologyData);
1118 locAssociationData->setCacheUpdateId(mNextIndiUpdateId++);
1119 #ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
1120 locAssociationData->mDebugUpdateFunction = QString("markIndividualAssociationCompletelyHandled");
1121 locAssociationData->mDebugUpdateRecomputationId = mCurrentUpdateHandlingRecomputationId;
1122 #endif
1123
1124
1125 if (associationData && associationData->getNeighbourRoleSetHash()) {
1126
1127 locAssociationData->setNeighbourRoleSetHash(associationData->getNeighbourRoleSetHash());
1128 locAssociationData->setRoleSetNeighbourArray(associationData->getRoleSetNeighbourArray());
1129 }
1130
1131 locAssociationData->setCompletelyHandled(true);
1132
1133 associationsUpdated = true;
1134
1135 storeIndividualIncompletelyMarked(locAssociationData, !locAssociationData->isCompletelyHandled(), ontologyData);
1136 setUpdatedIndividualAssociationData(individualID, locAssociationData, ontologyData);
1137
1138 }
1139 return associationsUpdated;
1140 }
1141
1142
1143
1144 bool CBackendRepresentativeMemoryCache::checkUpdateRejection(CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker* tempAssWriteDataLinker, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
1145 bool rejection = false;
1146
1147 cint64 indiUpdateCount = 0;
1148 cint64 incompatibleIndiUpdateCount = 0;
1149 cint64 neighbourLinkCount = 0;
1150 for (CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker* tempAssWriteDataLinkerIt = tempAssWriteDataLinker; tempAssWriteDataLinkerIt && !rejection; tempAssWriteDataLinkerIt = tempAssWriteDataLinkerIt->getNext()) {
1151 CIndividual* individual = tempAssWriteDataLinkerIt->getIndividual();
1152 cint64 individualID = tempAssWriteDataLinkerIt->getIndividualID();
1153 cint64 sameAsIndividualId = tempAssWriteDataLinkerIt->getDeterministicSameIndividualId();
1154 cint64 usedUpdateId = tempAssWriteDataLinkerIt->getUsedAssociationUpdateId();
1155
1156 if (tempAssWriteDataLinkerIt->getDeterministicSameIndividualId() == individualID) {
1157 ++indiUpdateCount;
1158
1159 CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData = nullptr;
1160 if (ontologyData->getIndividualIdAssoiationDataVector() && individualID < ontologyData->getIndividualIdAssoiationDataVectorSize()) {
1161 associationData = ontologyData->getIndividualIdAssoiationDataVector()[individualID];
1162 }
1163
1164 if (associationData && associationData->getAssociationDataUpdateId() != usedUpdateId) {
1165 ++incompatibleIndiUpdateCount;
1166
1167 if (associationData->getLastPropagationCuttingUpdateId() > usedUpdateId) {
1168 neighbourLinkCount += associationData->getNeighbourRoleSetHash()->getNeighbourCount();
1169 cint64 totalIndiCount = ontologyData->getIndividualAssociationsCount();
1170 if (mConfUpdateRejectingIncompatiblePropagationCuttedIndividualLinkedNeighbourCount >= 0 && neighbourLinkCount > mConfUpdateRejectingIncompatiblePropagationCuttedIndividualLinkedNeighbourCount) {
1171 rejection = true;
1172 }
1173 double propCutLinkRatio = (double)neighbourLinkCount / (double)totalIndiCount;
1174 if (propCutLinkRatio > mConfUpdateRejectingIncompatiblePropagationCuttedIndividualLinkedNeighbourRatio) {
1175 rejection = true;
1176 }
1177 }
1178 }
1179
1180 }
1181 }
1182
1183 double incomUpdateRatio = (double)incompatibleIndiUpdateCount / (double)indiUpdateCount;
1184 if (incomUpdateRatio > mConfUpdateRejectingIncompatibleIndividualAssociationsRatio) {
1185 rejection = true;
1186 }
1187
1188 return rejection;
1189 }
1190
1191
1192 bool CBackendRepresentativeMemoryCache::handleUpdateRejection(CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker* tempAssWriteDataLinker, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
1193 for (CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker* tempAssWriteDataLinkerIt = tempAssWriteDataLinker; tempAssWriteDataLinkerIt; tempAssWriteDataLinkerIt = tempAssWriteDataLinkerIt->getNext()) {
1194 CIndividual* individual = tempAssWriteDataLinkerIt->getIndividual();
1195 cint64 individualID = tempAssWriteDataLinkerIt->getIndividualID();
1196
1197 if (tempAssWriteDataLinkerIt->isScheduledIndividual()) {
1198 CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData = nullptr;
1199 if (ontologyData->getIndividualIdAssoiationDataVector() && individualID < ontologyData->getIndividualIdAssoiationDataVectorSize()) {
1200 associationData = ontologyData->getIndividualIdAssoiationDataVector()[individualID];
1201 }
1202 if (associationData) {
1203 associationData->setCacheTouchId(mNextIndiUpdateId++);
1204 }
1205 }
1206
1207 }
1208 return true;
1209 }
1210
1211
1212
1213 bool CBackendRepresentativeMemoryCache::analyseDeterministicSameAsAssociationInstallation(CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker* tempAssWriteDataLinker, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
1214 bool installable = false;
1215 for (CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker* tempAssWriteDataLinkerIt = tempAssWriteDataLinker; tempAssWriteDataLinkerIt; tempAssWriteDataLinkerIt = tempAssWriteDataLinkerIt->getNext()) {
1216 CIndividual* individual = tempAssWriteDataLinkerIt->getIndividual();
1217 cint64 individualID = tempAssWriteDataLinkerIt->getIndividualID();
1218 cint64 sameAsIndividualId = tempAssWriteDataLinkerIt->getDeterministicSameIndividualId();
1219 cint64 usedUpdateId = tempAssWriteDataLinkerIt->getUsedAssociationUpdateId();
1220
1221 if (tempAssWriteDataLinkerIt->getDeterministicSameIndividualId() != individualID) {
1222
1223 CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData = nullptr;
1224 if (ontologyData->getIndividualIdAssoiationDataVector() && individualID < ontologyData->getIndividualIdAssoiationDataVectorSize()) {
1225 associationData = ontologyData->getIndividualIdAssoiationDataVector()[individualID];
1226 }
1227
1228 if (!associationData || (associationData->getAssociationDataUpdateId() == usedUpdateId && !associationData->hasDeterministicSameIndividualMerging())) {
1229
1230 CDeterministicSameHandlingInstallationData& data = mDeterministicSameHandlingInstallationDataHash[sameAsIndividualId];
1231 data.mIdPossibleInstallationSet.insert(individualID);
1232 if (!associationData || !associationData->hasDeterministicSameIndividualMerging()) {
1233 data.mIdFirstPossibleInstallationSet.insert(individualID);
1234 DEBUG_HANDLING_STRING(QString("Adding %1 as first deterministic same handling installation possiblity to %2").arg(individualID).arg(sameAsIndividualId))
1235 } else {
1236 DEBUG_HANDLING_STRING(QString("Adding %1 as deterministic same handling installation possiblity to %2").arg(individualID).arg(sameAsIndividualId))
1237 }
1238 installable = true;
1239 if (associationData) {
1240 CBackendRepresentativeMemoryLabelCacheItem* detSameHandledLabel = associationData->getDeterministicMergedSameConsideredLabelCacheEntry();
1241 if (detSameHandledLabel) {
1242 for (CBackendRepresentativeMemoryLabelValueLinker* valueLinkerIt = detSameHandledLabel->getCacheValueLinker(); valueLinkerIt; valueLinkerIt = valueLinkerIt->getNext()) {
1243 cint64 otherId = valueLinkerIt->getData().getTag();
1244 data.mIdPossibleInstallationSet.insert(otherId);
1245 DEBUG_HANDLING_STRING(QString("Adding %1 referenced from %3 as deterministic same handling installation possiblity to %2").arg(otherId).arg(sameAsIndividualId).arg(individualID))
1246 }
1247 }
1248 }
1249
1250 }
1251 }
1252 }
1253 return installable;
1254 }
1255
1256
1257 bool CBackendRepresentativeMemoryCache::installDeterministicSameAsAssociationUpdates(CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker* tempAssWriteDataLinker, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
1258 CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
1259 bool associationsUpdated = false;
1260 for (CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker* tempAssWriteDataLinkerIt = tempAssWriteDataLinker; tempAssWriteDataLinkerIt; tempAssWriteDataLinkerIt = tempAssWriteDataLinkerIt->getNext()) {
1261 CIndividual* individual = tempAssWriteDataLinkerIt->getIndividual();
1262 cint64 individualID = tempAssWriteDataLinkerIt->getIndividualID();
1263 cint64 sameAsIndividualId = tempAssWriteDataLinkerIt->getDeterministicSameIndividualId();
1264 cint64 usedUpdateId = tempAssWriteDataLinkerIt->getUsedAssociationUpdateId();
1265
1266 if (tempAssWriteDataLinkerIt->getDeterministicSameIndividualId() != individualID) {
1267 ontologyData->setSameIndividualsMergings(true);
1268
1269 CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData = nullptr;
1270 if (ontologyData->getIndividualIdAssoiationDataVector() && individualID < ontologyData->getIndividualIdAssoiationDataVectorSize()) {
1271 associationData = ontologyData->getIndividualIdAssoiationDataVector()[individualID];
1272 }
1273
1274 if (associationData) {
1275 associationData->setCacheTouchId(mNextIndiUpdateId++);
1276 }
1277 DEBUG_HANDLING_STRING(QString("Checking determinitic same reference update for %1").arg(individualID))
1278
1279
1280 if (!associationData || tempAssWriteDataLinkerIt->getDeterministicSameIndividualId() != associationData->getDeterministicSameIndividualId()) {
1281
1282 CBackendRepresentativeMemoryCacheIndividualAssociationData* detSameAsAssociationData = nullptr;
1283
1284 if (ontologyData->getIndividualIdAssoiationDataVector() && sameAsIndividualId < ontologyData->getIndividualIdAssoiationDataVectorSize()) {
1285 detSameAsAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[sameAsIndividualId];
1286 }
1287
1288 //if (detSameAsAssociationData && detSameAsAssociationData->hasDeterministicSameIndividualMerging()) {
1289 // sameAsIndividualId = detSameAsAssociationData->getDeterministicSameIndividualId();
1290 // detSameAsAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[sameAsIndividualId];
1291 //}
1292 if (detSameAsAssociationData) {
1293
1294
1295 CBackendRepresentativeMemoryLabelCacheItem* detSameHandledLabel = detSameAsAssociationData->getDeterministicMergedSameConsideredLabelCacheEntry();
1296
1297
1298 //if (!detSameAsAssociationData->hasRepresentativeSameIndividualMerging() && detSameAsAssociationData->isCompletelyHandled() && (!associationData || associationData->getAssociationDataUpdateId() == usedUpdateId || associationData->hasDeterministicSameIndividualMerging())) {
1299 if (detSameHandledLabel && detSameHandledLabel->hasCachedTagValue(individualID)) {
1300
1301 associationsUpdated = installDeterministicSameAsAssociationUpdate(associationData, individualID, detSameAsAssociationData, sameAsIndividualId, ontologyData);
1302 mStatDetSameAssociationInstallCount++;
1303
1304
1305
1306 } else {
1307 DEBUG_HANDLING_STRING(QString("Determinitic same reference update for %1 failed").arg(individualID))
1308
1309 ++mStatDetSameAssociationFailedCount;
1310 if (detSameAsAssociationData->hasRepresentativeSameIndividualMerging()) {
1311 ++mStatDetSameAssociationRepMergedDestFailedCount;
1312 } else if (!detSameAsAssociationData->isCompletelyHandled()) {
1313 ++mStatDetSameAssociationIncompleteHandledDestFailedCount;
1314 } else {
1315 if (associationData && associationData->getAssociationDataUpdateId() == usedUpdateId) {
1316 ++mStatDetSameAssociationDifferentUpdateIdFailedCount;
1317 }
1318 if (associationData && !associationData->hasDeterministicSameIndividualMerging()) {
1319 ++mStatDetSameAssociationDifferentDestIdFailedCount;
1320 }
1321 }
1322
1323 if (associationData && detSameAsAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL) != associationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL)) {
1324 mTmpDetSameMergingCompletionReferenceHash.insert(individualID, sameAsIndividualId);
1325 } else if (!associationData || !associationData->isCompletelyHandled()) {
1326
1327 markIndividualAssociationCompletelyHandled(individualID, associationData, ontologyData);
1328 }
1329 markRepresentativeReferencedIndividualAssociationIncompletelyHandled(sameAsIndividualId, detSameAsAssociationData, ontologyData);
1330
1331 }
1332 }
1333 } else if (associationData && associationData->getAssociationDataUpdateId() == usedUpdateId && !associationData->isCompletelyHandled()) {
1334 markIndividualAssociationCompletelyHandled(individualID, associationData, ontologyData);
1335 }
1336 }
1337 }
1338 return associationsUpdated;
1339 }
1340
1341
1342
1343 bool CBackendRepresentativeMemoryCache::checkRequiresDeterministicSameAsAssociationUpdateInstallation(CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData, cint64 individualID, CBackendRepresentativeMemoryCacheIndividualAssociationData* detSameAsAssociationData, cint64 sameAsIndividualId, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
1344 if (associationData->getDeterministicSameIndividualId() != sameAsIndividualId) {
1345 return true;
1346 }
1347 if (associationData->getRepresentativeSameIndividualId() != detSameAsAssociationData->getRepresentativeSameIndividualId()) {
1348 return true;
1349 }
1350 if (associationData->getRoleSetNeighbourArray() != detSameAsAssociationData->getRoleSetNeighbourArray()) {
1351 return true;
1352 }
1353 if (!associationData->isCompletelyHandled()) {
1354 return true;
1355 }
1356 for (cint64 i = 0; i < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_ASSOCIATABLE_TYPE_COUNT; ++i) {
1357 if (associationData->getLabelCacheEntry(i) != detSameAsAssociationData->getLabelCacheEntry(i)) {
1358 return true;
1359 }
1360 }
1361
1362 return false;
1363 }
1364
1365
1366 bool CBackendRepresentativeMemoryCache::installDeterministicSameAsAssociationUpdate(CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData, cint64 individualID, CBackendRepresentativeMemoryCacheIndividualAssociationData* detSameAsAssociationData, cint64 sameAsIndividualId, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
1367
1368 DEBUG_HANDLING_STRING(QString("Updating deterministic same handled references for %1 to %2").arg(associationData->getAssociatedIndividualId()).arg(sameAsIndividualId))
1369
1370 CBackendRepresentativeMemoryCacheIndividualAssociationData* locAssociationData = createLocalizedIndividualAssociationData(individualID, associationData, ontologyData, false);
1371 CBackendRepresentativeMemoryCacheContext* context = getIndividualAssociationDataMemoryContext(locAssociationData, ontologyData);
1372 #ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
1373 locAssociationData->mDebugUpdateFunction = QString("installDeterministicSameAsAssociationUpdate");
1374 locAssociationData->mDebugUpdateRecomputationId = mCurrentUpdateHandlingRecomputationId;
1375 #endif
1376 locAssociationData->setCacheUpdateId(mNextIndiUpdateId++);
1377 locAssociationData->setDeterministicSameIndividualId(sameAsIndividualId);
1378 locAssociationData->setRepresentativeSameIndividualId(detSameAsAssociationData->getRepresentativeSameIndividualId());
1379 if (locAssociationData->hasRepresentativeSameIndividualMerging()) {
1380 if (!associationData || !associationData->hasRepresentativeSameIndividualMerging()) {
1381 mStatDetSameRepresentativeMergingCount++;
1382 ontologyData->incIndividualAssociationMergingCount();
1383 }
1384 }
1385 locAssociationData->setRoleSetNeighbourArray(detSameAsAssociationData->getRoleSetNeighbourArray());
1386 locAssociationData->setNeighbourRoleSetHash(detSameAsAssociationData->getNeighbourRoleSetHash());
1387 locAssociationData->setCompletelyHandled(true);
1388 locAssociationData->setDeterministicConceptSetLabelCacheEntry(nullptr);
1389
1390
1391 for (cint64 i = 0; i < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_ASSOCIATABLE_TYPE_COUNT; ++i) {
1392 locAssociationData->setLabelCacheEntry(i, detSameAsAssociationData->getLabelCacheEntry(i));
1393 updateIndexedAssociationCount(locAssociationData, associationData, i, ontologyData);
1394 }
1395
1396 CBackendRepresentativeMemoryLabelCacheItem* mergedSameAsLabelItem = nullptr;
1397 if (associationData->hasDeterministicSameIndividualMerging() && associationData->getDeterministicSameIndividualId() != sameAsIndividualId) {
1398 CBackendRepresentativeMemoryLabelCacheItem* detSameAsLabelItem = detSameAsAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL);
1399 CBackendRepresentativeMemoryLabelCacheItem* prevSameAsLabelItem = associationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL);
1400
1401 mergedSameAsLabelItem = getAdditionMergedLabel(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL, prevSameAsLabelItem, detSameAsLabelItem, ontologyData);
1402 if (mergedSameAsLabelItem != detSameAsLabelItem) {
1403 locAssociationData->setLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL, mergedSameAsLabelItem);
1404 updateIndexedAssociationCount(locAssociationData, detSameAsLabelItem, CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL, ontologyData);
1405 mTmpDetSameMergingCompletionReferenceHash.insert(individualID, sameAsIndividualId);
1406 }
1407 }
1408
1409 storeIndividualIncompletelyMarked(locAssociationData, !locAssociationData->isCompletelyHandled(), ontologyData);
1410 setUpdatedIndividualAssociationData(individualID, locAssociationData, ontologyData);
1411 return true;
1412 }
1413
1414
1415
1416 bool CBackendRepresentativeMemoryCache::installAssociationUpdates(CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker* tempAssWriteDataLinker, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
1417 CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
1418 bool associationsUpdated = false;
1419 for (CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker* tempAssWriteDataLinkerIt = tempAssWriteDataLinker; tempAssWriteDataLinkerIt; tempAssWriteDataLinkerIt = tempAssWriteDataLinkerIt->getNext()) {
1420 cint64 individualID = tempAssWriteDataLinkerIt->getIndividualID();
1421
1422
1423 DEBUG_HANDLING_STRING(QString("Handling update data for individual %1").arg(individualID))
1424
1425
1426 if (tempAssWriteDataLinkerIt->getDeterministicSameIndividualId() == individualID) {
1427
1428 cint64 usedUpdateId = tempAssWriteDataLinkerIt->getUsedAssociationUpdateId();
1429 CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData = nullptr;
1430 if (ontologyData->getIndividualIdAssoiationDataVector() && individualID < ontologyData->getIndividualIdAssoiationDataVectorSize()) {
1431 associationData = ontologyData->getIndividualIdAssoiationDataVector()[individualID];
1432 }
1433 if (associationData && associationData->hasDeterministicSameIndividualMerging()) {
1434 if (!associationData->isCompletelyHandled()) {
1435 markIndividualAssociationCompletelyHandled(individualID, associationData, ontologyData);
1436 }
1437 individualID = associationData->getDeterministicSameIndividualId();
1438 associationData = ontologyData->getIndividualIdAssoiationDataVector()[individualID];
1439 DEBUG_HANDLING_STRING(QString("Resolving update data for individual %1 to %2").arg(tempAssWriteDataLinkerIt->getIndividualID()).arg(individualID))
1440 }
1441
1442 associationsUpdated |= installAssociationUpdate(individualID, associationData, tempAssWriteDataLinkerIt, ontologyData);
1443 }
1444 }
1445 return associationsUpdated;
1446 }
1447
1448
1449
1450 CBackendRepresentativeMemoryCacheIndividualAssociationData* CBackendRepresentativeMemoryCache::createLocalizedIndividualAssociationData(cint64 individualID, CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData, CBackendRepresentativeMemoryCacheOntologyData* ontologyData, bool allowSeparatedManagement, bool incrementUpdateId) {
1451 CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
1452
1453 bool separatedMemoryManagementReusing = false;
1454 bool separatedMemoryManagementNew = false;
1455 CBackendRepresentativeMemoryCacheIndividualAssociationContext* existingIndiAssContext = nullptr;
1456 if (associationData) {
1457 existingIndiAssContext = associationData->getIndividualAssociationMemoryContext();
1458 }
1459 if (associationData && existingIndiAssContext) {
1460 CMemoryPoolContainer* memPoolContainer = existingIndiAssContext->getMemoryPoolContainer();
1461 if (existingIndiAssContext->getIndividualAssociationDataUsageCount() < 50 && memPoolContainer->getMemoryPoolCount() < 10 && memPoolContainer->getMemorySize() <= 10000000) {
1462 separatedMemoryManagementReusing = true;
1463 }
1464 }
1465 if (associationData && existingIndiAssContext && !allowSeparatedManagement) {
1466 separatedMemoryManagementReusing = true;
1467 }
1468 if (!separatedMemoryManagementReusing && associationData && existingIndiAssContext && allowSeparatedManagement) {
1469 separatedMemoryManagementNew = true;
1470 }
1471 if (!separatedMemoryManagementReusing && !separatedMemoryManagementNew && associationData && associationData->getAssociationDataUpdateId() > 10 && ontologyData->isFirstIncompletelyHandledIndividualsRetrieved() && allowSeparatedManagement) {
1472 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* roleSetNeighArray = associationData->getRoleSetNeighbourArray();
1473 cint64 neighbourCount = 0;
1474 if (roleSetNeighArray) {
1475 CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* roleSetNeighArrayInd = roleSetNeighArray->getIndexData();
1476 if (roleSetNeighArrayInd) {
1477 for (cint64 i = 0; i < roleSetNeighArrayInd->getArraySize(); ++i) {
1478 neighbourCount += roleSetNeighArray->at(i).getIndividualCount();
1479 }
1480 }
1481 }
1482 if (neighbourCount > 100) {
1483 separatedMemoryManagementNew = true;
1484 }
1485 }
1486 if (!separatedMemoryManagementReusing && !separatedMemoryManagementNew && associationData && associationData->getAssociationDataUpdateId() > 20 && ontologyData->isFirstIncompletelyHandledIndividualsRetrieved() && !existingIndiAssContext) {
1487 separatedMemoryManagementNew = true;
1488 }
1489
1490
1491 CBackendRepresentativeMemoryCacheIndividualAssociationData* locAssociationData = nullptr;
1492 if (separatedMemoryManagementReusing) {
1493 locAssociationData = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualAssociationData >::allocateAndConstruct(existingIndiAssContext->getMemoryAllocationManager());
1494 locAssociationData->setIndividualAssociationMemoryContext(existingIndiAssContext);
1495 existingIndiAssContext->incIndividualAssociationDataUsageCount();
1496 existingIndiAssContext->setLastRecomputationReferenceLinker(ontologyData->getRecomputationReferenceLinker());
1497 ++mStatIndividualAssociationSeparateMemoryManagmentContextReuseCount;
1498 } else if (separatedMemoryManagementNew) {
1499 CBackendRepresentativeMemoryCacheIndividualAssociationContext* indiAssContext = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualAssociationContext, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
1500 if (mStatCollectStatistics) {
1501 #ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
1502 indiAssContext->mIndiId = individualID;
1503 mIndiContextCountHash[individualID]++;
1504 #endif
1505 }
1506
1507 locAssociationData = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualAssociationData >::allocateAndConstruct(indiAssContext->getMemoryAllocationManager());
1508 locAssociationData->setIndividualAssociationMemoryContext(indiAssContext);
1509 indiAssContext->incIndividualAssociationDataUsageCount();
1510 indiAssContext->setFirstRecomputationReferenceLinker(ontologyData->getLastActiveRecomputationReferenceLinker());
1511 indiAssContext->setLastRecomputationReferenceLinker(ontologyData->getRecomputationReferenceLinker());
1512 ++mStatIndividualAssociationSeparateMemoryManagmentContextCreationCount;
1513
1514 if (associationData && existingIndiAssContext) {
1515 indiAssContext->setPreviousMemoryManagementCount(existingIndiAssContext->getPreviousMemoryManagementCount() + 1);
1516 existingIndiAssContext->setLastRecomputationReferenceLinker(ontologyData->getRecomputationReferenceLinker());
1517 queueIndividualAssociationMemoryContextDeletion(existingIndiAssContext, ontologyData);
1518 }
1519 } else {
1520 locAssociationData = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualAssociationData >::allocateAndConstruct(context->getMemoryAllocationManager());
1521 ++mStatIndividualAssociationWithoutSeparateMemoryManagmentCount;
1522 }
1523
1524 if (associationData) {
1525 locAssociationData->initAssociationData(associationData, incrementUpdateId);
1526 } else {
1527 locAssociationData->initAssociationData(individualID);
1528 }
1529 return locAssociationData;
1530
1531 }
1532
1533 CBackendRepresentativeMemoryCacheContext* CBackendRepresentativeMemoryCache::getIndividualAssociationDataMemoryContext(CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData, CBackendRepresentativeMemoryCacheOntologyData* ontologyData, bool* requiresDataCopying) {
1534 CBackendRepresentativeMemoryCacheIndividualAssociationContext* context = associationData->getIndividualAssociationMemoryContext();
1535 if (context) {
1536 if (requiresDataCopying && context->getIndividualAssociationDataUsageCount() <= 1 && context->getPreviousMemoryManagementCount() > 0) {
1537 *requiresDataCopying = true;
1538 }
1539 return context;
1540 }
1541 return ontologyData->getOntologyContext();
1542 }
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573 bool CBackendRepresentativeMemoryCache::isCacheValueRoleInverse(const CCacheValue& cacheValue) {
1574 CCacheValue::CACHEVALUEIDENTIFIER cacheValueIdentifier = cacheValue.getCacheValueIdentifier();
1575 return cacheValueIdentifier == CCacheValue::CACHEVALTAGANDINVERSEDROLE || cacheValueIdentifier == CCacheValue::CACHEVALTAGANDINVERSEDASSERTEDROLE || cacheValueIdentifier == CCacheValue::CACHEVALTAGANDINVERSED_NOMINAL_CONNECTED_ROLE
1576 || cacheValueIdentifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_INVERSEDROLE || cacheValueIdentifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_INVERSEDASSERTEDROLE || cacheValueIdentifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_INVERSED_NOMINAL_CONNECTED_ROLE;
1577 }
1578
1579
1580 bool CBackendRepresentativeMemoryCache::isCacheValueRoleNondeterministic(const CCacheValue& cacheValue) {
1581 CCacheValue::CACHEVALUEIDENTIFIER cacheValueIdentifier = cacheValue.getCacheValueIdentifier();
1582 return cacheValueIdentifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_ROLE || cacheValueIdentifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_INVERSEDROLE || cacheValueIdentifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_ASSERTEDROLE
1583 || cacheValueIdentifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_INVERSEDASSERTEDROLE || cacheValueIdentifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_NOMINAL_CONNECTED_ROLE || cacheValueIdentifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_INVERSED_NOMINAL_CONNECTED_ROLE;
1584 }
1585
1586
1587
1588 bool CBackendRepresentativeMemoryCache::isRoleNeighbourLinkLabelItemCompatibility(CBackendRepresentativeMemoryLabelCacheItem* itemPrev, CBackendRepresentativeMemoryLabelCacheItem* itemNew) {
1589 if (itemPrev->getCacheValueCount() != itemNew->getCacheValueCount()) {
1590 return false;
1591 }
1592
1593 bool sameRolesCompatible = true;
1594 for (CBackendRepresentativeMemoryLabelValueLinker* linkerIt = itemNew->getCacheValueLinker(); linkerIt && sameRolesCompatible; linkerIt = linkerIt->getNext()) {
1595 cint64 tag = linkerIt->getCacheValue().getTag();
1596 if (!itemPrev->getTagCacheValueHash(false)->contains(tag)) {
1597 sameRolesCompatible = false;
1598 } else {
1599 const CCacheValue& prevCacheValue = itemPrev->getTagCacheValueHash(false)->value(tag)->getCacheValue();
1600 const CCacheValue& newCacheValue = linkerIt->getCacheValue();
1601 if (isCacheValueRoleInverse(prevCacheValue) != isCacheValueRoleInverse(newCacheValue)) {
1602 sameRolesCompatible = false;
1603 }
1604 bool prevNondeterministic = isCacheValueRoleNondeterministic(prevCacheValue);
1605 bool newNondeterministic = isCacheValueRoleNondeterministic(newCacheValue);
1606 if (!prevNondeterministic && newNondeterministic) {
1607 sameRolesCompatible = false;
1608 }
1609 }
1610 }
1611 return sameRolesCompatible;
1612 }
1613
1614
1615
1616
1617
1618
1619
1620 bool CBackendRepresentativeMemoryCache::installAssociationUpdate(cint64 individualID, CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData, CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker* tempAssWriteDataLinkerIt, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
1621
1622 bool associationsUpdated = false;
1623 CIndividual* individual = tempAssWriteDataLinkerIt->getIndividual();
1624
1625 cint64 usedUpdateId = tempAssWriteDataLinkerIt->getUsedAssociationUpdateId();
1626
1627
1628 if (associationData) {
1629 associationData->setCacheTouchId(mNextIndiUpdateId++);
1630 }
1631
1632 bool labelAddition = tempAssWriteDataLinkerIt->getLabelUpdateType() == CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker::ADDITION;
1633 bool labelReplacement = tempAssWriteDataLinkerIt->getLabelUpdateType() == CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker::REPLACEMENT;
1634 bool labelRemoval = tempAssWriteDataLinkerIt->getLabelUpdateType() == CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker::REMOVAL;
1635
1636
1637 bool linksAddition = tempAssWriteDataLinkerIt->getLinksUpdateType() == CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker::ADDITION;
1638 bool linksReplacement = tempAssWriteDataLinkerIt->getLinksUpdateType() == CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker::REPLACEMENT;
1639 bool linksRemoval = tempAssWriteDataLinkerIt->getLinksUpdateType() == CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker::REMOVAL;
1640
1641
1642
1643
1644 CBackendRepresentativeMemoryLabelCacheItem* referredLabels[CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_ASSOCIATABLE_TYPE_COUNT];
1645 for (cint64 i = 0; i < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_ASSOCIATABLE_TYPE_COUNT; ++i) {
1646 CBackendRepresentativeMemoryLabelCacheItem* referredLabelCacheItem = tempAssWriteDataLinkerIt->getReferredLabelData(i);
1647 if (!referredLabelCacheItem) {
1648 CBackendRepresentativeMemoryCacheTemporaryLabelWriteDataLinker* tmpRefferedLabelItem = tempAssWriteDataLinkerIt->getReferredTemporaryLabelData(i);
1649 if (tmpRefferedLabelItem) {
1650 referredLabelCacheItem = (CBackendRepresentativeMemoryLabelCacheItem*)tmpRefferedLabelItem->getTemporaryData();
1651 }
1652 }
1653 referredLabels[i] = referredLabelCacheItem;
1654 }
1655
1656
1657 bool propCut = mPropagationCutIndiSet.contains(individualID);
1658 cint64 integratedIndirectlyConnectedIndividualsChangeId = tempAssWriteDataLinkerIt->getIntegratedIndirectlyConnectedIndividualsChangeId();
1659
1660 bool incompatibleChanges = false;
1661 bool detSameNeighbourCompletion = tempAssWriteDataLinkerIt->requireSameAsNeighboursCompletion();
1662 if (associationData && associationData->getAssociationDataUpdateId() != usedUpdateId || associationData && associationData->getAssociatedIndividualId() != tempAssWriteDataLinkerIt->getIndividualID()) {
1663
1664
1665
1666 bool labelCompatible = false;
1667 if (associationData && mConfInterpretUnchangedLabelsAsCompatible && associationData->isCompletelyHandled()) {
1668 CBackendRepresentativeMemoryCacheIndividualAssociationData* usedReferredAssociationData = associationData->getPreviousData();
1669 while (usedReferredAssociationData && usedReferredAssociationData->getAssociationDataUpdateId() > usedUpdateId) {
1670 usedReferredAssociationData = usedReferredAssociationData->getPreviousData();
1671 }
1672 if (usedReferredAssociationData && usedReferredAssociationData->getAssociationDataUpdateId() != usedUpdateId) {
1673 usedReferredAssociationData = nullptr;
1674 }
1675
1676 if (usedReferredAssociationData) {
1677 labelCompatible = true;
1678 for (cint64 i = 0; associationData && labelCompatible && i < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_ASSOCIATABLE_TYPE_COUNT; ++i) {
1679 if (referredLabels[i] != associationData->getLabelCacheEntry(i)) {
1680 if (i == CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL) {
1681 if (labelReplacement && (linksAddition || propCut)) {
1682 labelAddition = true;
1683 labelReplacement = false;
1684 }
1685 } else if (i == CBackendRepresentativeMemoryLabelCacheItem::NONDETERMINISTIC_COMBINED_NEIGHBOUR_INSTANTIATED_ROLE_SET_LABEL || i == CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_COMBINED_NEIGHBOUR_INSTANTIATED_ROLE_SET_LABEL) {
1686 labelAddition = true;
1687 labelReplacement = false;
1688 } else {
1689 labelCompatible = false;
1690 }
1691
1692 }
1693 }
1694
1695 if (labelCompatible && (linksReplacement && propCut || linksAddition)) {
1696 // only replacing/adding linkers must be checked and the same as for the referred association
1697
1698 CBackendRepresentativeMemoryLabelCacheItem* conSetLabel = associationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::FULL_CONCEPT_SET_LABEL);
1699 CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData* extensionData = (CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData*)conSetLabel->getExtensionData(CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData::CARDINALITY_HASH);
1700 if (extensionData) {
1701 labelCompatible = false;
1702 }
1703 CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash* currentNeighbourRoleSetHash = associationData->getNeighbourRoleSetHash();
1704 CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash* referredNeighbourRoleSetHash = usedReferredAssociationData->getNeighbourRoleSetHash();
1705 for (CBackendRepresentativeMemoryCacheTemporaryIndividualRoleSetNeighbourUpdateDataLinker* roleSetNeighbourUpdateDataLinkerIt = tempAssWriteDataLinkerIt->getRoleSetNeighbourUpdateDataLinker(); labelCompatible && roleSetNeighbourUpdateDataLinkerIt; roleSetNeighbourUpdateDataLinkerIt = roleSetNeighbourUpdateDataLinkerIt->getNext()) {
1706 CIndividualReference neighbourIndiRef = roleSetNeighbourUpdateDataLinkerIt->getNeighbourIndividualReference();
1707 cint64 neighbourId = roleSetNeighbourUpdateDataLinkerIt->getNeighbourIndividualReference().getIndividualID();
1708 CBackendRepresentativeMemoryLabelCacheItem* newNeighbourLabel = roleSetNeighbourUpdateDataLinkerIt->getNeighbourRoleInstantiatedCompinationLabelReference().getReferredLabelData();
1709 if (!newNeighbourLabel) {
1710 CBackendRepresentativeMemoryCacheTemporaryLabelWriteDataLinker* tmpRefferedLabelItem = roleSetNeighbourUpdateDataLinkerIt->getNeighbourRoleInstantiatedCompinationLabelReference().getReferredTemporaryLabelData();
1711 if (tmpRefferedLabelItem) {
1712 newNeighbourLabel = (CBackendRepresentativeMemoryLabelCacheItem*)tmpRefferedLabelItem->getTemporaryData();
1713 }
1714 }
1715 CBackendRepresentativeMemoryLabelCacheItem* referredNeighbourLabel = nullptr;
1716 if (referredNeighbourRoleSetHash) {
1717 referredNeighbourLabel = referredNeighbourRoleSetHash->getNeighbourRoleSetLabel(neighbourId);
1718 }
1719 CBackendRepresentativeMemoryLabelCacheItem* currentNeighbourLabel = nullptr;
1720 if (currentNeighbourRoleSetHash) {
1721 currentNeighbourLabel = currentNeighbourRoleSetHash->getNeighbourRoleSetLabel(neighbourId);
1722 }
1723 if (currentNeighbourLabel && referredNeighbourLabel != currentNeighbourLabel && !isRoleNeighbourLinkLabelItemCompatibility(currentNeighbourLabel, newNeighbourLabel)) {
1724 labelCompatible = false;
1725 } else if (!currentNeighbourLabel && referredNeighbourLabel) {
1726 labelCompatible = false;
1727 }
1728
1729 }
1730
1731 } else {
1732 labelCompatible = false;
1733 }
1734
1735 }
1736 }
1737
1738 if (!labelCompatible) {
1739 labelAddition = true;
1740 labelRemoval = false;
1741 labelReplacement = false;
1742 linksAddition = true;
1743 linksRemoval = false;
1744 linksReplacement = false;
1745 incompatibleChanges = true;
1746 DEBUG_HANDLING_STRING(QString("Installing incompatible update data to %1").arg(individualID))
1747 }
1748 }
1749
1750
1751
1752
1753
1754
1755 bool repLabelUpdateRequired = false;
1756 for (cint64 i = 0; associationData && !repLabelUpdateRequired && i < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_ASSOCIATABLE_TYPE_COUNT; ++i) {
1757 if ((referredLabels[i] || labelReplacement || labelRemoval) && (!associationData || referredLabels[i] != associationData->getLabelCacheEntry(i))) {
1758 DEBUG_HANDLING_STRING(QString("Requiring label %1 update for %2").arg(getRepresentativeCacheLabelItemString(i)).arg(individualID))
1759 repLabelUpdateRequired = true;
1760 }
1761 }
1762
1763
1764 bool statusFlagsUpdateRequired = false;
1765 if ((tempAssWriteDataLinkerIt->getStatusFlags() != 0 || labelReplacement || labelRemoval) && (!associationData || associationData->getStatusFlags() != tempAssWriteDataLinkerIt->getStatusFlags()) || incompatibleChanges) {
1766 statusFlagsUpdateRequired = true;
1767 }
1768
1769
1770
1771 bool linksUpdateRequired = false;
1772 if ((tempAssWriteDataLinkerIt->getRoleSetNeighbourUpdateDataLinker() != nullptr || linksReplacement || linksRemoval)) {
1773 CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash* existingNeighbourRoleSetHash = nullptr;
1774 if (associationData) {
1775 existingNeighbourRoleSetHash = associationData->getNeighbourRoleSetHash();
1776 }
1777 cint64 updateLinksCount = 0;
1778 for (CBackendRepresentativeMemoryCacheTemporaryIndividualRoleSetNeighbourUpdateDataLinker* roleSetNeighbourUpdateDataLinkerIt = tempAssWriteDataLinkerIt->getRoleSetNeighbourUpdateDataLinker(); !linksUpdateRequired && roleSetNeighbourUpdateDataLinkerIt; roleSetNeighbourUpdateDataLinkerIt = roleSetNeighbourUpdateDataLinkerIt->getNext()) {
1779 CIndividualReference neighbourIndiRef = roleSetNeighbourUpdateDataLinkerIt->getNeighbourIndividualReference();
1780 cint64 neighbourId = roleSetNeighbourUpdateDataLinkerIt->getNeighbourIndividualReference().getIndividualID();
1781 CBackendRepresentativeMemoryLabelCacheItem* referredLabelCacheItem = roleSetNeighbourUpdateDataLinkerIt->getNeighbourRoleInstantiatedCompinationLabelReference().getReferredLabelData();
1782 if (!referredLabelCacheItem) {
1783 CBackendRepresentativeMemoryCacheTemporaryLabelWriteDataLinker* tmpRefferedLabelItem = roleSetNeighbourUpdateDataLinkerIt->getNeighbourRoleInstantiatedCompinationLabelReference().getReferredTemporaryLabelData();
1784 if (tmpRefferedLabelItem) {
1785 referredLabelCacheItem = (CBackendRepresentativeMemoryLabelCacheItem*)tmpRefferedLabelItem->getTemporaryData();
1786 }
1787 }
1788 if (linksAddition) {
1789 if (!existingNeighbourRoleSetHash) {
1790 linksUpdateRequired = true;
1791 } else if (existingNeighbourRoleSetHash->getNeighbourRoleSetLabel(neighbourId) != referredLabelCacheItem) {
1792 linksUpdateRequired = true;
1793 }
1794 }
1795 if (linksRemoval) {
1796 if (existingNeighbourRoleSetHash && existingNeighbourRoleSetHash->getNeighbourRoleSetLabel(neighbourId)) {
1797 linksUpdateRequired = true;
1798 }
1799 }
1800 updateLinksCount++;
1801 }
1802 if (!linksUpdateRequired && (updateLinksCount > 0 && !existingNeighbourRoleSetHash || existingNeighbourRoleSetHash && updateLinksCount != existingNeighbourRoleSetHash->getNeighbourCount())) {
1803 linksUpdateRequired = true;
1804 }
1805 }
1806 QSet<cint64> newCompletingDetSameNeighbours;
1807 QSet<cint64> newlyCompletedDetSameNeighbours;
1808 bool completeOnlyCompatibleNeighbourRoleLabels = false;
1809 if (detSameNeighbourCompletion) {
1810 CDeterministicSameHandlingInstallationData& installData = mDeterministicSameHandlingInstallationDataHash[individualID];
1811 CBackendRepresentativeMemoryLabelCacheItem* existDetSameHandlIdLabel = nullptr;
1812 if (associationData) {
1813 existDetSameHandlIdLabel = associationData->getDeterministicMergedSameConsideredLabelCacheEntry();
1814 }
1815 if (!existDetSameHandlIdLabel) {
1816 newCompletingDetSameNeighbours = installData.mIdPossibleInstallationSet;
1817 } else {
1818 CBackendRepresentativeMemoryLabelCacheItem* newDetSameLabel = referredLabels[CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL];
1819 for (CBackendRepresentativeMemoryLabelValueLinker* valueLinkerIt = existDetSameHandlIdLabel->getCacheValueLinker(); valueLinkerIt && detSameNeighbourCompletion; valueLinkerIt = valueLinkerIt->getNext()) {
1820 if (!newDetSameLabel->hasCachedTagValue(valueLinkerIt->getCacheValue().getTag())) {
1821 detSameNeighbourCompletion = false;
1822 DEBUG_HANDLING_STRING(QString("Previous deterministic same handled individual %1 for %2 not newly suggested, installation update not possible").arg(valueLinkerIt->getCacheValue().getTag()).arg(individualID))
1823 }
1824 }
1825
1826 for (cint64 possDetSameInstallId : installData.mIdPossibleInstallationSet) {
1827 if (!existDetSameHandlIdLabel->hasCachedTagValue(possDetSameInstallId)) {
1828 newCompletingDetSameNeighbours.insert(possDetSameInstallId);
1829 }
1830 }
1831 }
1832 if (!newCompletingDetSameNeighbours.isEmpty()) {
1833
1834 cint64 unchangedDetMergedUpdateCount = 0;
1835 CBackendRepresentativeMemoryCacheIndividualAssociationData* prevAssociationData = associationData;
1836 while (prevAssociationData && prevAssociationData->getRepresentativeSameIndividualId() == tempAssWriteDataLinkerIt->getRepresentativeSameIndividualId() && unchangedDetMergedUpdateCount < mConfUnchangedDeterministicSameMergeUpdatesForDeterministicSameNeighbourCompletion) {
1837 ++unchangedDetMergedUpdateCount;
1838 prevAssociationData = prevAssociationData->getPreviousData();
1839 }
1840
1841 if (!incompatibleChanges && newCompletingDetSameNeighbours.size() < mConfMinRequiredDeterministicSameMergedHandledInstallationPossiblitiesForNeighbourCompletion && unchangedDetMergedUpdateCount < mConfUnchangedDeterministicSameMergeUpdatesForDeterministicSameNeighbourCompletion) {
1842 incompatibleChanges = true;
1843 statusFlagsUpdateRequired = true;
1844 detSameNeighbourCompletion = false;
1845 } else {
1846 if (incompatibleChanges && (!mConfInstallingDeterministicSameHandlingLargeDifferenceReached || installData.mIdFirstPossibleInstallationSet.size() < mConfInstallingDeterministicSameHandlingLargeDifference)) {
1847 detSameNeighbourCompletion = false;
1848 } else {
1849 DEBUG_HANDLING_STRING(QString("Scheduling installation of %1 newly deterministic same handled individuals for %2").arg(installData.mIdFirstPossibleInstallationSet.size()).arg(individualID))
1850 linksUpdateRequired = true;
1851 }
1852 if (incompatibleChanges && detSameNeighbourCompletion) {
1853 completeOnlyCompatibleNeighbourRoleLabels = true;
1854 }
1855 }
1856 } else {
1857 detSameNeighbourCompletion = false;
1858 }
1859 }
1860
1861
1862 bool integrationUpdated = false;
1863 if (!associationData || (associationData && (tempAssWriteDataLinkerIt->hasIndirectlyConnectedIndividualIntegration() && !associationData->hasIndirectlyConnectedIndividualIntegration() ||
1864 tempAssWriteDataLinkerIt->isIndirectlyConnectedNominalIndividual() && !associationData->isIndirectlyConnectedNominalIndividual() ||
1865 integratedIndirectlyConnectedIndividualsChangeId > 0 && integratedIndirectlyConnectedIndividualsChangeId != associationData->getLastIntegratedIndirectlyConnectedIndividualsChangeId()))) {
1866 integrationUpdated = true;
1867 }
1868
1869
1870 if (!associationData || repLabelUpdateRequired || statusFlagsUpdateRequired || linksUpdateRequired || integrationUpdated) {
1871
1872 ++mUpdatedIndiCount;
1873 ++mAssociationUpdatedIndiCount;
1874
1875
1876 CBackendRepresentativeMemoryCacheIndividualAssociationData* locAssociationData = nullptr;
1877 locAssociationData = createLocalizedIndividualAssociationData(individualID, associationData, ontologyData, true);
1878 bool requiesDataCopying = false;
1879 CBackendRepresentativeMemoryCacheContext* context = getIndividualAssociationDataMemoryContext(locAssociationData, ontologyData, &requiesDataCopying);
1880
1881
1882 if (propCut) {
1883 locAssociationData->setLastPropagationCuttingUpdateId(locAssociationData->getAssociationDataUpdateId());
1884 }
1885
1886 #ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
1887 locAssociationData->mDebugUpdateFunction = QString("installAssociationUpdate");
1888 if (!locAssociationData->mDebugIndi) {
1889 locAssociationData->mDebugIndi = individual;
1890 }
1891 locAssociationData->mDebugUpdateRecomputationId = mCurrentUpdateHandlingRecomputationId;
1892 locAssociationData->mDebugNeighbourCompletionRequested = tempAssWriteDataLinkerIt->requireSameAsNeighboursCompletion();
1893 #endif
1894 DEBUG_HANDLING_STRING(QString("Updating data for %1").arg(individualID))
1895
1896 locAssociationData->setCacheUpdateId(mNextIndiUpdateId++);
1897 ontologyData->setMaxIndividualAssociationDataUpdateCount(qMax(locAssociationData->getAssociationDataUpdateId(), ontologyData->getMaxIndividualAssociationDataUpdateCount()));
1898
1899 locAssociationData->setRepresentativeSameIndividualId(qMin(tempAssWriteDataLinkerIt->getRepresentativeSameIndividualId(), locAssociationData->getRepresentativeSameIndividualId()));
1900
1901 bool sameIndiMergedChanged = false;
1902 if (locAssociationData->getRepresentativeSameIndividualId() != locAssociationData->getAssociatedIndividualId()) {
1903 ontologyData->setSameIndividualsMergings(true);
1904 if (!associationData || !associationData->hasRepresentativeSameIndividualMerging()) {
1905 mStatDetSameRepresentativeMergingCount++;
1906 ontologyData->incIndividualAssociationMergingCount();
1907 }
1908 }
1909 if (!associationData || locAssociationData->getRepresentativeSameIndividualId() != associationData->getRepresentativeSameIndividualId()) {
1910 sameIndiMergedChanged = true;
1911 }
1912
1913 if (incompatibleChanges && locAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL) && locAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL) != referredLabels[CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL]) {
1914 DEBUG_HANDLING_STRING(QString("Remembering deterministic same mergings required for %1").arg(locAssociationData->getAssociatedIndividualId()))
1915 mTmpDetSameMergingCompletionReferenceHash.insert(locAssociationData->getAssociatedIndividualId(), locAssociationData->getAssociatedIndividualId());
1916 }
1917
1918 bool labelsUpdated = false;
1919 if (!associationData || repLabelUpdateRequired) {
1920
1921 DEBUG_HANDLING_STRING(QString("Updating labels for %1").arg(individualID))
1922 for (cint64 i = 0; i < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_ASSOCIATABLE_TYPE_COUNT; ++i) {
1923 if (referredLabels[i] != locAssociationData->getLabelCacheEntry(i)) {
1924 if (referredLabels[i] || labelReplacement || labelRemoval) {
1925 if (locAssociationData->getLabelCacheEntry(i) && labelAddition) {
1926 CBackendRepresentativeMemoryLabelCacheItem* mergedLabel = getAdditionMergedLabel(i, referredLabels[i], locAssociationData->getLabelCacheEntry(i), ontologyData);
1927 if (mergedLabel != locAssociationData->getLabelCacheEntry(i)) {
1928 DEBUG_HANDLING_STRING(QString("Replacing label %1 for %4 with %5 (merged from %2 and %3)").arg(getRepresentativeCacheLabelItemString(i)).arg(locAssociationData->getLabelCacheEntry(i) ? locAssociationData->getLabelCacheEntry(i)->getCacheEntryID() : 0).arg(referredLabels[i] ? referredLabels[i]->getCacheEntryID() : 0).arg(locAssociationData->getAssociatedIndividualId()).arg(mergedLabel->getCacheEntryID()))
1929 locAssociationData->setLabelCacheEntry(i, mergedLabel);
1930 }
1931 } else {
1932 DEBUG_HANDLING_STRING(QString("Replacing label %1 for %4 with %2 from %3").arg(getRepresentativeCacheLabelItemString(i)).arg(locAssociationData->getLabelCacheEntry(i) ? locAssociationData->getLabelCacheEntry(i)->getCacheEntryID() : 0).arg(referredLabels[i] ? referredLabels[i]->getCacheEntryID() : 0).arg(locAssociationData->getAssociatedIndividualId()))
1933 locAssociationData->setLabelCacheEntry(i, referredLabels[i]);
1934 }
1935 }
1936 }
1937 updateIndexedAssociationCount(locAssociationData, associationData, i, ontologyData);
1938 }
1939 labelsUpdated = true;
1940
1941 associationsUpdated = true;
1942 if (associationData) {
1943 CBackendRepresentativeMemoryLabelCacheItem* indConnNomLabelItem = associationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::INDIRECTLY_CONNECTED_NOMINAL_INDIVIDUAL_SET_LABEL);
1944 if (indConnNomLabelItem) {
1945 mTmpIndiIndirectlyConnNomLabelItemHash.insert(individualID, indConnNomLabelItem);
1946 }
1947 }
1948 } else if (sameIndiMergedChanged) {
1949 for (cint64 i = 0; i < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_ASSOCIATABLE_TYPE_COUNT; ++i) {
1950 bool exactIndiAssocTracking = requiresIndividualAssociations(i);
1951 if (exactIndiAssocTracking && !mConfLateIndividualLabelAssociationIndexing) {
1952 CBackendRepresentativeMemoryLabelCacheItem* labelItem = locAssociationData->getLabelCacheEntry(i);
1953 CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData* indiAssoExtData = getIndividualAssociationsExtensionData(labelItem, ontologyData);
1954 indiAssoExtData->removeIndividualIdAssociation(locAssociationData);
1955 indiAssoExtData->addIndividualIdAssociation(locAssociationData);
1956 }
1957 }
1958 }
1959
1960 if (integrationUpdated) {
1961 if (integratedIndirectlyConnectedIndividualsChangeId > 0) {
1962 locAssociationData->setLastIntegratedIndirectlyConnectedIndividualsChangeId(integratedIndirectlyConnectedIndividualsChangeId);
1963 }
1964 if (tempAssWriteDataLinkerIt->hasIndirectlyConnectedIndividualIntegration()) {
1965 locAssociationData->setIndirectlyConnectedIndividualIntegration(true);
1966 }
1967 if (tempAssWriteDataLinkerIt->isIndirectlyConnectedNominalIndividual()) {
1968 locAssociationData->setIndirectlyConnectedNominalIndividual(true);
1969 }
1970 associationsUpdated = true;
1971 }
1972
1973 if (statusFlagsUpdateRequired) {
1974 locAssociationData->setStatusFlags(tempAssWriteDataLinkerIt->getStatusFlags());
1975 associationsUpdated = true;
1976 }
1977
1978
1979 CBackendRepresentativeMemoryLabelCacheItem* newNeighbourRoleSetCompLabel = locAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL);
1980 CBackendRepresentativeMemoryLabelCacheItem* prevNeighbourRoleSetCompLabel = nullptr;
1981 if (associationData) {
1982 prevNeighbourRoleSetCompLabel = associationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL);
1983 }
1984
1985 if (!linksUpdateRequired && prevNeighbourRoleSetCompLabel && newNeighbourRoleSetCompLabel != prevNeighbourRoleSetCompLabel) {
1986 linksUpdateRequired = true;
1987 associationsUpdated = true;
1988 }
1989
1990
1991 QSet< cint64 >* updateOrRemovalNeighbourIndiIdSet = nullptr;
1992 bool needsNeighCompLabelCleanUp = false;
1993 bool neighbourRoleSetCompLabelReductionRequired = false;
1994 bool neighbourRoleSetCompLabelReductionChecking = false;
1995 bool propCutReadding = false;
1996 QSet<cint64>* propCutReaddingArrayIds = nullptr;
1997 QSet<cint64>* propCutRemovingArrayIds = nullptr;
1998 bool propCutRemovedNeighbourIndiLinkerCopied = false;
1999 QHash<cint64,cint64>* propCutArrayNewPrevReductionIdHash = nullptr;
2000 bool hasProblematicLeveledNeighbours = locAssociationData->hasProblematicLeveledNeigbour();
2001 //QHash< cint64, cint64 >* idProblematicLeveledNeighboursHash = nullptr;
2002 QSet< cint64 >** idProblematicLeveledNeighboursSetArray = nullptr;
2003 // update neighbour data
2004 bool linksUpdated = false;
2005 if (linksUpdateRequired) {
2006 linksUpdated = true;
2007 DEBUG_HANDLING_STRING(QString("Updating links for %1").arg(individualID))
2008 CBackendRepresentativeMemoryLabelCacheItem* propMarkLabelItem = ontologyData->getPrioritizedPropagationMarkedNeighbourLabelItem();
2009 CBackendRepresentativeMemoryCacheTemporaryIndividualRoleSetNeighbourUpdateDataLinker* roleSetNeighbourUpdateDataLinker = tempAssWriteDataLinkerIt->getRoleSetNeighbourUpdateDataLinker();
2010 // create role set neighbour array and neighbour role set hash, if addition, then copy previous data
2011
2012 CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* newArrayIndexData = getIndividualNeighbourArrayIndexExtensionData(newNeighbourRoleSetCompLabel, ontologyData);
2013
2014 CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash* newNeighbourRoleSetHash = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
2015 CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash* prevNeighbourRoleSetHash = nullptr;
2016 if (associationData) {
2017 prevNeighbourRoleSetHash = associationData->getNeighbourRoleSetHash();
2018 }
2019
2020 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* newArray = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
2021 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* prevArray = nullptr;
2022 CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* prevArrayIndexData = nullptr;
2023 if (associationData) {
2024 prevArray = associationData->getRoleSetNeighbourArray();
2025 }
2026 if (prevArray) {
2027 prevArrayIndexData = prevArray->getIndexData();
2028 }
2029
2030 if ((linksRemoval || linksReplacement || tempAssWriteDataLinkerIt->requireSameAsNeighboursCompletion() && !detSameNeighbourCompletion) && prevArray) {
2031 updateOrRemovalNeighbourIndiIdSet = new QSet< cint64 >();
2032 ++mStatUpdatedOrRemovedNeighbourLinksAssociationUpdateCount;
2033 } else {
2034 ++mStatAddingNeighbourLinksAssociationUpdateCount;
2035 }
2036
2037 if (updateOrRemovalNeighbourIndiIdSet && mTmpPropCutIndiArrayNeighboursHandlingDataHash) {
2038 CPropagationCutNeighbourArrayHandlingData* handlingData = mTmpPropCutIndiArrayNeighboursHandlingDataHash->value(individualID);
2039 if (handlingData) {
2040 propCutReadding = true;
2041 if (!handlingData->mReaddingArrayPosSet.isEmpty()) {
2042 propCutReaddingArrayIds = &handlingData->mReaddingArrayPosSet;
2043 }
2044 if (!handlingData->mReductionArrayPosSet.isEmpty()) {
2045
2046 for (cint64 i : handlingData->mReductionArrayPosSet) {
2047 CBackendRepresentativeMemoryLabelCacheItem* prevNeighLabelItem = prevArrayIndexData->getNeighbourRoleSetLabel(i);
2048 CBackendRepresentativeMemoryLabelCacheItem* reducedNeighLabelItem = getReducedLabel(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_LABEL, prevNeighLabelItem, [&](const CCacheValue& cacheValue)->bool {
2049 if (isCacheValueRoleNondeterministic(cacheValue)) {
2050 return true;
2051 } else {
2052 return false;
2053 }
2054 }, ontologyData);
2055
2056 cint64 newIndex = newArrayIndexData->getIndex(reducedNeighLabelItem);
2057 //KONCLUDE_ASSERT_X(newIndex < 0, "BackendCache", "reduction label not available");
2058 if (newIndex >= 0) {
2059 if (!propCutArrayNewPrevReductionIdHash) {
2060 propCutArrayNewPrevReductionIdHash = new QHash<cint64, cint64>();
2061 }
2062 propCutArrayNewPrevReductionIdHash->insertMulti(newIndex, i);
2063 }
2064 }
2065
2066 }
2067 if (!handlingData->mRemovalArrayPosSet.isEmpty()) {
2068 propCutRemovingArrayIds = &handlingData->mRemovalArrayPosSet;
2069 }
2070
2071 }
2072 }
2073
2074
2075 for (CBackendRepresentativeMemoryCacheTemporaryIndividualRoleSetNeighbourUpdateDataLinker* roleSetNeighbourUpdateDataLinkerIt = roleSetNeighbourUpdateDataLinker; roleSetNeighbourUpdateDataLinkerIt; roleSetNeighbourUpdateDataLinkerIt = roleSetNeighbourUpdateDataLinkerIt->getNext()) {
2076 CIndividualReference neighbourIndiRef = roleSetNeighbourUpdateDataLinkerIt->getNeighbourIndividualReference();
2077 CBackendRepresentativeMemoryLabelCacheItem* referredLabelCacheItem = roleSetNeighbourUpdateDataLinkerIt->getNeighbourRoleInstantiatedCompinationLabelReference().getReferredLabelData();
2078 if (!referredLabelCacheItem) {
2079 CBackendRepresentativeMemoryCacheTemporaryLabelWriteDataLinker* tmpRefferedLabelItem = roleSetNeighbourUpdateDataLinkerIt->getNeighbourRoleInstantiatedCompinationLabelReference().getReferredTemporaryLabelData();
2080 if (tmpRefferedLabelItem) {
2081 referredLabelCacheItem = (CBackendRepresentativeMemoryLabelCacheItem*)tmpRefferedLabelItem->getTemporaryData();
2082 }
2083 }
2084 if (referredLabelCacheItem && prevNeighbourRoleSetHash) {
2085 if (!updateOrRemovalNeighbourIndiIdSet && prevNeighbourRoleSetHash->getNeighbourRoleSetLabel(neighbourIndiRef.getIndividualID())) {
2086 updateOrRemovalNeighbourIndiIdSet = new QSet< cint64 >();
2087 ++mStatUpdatedOrRemovedNeighbourLinksAssociationUpdateCount;
2088 --mStatAddingNeighbourLinksAssociationUpdateCount;
2089 } else if (!hasProblematicLeveledNeighbours && ontologyData->hasInvolvedIndividuals()) {
2090 CBackendRepresentativeMemoryCacheIndividualAssociationData* neighbourAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[neighbourIndiRef.getIndividualID()];
2091 if (neighbourAssociationData && neighbourAssociationData->hasProblematicLevel()) {
2092 locAssociationData->setProblematicLeveledNeigbour(true);
2093 hasProblematicLeveledNeighbours = true;
2094 }
2095 }
2096 }
2097 }
2098 cint64 prevPrioPropMarkIndex = -1;
2099 cint64 newPrioPropMarkIndex = -1;
2100 bool readdingPrioPropMarkNeighbours = false;
2101 if (prevNeighbourRoleSetCompLabel && prevArray && propMarkLabelItem && linksAddition && incompatibleChanges) {
2102 prevPrioPropMarkIndex = prevArrayIndexData->getIndex(propMarkLabelItem);
2103 if (prevPrioPropMarkIndex >= 0 && prevArray->at(prevPrioPropMarkIndex).getIndividualCount() > 0) {
2104 readdingPrioPropMarkNeighbours = true;
2105 }
2106 }
2107 if (readdingPrioPropMarkNeighbours) {
2108 newPrioPropMarkIndex = newArrayIndexData->getIndex(propMarkLabelItem);
2109 if (newPrioPropMarkIndex < 0) {
2110 CCacheValue extendingCacheValue;
2111 extendingCacheValue.initCacheValue(propMarkLabelItem->getCacheEntryID(), (cint64)propMarkLabelItem, CCacheValue::CACHE_VALUE_TAG_AND_ENTRY);
2112 CBackendRepresentativeMemoryLabelCacheItem* prevNewNeighbourRoleSetCompLabel = newNeighbourRoleSetCompLabel;
2113 newNeighbourRoleSetCompLabel = getExtendedLabel(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL, prevNewNeighbourRoleSetCompLabel, extendingCacheValue, ontologyData);
2114 locAssociationData->setLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL, newNeighbourRoleSetCompLabel);
2115 updateIndexedAssociationCount(locAssociationData, prevNewNeighbourRoleSetCompLabel, CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL, ontologyData);
2116 newArrayIndexData = getIndividualNeighbourArrayIndexExtensionData(newNeighbourRoleSetCompLabel, ontologyData);
2117 newPrioPropMarkIndex = newArrayIndexData->getIndex(propMarkLabelItem);
2118 }
2119 }
2120
2121
2122 if (hasProblematicLeveledNeighbours) {
2123 //idProblematicLeveledNeighboursHash = new QHash< cint64, cint64 >();
2124 idProblematicLeveledNeighboursSetArray = new QSet<cint64>*[newArrayIndexData->getArraySize()];
2125 for (cint64 i = 0; i < newArrayIndexData->getArraySize(); ++i) {
2126 idProblematicLeveledNeighboursSetArray[i] = new QSet<cint64>();
2127 }
2128 }
2129
2130
2131 if (prevNeighbourRoleSetCompLabel == newNeighbourRoleSetCompLabel && prevArray && linksAddition && !updateOrRemovalNeighbourIndiIdSet) {
2132
2133 newArray->initNeighbourArray(prevArray);
2134 newNeighbourRoleSetHash->initNeighbourRoleSetHash(prevNeighbourRoleSetHash, requiesDataCopying);
2135 if (requiesDataCopying) {
2136 for (cint64 i = 0; i < newArrayIndexData->getArraySize(); ++i) {
2137 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourData& neighbourData = newArray->at(i);
2138 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* firstNewIndiIdLinker = copyNeighbourIndividualIdLinkers(neighbourData.getIndividualIdLinker(), context, locAssociationData, i);
2139 neighbourData.setIndividualIdLinker(firstNewIndiIdLinker, false);
2140 }
2141 }
2142
2143 if (hasProblematicLeveledNeighbours) {
2144 for (cint64 i = 0; i < newArrayIndexData->getArraySize(); ++i) {
2145 if (i != newPrioPropMarkIndex) {
2146 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourData& neighbourData = newArray->at(i);
2147 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiIdLinker = neighbourData.getIndividualIdLinker();
2148 for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiIdLinkerIt = indiIdLinker; indiIdLinkerIt; indiIdLinkerIt = indiIdLinkerIt->getNext()) {
2149 cint64 neighbourId = indiIdLinkerIt->getIndividualId();
2150 CBackendRepresentativeMemoryCacheIndividualAssociationData* neighbourAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[neighbourId];
2151 if (neighbourAssociationData->hasProblematicLevel()) {
2152 //idProblematicLeveledNeighboursHash->insertMulti(i, neighbourId);
2153 idProblematicLeveledNeighboursSetArray[i]->insert(neighbourId);
2154 indiIdLinker = indiIdLinkerIt->getNext();
2155 } else {
2156 break;
2157 }
2158 }
2159 neighbourData.setIndividualIdLinker(indiIdLinker, false);
2160 }
2161 }
2162 }
2163
2164
2165 } else {
2166
2167 newArray->initNeighbourArray(newArrayIndexData);
2168 // copy data from previous array
2169 if (linksAddition && prevArray && !updateOrRemovalNeighbourIndiIdSet) {
2170 CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* prevArrayIndexData = prevArray->getIndexData();
2171 for (cint64 i = 0; i < newArrayIndexData->getArraySize(); ++i) {
2172 cint64 prevIndex = prevArrayIndexData->getIndex(newArrayIndexData->getNeighbourRoleSetLabel(i));
2173 if (prevIndex >= 0 && i != newPrioPropMarkIndex) {
2174 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourData& neighbourData = newArray->at(i);
2175 neighbourData = prevArray->at(prevIndex);
2176 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiIdLinker = neighbourData.getIndividualIdLinker();
2177 if (hasProblematicLeveledNeighbours) {
2178 for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiIdLinkerIt = indiIdLinker; indiIdLinkerIt; indiIdLinkerIt = indiIdLinkerIt->getNext()) {
2179 cint64 neighbourId = indiIdLinkerIt->getIndividualId();
2180 CBackendRepresentativeMemoryCacheIndividualAssociationData* neighbourAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[neighbourId];
2181 if (neighbourAssociationData->hasProblematicLevel()) {
2182 //idProblematicLeveledNeighboursHash->insertMulti(i, neighbourId);
2183 idProblematicLeveledNeighboursSetArray[i]->insert(neighbourId);
2184 indiIdLinker = indiIdLinkerIt->getNext();
2185 } else {
2186 break;
2187 }
2188 }
2189 }
2190 if (requiesDataCopying) {
2191 indiIdLinker = copyNeighbourIndividualIdLinkers(indiIdLinker, context, locAssociationData, i);
2192 }
2193 neighbourData.setIndividualIdLinker(indiIdLinker, false);
2194 }
2195 }
2196 newNeighbourRoleSetHash->initNeighbourRoleSetHash(prevNeighbourRoleSetHash, requiesDataCopying);
2197 } else {
2198 if (!linksReplacement) {
2199 newNeighbourRoleSetHash->initNeighbourRoleSetHash(prevNeighbourRoleSetHash, requiesDataCopying);
2200 } else {
2201 newNeighbourRoleSetHash->initNeighbourRoleSetHash(nullptr);
2202 }
2203 }
2204 }
2205
2206 QHash<cint64,cint64> updatedPrevArrayPosCountHash;
2207 // add/update new data, TODO: first add non-problematic leveled neighbours
2208 for (CBackendRepresentativeMemoryCacheTemporaryIndividualRoleSetNeighbourUpdateDataLinker* roleSetNeighbourUpdateDataLinkerIt = roleSetNeighbourUpdateDataLinker; roleSetNeighbourUpdateDataLinkerIt; roleSetNeighbourUpdateDataLinkerIt = roleSetNeighbourUpdateDataLinkerIt->getNext()) {
2209 CIndividualReference neighbourIndiRef = roleSetNeighbourUpdateDataLinkerIt->getNeighbourIndividualReference();
2210 CBackendRepresentativeMemoryLabelCacheItem* referredLabelCacheItem = roleSetNeighbourUpdateDataLinkerIt->getNeighbourRoleInstantiatedCompinationLabelReference().getReferredLabelData();
2211 if (!referredLabelCacheItem) {
2212 CBackendRepresentativeMemoryCacheTemporaryLabelWriteDataLinker* tmpRefferedLabelItem = roleSetNeighbourUpdateDataLinkerIt->getNeighbourRoleInstantiatedCompinationLabelReference().getReferredTemporaryLabelData();
2213 if (tmpRefferedLabelItem) {
2214 referredLabelCacheItem = (CBackendRepresentativeMemoryLabelCacheItem*)tmpRefferedLabelItem->getTemporaryData();
2215 }
2216 }
2217 if (referredLabelCacheItem) {
2218 cint64 index = newArrayIndexData->getIndex(referredLabelCacheItem);
2219
2220 CBackendRepresentativeMemoryCacheIndividualAssociationData* neighbourAssociationData = nullptr;
2221 if (hasProblematicLeveledNeighbours) {
2222 neighbourAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[neighbourIndiRef.getIndividualID()];
2223 }
2224 if (hasProblematicLeveledNeighbours && neighbourAssociationData && neighbourAssociationData->hasProblematicLevel()) {
2225 cint64 prevIndex = newArrayIndexData->getIndex(newNeighbourRoleSetHash->getNeighbourRoleSetLabel(neighbourIndiRef.getIndividualID()));
2226 if (prevIndex >= 0) {
2227 idProblematicLeveledNeighboursSetArray[prevIndex]->remove(neighbourIndiRef.getIndividualID());
2228 newArray->at(prevIndex).decIndividualCount();
2229 }
2230 //idProblematicLeveledNeighboursHash->insertMulti(index, neighbourIndiRef.getIndividualID());
2231 idProblematicLeveledNeighboursSetArray[index]->insert(neighbourIndiRef.getIndividualID());
2232 newArray->at(index).incIndividualCount();
2233
2234
2235 } else {
2236 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newLinker = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
2237 ++mStatCreatedNeighbourLinks;
2238 newLinker->initIndividualIdLinker(neighbourIndiRef.getIndividualID());
2239 newArray->at(index).addIndividualIdLinker(newLinker);
2240 }
2241 newNeighbourRoleSetHash->setNeighbourRoleSetLabel(neighbourIndiRef.getIndividualID(), referredLabelCacheItem);
2242 if (updateOrRemovalNeighbourIndiIdSet) {
2243 cint64 prevIndex = -1;
2244 if (prevArrayIndexData && prevNeighbourRoleSetHash) {
2245 prevIndex = prevArrayIndexData->getIndex(prevNeighbourRoleSetHash->getNeighbourRoleSetLabel(neighbourIndiRef.getIndividualID()));
2246 if (prevIndex >= 0) {
2247 updatedPrevArrayPosCountHash[prevIndex]++;
2248 }
2249 }
2250 updateOrRemovalNeighbourIndiIdSet->insert(neighbourIndiRef.getIndividualID());
2251 }
2252 }
2253 }
2254
2255 // copy previous data
2256 if (updateOrRemovalNeighbourIndiIdSet && (!linksReplacement || propCutReadding)) {
2257
2258 CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* prevArrayIndexData = prevArray->getIndexData();
2259 for (cint64 i = 0; i < newArrayIndexData->getArraySize(); ++i) {
2260 CBackendRepresentativeMemoryLabelCacheItem* referredLabelCacheItem = newArrayIndexData->getNeighbourRoleSetLabel(i);
2261 cint64 prevIndex = prevArrayIndexData->getIndex(referredLabelCacheItem);
2262 if (prevIndex >= 0 && i != newPrioPropMarkIndex && (!propCutReadding || propCutReaddingArrayIds->contains(prevIndex))) {
2263 bool firstNonProblematicNeighbourIndiFound = false;
2264 cint64 remainingPrevUpdatedRemovedCount = updatedPrevArrayPosCountHash.value(prevIndex);
2265 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourData& prevNeighbourData = prevArray->at(prevIndex);
2266 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourData& newNeighbourData = newArray->at(i);
2267 cint64 remainingNeighbourCount = prevNeighbourData.getIndividualCount();
2268 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* addingNeighbourLinker = newNeighbourData.getIndividualIdLinker();
2269 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* lastAddingNeighbourLinker = nullptr;
2270 if (newNeighbourData.getIndividualIdLinker()) {
2271 lastAddingNeighbourLinker = newNeighbourData.getIndividualIdLinker()->getLastListLink();
2272 }
2273 cint64 addingNeighbourCount = 0;
2274 for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* neighbourLinkerIt = prevNeighbourData.getIndividualIdLinker(); neighbourLinkerIt; neighbourLinkerIt = neighbourLinkerIt->getNext()) {
2275 cint64 neighbourIndiId = neighbourLinkerIt->getIndividualId();
2276
2277 if (!updateOrRemovalNeighbourIndiIdSet->contains(neighbourIndiId)) {
2278
2279 newNeighbourRoleSetHash->setNeighbourRoleSetLabel(neighbourIndiId, referredLabelCacheItem);
2280 CBackendRepresentativeMemoryCacheIndividualAssociationData* neighbourAssociationData = nullptr;
2281 if (!firstNonProblematicNeighbourIndiFound && hasProblematicLeveledNeighbours) {
2282 neighbourAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[neighbourIndiId];
2283 }
2284 if (!firstNonProblematicNeighbourIndiFound && hasProblematicLeveledNeighbours && neighbourAssociationData && neighbourAssociationData->hasProblematicLevel()) {
2285 idProblematicLeveledNeighboursSetArray[i]->insert(neighbourIndiId);
2286 //idProblematicLeveledNeighboursHash->insertMulti(i, neighbourIndiId);
2287 addingNeighbourCount++;
2288 --remainingNeighbourCount;
2289 } else {
2290 firstNonProblematicNeighbourIndiFound = true;
2291
2292 if (remainingPrevUpdatedRemovedCount <= 0) {
2293 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* readdingNeighbourLinker = neighbourLinkerIt;
2294 if (propCutReadding) {
2295 for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* readdingNeighbourLinkerIt = readdingNeighbourLinker; readdingNeighbourLinkerIt; readdingNeighbourLinkerIt = readdingNeighbourLinkerIt->getNext()) {
2296 cint64 neighbourIndiId = readdingNeighbourLinkerIt->getIndividualId();
2297 newNeighbourRoleSetHash->setNeighbourRoleSetLabel(neighbourIndiId, referredLabelCacheItem);
2298 }
2299 }
2300 if (requiesDataCopying) {
2301 readdingNeighbourLinker = copyNeighbourIndividualIdLinkers(readdingNeighbourLinker, context, locAssociationData, i);
2302 }
2303 if (!addingNeighbourLinker) {
2304 addingNeighbourLinker = readdingNeighbourLinker;
2305 } else {
2306 lastAddingNeighbourLinker->setNext(readdingNeighbourLinker);
2307 }
2308 addingNeighbourCount += remainingNeighbourCount;
2309 remainingNeighbourCount = 0;
2310 break;
2311 } else {
2312 --remainingNeighbourCount;
2313 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newLinker = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
2314 ++mStatCreatedNeighbourLinks;
2315 newLinker->initIndividualIdLinker(neighbourIndiId);
2316 addingNeighbourCount++;
2317 if (!addingNeighbourLinker) {
2318 lastAddingNeighbourLinker = addingNeighbourLinker = newLinker;
2319 } else {
2320 lastAddingNeighbourLinker->setNext(newLinker);
2321 lastAddingNeighbourLinker = newLinker;
2322 }
2323 }
2324 }
2325 } else {
2326 remainingPrevUpdatedRemovedCount--;
2327 --remainingNeighbourCount;
2328 }
2329
2330 //prevArray->at(prevIndex).visitNeighbourIndividualIds([&](cint64 neighbourIndiId)->bool {
2331 // if (!updateOrRemovalNeighbourIndiIdSet->contains(neighbourIndiId)) {
2332
2333 // newNeighbourRoleSetHash->setNeighbourRoleSetLabel(neighbourIndiId, referredLabelCacheItem);
2334 // CBackendRepresentativeMemoryCacheIndividualAssociationData* neighbourAssociationData = nullptr;
2335 // if (!firstNonProblematicNeighbourIndiFound && hasProblematicLeveledNeighbours) {
2336 // neighbourAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[neighbourIndiId];
2337 // }
2338 // if (!firstNonProblematicNeighbourIndiFound && hasProblematicLeveledNeighbours && neighbourAssociationData && neighbourAssociationData->hasProblematicLevel()) {
2339 // idProblematicLeveledNeighboursSetArray[i]->insert(neighbourIndiId);
2340 // //idProblematicLeveledNeighboursHash->insertMulti(i, neighbourIndiId);
2341 // newArray->at(i).incIndividualCount();
2342 // } else {
2343 // firstNonProblematicNeighbourIndiFound = true;
2344 // CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newLinker = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
2345 // ++mStatCreatedNeighbourLinks;
2346 // newLinker->initIndividualIdLinker(neighbourIndiId);
2347 // newArray->at(i).addIndividualIdLinker(newLinker);
2348 // }
2349 // }
2350 // return true;
2351 //});
2352 }
2353 newNeighbourData.setIndividualIdLinker(addingNeighbourLinker, false);
2354 newNeighbourData.incIndividualCount(addingNeighbourCount);
2355
2356 }
2357
2358 }
2359
2360 if (propCutArrayNewPrevReductionIdHash) {
2361
2362 for (QHash<cint64, cint64>::const_iterator it = propCutArrayNewPrevReductionIdHash->constBegin(), itEnd = propCutArrayNewPrevReductionIdHash->constEnd(); it != itEnd; ++it) {
2363 cint64 newArrayPos = it.key();
2364 cint64 prevIndex = it.value();
2365 if (prevIndex >= 0 && newArrayPos >= 0) {
2366 CBackendRepresentativeMemoryLabelCacheItem* referredLabelCacheItem = newArrayIndexData->getNeighbourRoleSetLabel(newArrayPos);
2367 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourData& newNeighbourData = newArray->at(newArrayPos);
2368 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourData& prevNeighbourData = prevArray->at(prevIndex);
2369 if (newNeighbourData.getIndividualCount() <= 0) {
2370 newNeighbourData = prevNeighbourData;
2371 } else {
2372 bool firstNonProblematicNeighbourIndiFound = false;
2373 for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* neighbourLinkerIt = prevNeighbourData.getIndividualIdLinker(); neighbourLinkerIt; neighbourLinkerIt = neighbourLinkerIt->getNext()) {
2374 cint64 neighbourIndiId = neighbourLinkerIt->getIndividualId();
2375 if (!updateOrRemovalNeighbourIndiIdSet->contains(neighbourIndiId)) {
2376 newNeighbourRoleSetHash->setNeighbourRoleSetLabel(neighbourIndiId, referredLabelCacheItem);
2377 CBackendRepresentativeMemoryCacheIndividualAssociationData* neighbourAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[neighbourIndiId];
2378
2379 if (!firstNonProblematicNeighbourIndiFound && hasProblematicLeveledNeighbours && neighbourAssociationData && neighbourAssociationData->hasProblematicLevel()) {
2380 idProblematicLeveledNeighboursSetArray[newArrayPos]->insert(neighbourIndiId);
2381 newNeighbourData.incIndividualCount();
2382 } else {
2383 firstNonProblematicNeighbourIndiFound = true;
2384 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newLinker = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
2385 ++mStatCreatedNeighbourLinks;
2386 newLinker->initIndividualIdLinker(neighbourIndiId);
2387 newNeighbourData.addIndividualIdLinker(newLinker);
2388 }
2389 }
2390 }
2391
2392 }
2393 }
2394 }
2395 }
2396
2397
2398 //if (propCutReaddingArrayIds) {
2399 // delete propCutReaddingArrayIds;
2400 //}
2401 if (propCutArrayNewPrevReductionIdHash) {
2402 delete propCutArrayNewPrevReductionIdHash;
2403 }
2404 delete updateOrRemovalNeighbourIndiIdSet;
2405
2406 if (propCutRemovingArrayIds) {
2407 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* prevRemNeighIndiLinker = locAssociationData->getPropagationCutRemovedNeighbourIndividualLinker();
2408 if (requiesDataCopying) {
2409 propCutRemovedNeighbourIndiLinkerCopied = true;
2410 prevRemNeighIndiLinker = copyNeighbourIndividualIdLinkers(prevRemNeighIndiLinker, context, locAssociationData, -1);
2411 }
2412 QSet<cint64> prevRemNeighIndiSet;
2413
2414 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newRemNeighIndiLinker = nullptr;
2415 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newLastRemNeighIndiLinker = nullptr;
2416 for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* neighbourLinkerIt = prevRemNeighIndiLinker; neighbourLinkerIt; ) {
2417 cint64 neighbourIndiId = neighbourLinkerIt->getIndividualId();
2418 CBackendRepresentativeMemoryLabelCacheItem* neighbourLabelItem = newNeighbourRoleSetHash->getNeighbourRoleSetLabel(neighbourIndiId);
2419 if (!neighbourLabelItem) {
2420 prevRemNeighIndiSet.insert(neighbourIndiId);
2421 if (!newLastRemNeighIndiLinker) {
2422 newLastRemNeighIndiLinker = newRemNeighIndiLinker = neighbourLinkerIt;
2423 } else {
2424 newLastRemNeighIndiLinker->setNext(neighbourLinkerIt);
2425 newLastRemNeighIndiLinker = neighbourLinkerIt;
2426 }
2427 neighbourLinkerIt = neighbourLinkerIt->getNext();
2428 if (newLastRemNeighIndiLinker) {
2429 newLastRemNeighIndiLinker->setNext(nullptr);
2430 }
2431 } else {
2432 neighbourLinkerIt = neighbourLinkerIt->getNext();
2433 }
2434 }
2435
2436
2437 for (cint64 prevIndex : *propCutRemovingArrayIds) {
2438 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourData& prevNeighbourData = prevArray->at(prevIndex);
2439
2440 for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* neighbourLinkerIt = prevNeighbourData.getIndividualIdLinker(); neighbourLinkerIt; neighbourLinkerIt = neighbourLinkerIt->getNext()) {
2441 cint64 neighbourIndiId = neighbourLinkerIt->getIndividualId();
2442 CBackendRepresentativeMemoryLabelCacheItem* neighbourLabelItem = newNeighbourRoleSetHash->getNeighbourRoleSetLabel(neighbourIndiId);
2443 if (!prevRemNeighIndiSet.contains(neighbourIndiId) && !neighbourLabelItem) {
2444 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newLinker = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
2445 ++mStatCreatedNeighbourLinks;
2446 newLinker->initIndividualIdLinker(neighbourIndiId);
2447 newRemNeighIndiLinker = newLinker->append(newRemNeighIndiLinker);
2448 }
2449 }
2450 }
2451 locAssociationData->setPropagationCutRemovedNeighbourIndividualLinker(newRemNeighIndiLinker);
2452 }
2453
2454
2455 if (readdingPrioPropMarkNeighbours) {
2456 newArray->at(newPrioPropMarkIndex) = prevArray->at(prevPrioPropMarkIndex);
2457 if (requiesDataCopying) {
2458 newArray->at(newPrioPropMarkIndex).setIndividualIdLinker(copyNeighbourIndividualIdLinkers(newArray->at(newPrioPropMarkIndex).getIndividualIdLinker(), context, locAssociationData, newPrioPropMarkIndex), false);
2459 }
2460 }
2461
2462
2463 if (detSameNeighbourCompletion) {
2464 locAssociationData->setNeighbourRoleSetHash(newNeighbourRoleSetHash);
2465 locAssociationData->setRoleSetNeighbourArray(newArray);
2466 if (completeSameAsNeighbours(locAssociationData, newCompletingDetSameNeighbours, newlyCompletedDetSameNeighbours, completeOnlyCompatibleNeighbourRoleLabels, ontologyData)) {
2467 mTmpCompleteNeighbourSameIndiMergingSet.insert(locAssociationData->getAssociatedIndividualId());
2468 }
2469 }
2470
2471 neighbourRoleSetCompLabelReductionChecking = true;
2472
2473
2474 } else {
2475 if (detSameNeighbourCompletion) {
2476 locAssociationData->setNeighbourRoleSetHash(newNeighbourRoleSetHash);
2477 locAssociationData->setRoleSetNeighbourArray(newArray);
2478 if (completeSameAsNeighbours(locAssociationData, newCompletingDetSameNeighbours, newlyCompletedDetSameNeighbours, completeOnlyCompatibleNeighbourRoleLabels, ontologyData)) {
2479 mTmpCompleteNeighbourSameIndiMergingSet.insert(locAssociationData->getAssociatedIndividualId());
2480 }
2481 }
2482 if (incompatibleChanges || detSameNeighbourCompletion) {
2483 neighbourRoleSetCompLabelReductionChecking = true;
2484 }
2485
2486 }
2487
2488
2489
2490
2491 if (hasProblematicLeveledNeighbours) {
2492 //for (QHash< cint64, cint64 >::const_iterator idProbNeighIt = idProblematicLeveledNeighboursHash->constBegin(), idProbNeighItEnd = idProblematicLeveledNeighboursHash->constEnd(); idProbNeighIt != idProbNeighItEnd; ++idProbNeighIt) {
2493 // cint64 idx = idProbNeighIt.key();
2494 // cint64 neighbourId = idProbNeighIt.value();
2495
2496 for (cint64 idx = 0; idx < newArrayIndexData->getArraySize(); ++idx) {
2497 for (cint64 neighbourId : *(idProblematicLeveledNeighboursSetArray[idx])) {
2498 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourData& neighbourData = newArray->at(idx);
2499
2500 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newLinker = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
2501 ++mStatCreatedNeighbourLinks;
2502 newLinker->initIndividualIdLinker(neighbourId);
2503 neighbourData.addIndividualIdLinker(newLinker, false);
2504 }
2505 delete (idProblematicLeveledNeighboursSetArray[idx]);
2506 }
2507 //delete idProblematicLeveledNeighboursHash;
2508 delete[] idProblematicLeveledNeighboursSetArray;
2509 }
2510
2511
2512 if (neighbourRoleSetCompLabelReductionChecking) {
2513 for (cint64 i = 0; i < newArrayIndexData->getArraySize(); ++i) {
2514 if (!newArray->at(i).getIndividualIdLinker()) {
2515 //if (!newArray->at(i).getIndividualIdLinker() && (!idProblematicLeveledNeighboursHash || !idProblematicLeveledNeighboursHash->contains(i))) {
2516 neighbourRoleSetCompLabelReductionRequired = true;
2517 }
2518 }
2519 if (neighbourRoleSetCompLabelReductionRequired) {
2520 ++mReducedNeighbourArrayCount;
2521 CBackendRepresentativeMemoryLabelCacheItem* neighbourRoleSetCombinationLabelSet = locAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL);
2522
2523 CBackendRepresentativeMemoryLabelCacheItem* reducedNeighbourRoleSetCombinationLabelSet = getReducedLabel(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL, neighbourRoleSetCombinationLabelSet, [&](const CCacheValue& cacheValue)->bool {
2524 CBackendRepresentativeMemoryLabelCacheItem* neighbourRoleSetLabel = (CBackendRepresentativeMemoryLabelCacheItem*)cacheValue.getIdentification();
2525 cint64 index = newArrayIndexData->getIndex(neighbourRoleSetLabel);
2526 if (index < 0) {
2527 return true;
2528 }
2529 //if (!newArray->at(index).getIndividualIdLinker() && (!idProblematicLeveledNeighboursHash || !idProblematicLeveledNeighboursHash->contains(index))) {
2530 if (!newArray->at(index).getIndividualIdLinker()) {
2531 return true;
2532 }
2533 return false;
2534 }, ontologyData);
2535
2536 if (neighbourRoleSetCombinationLabelSet != reducedNeighbourRoleSetCombinationLabelSet) {
2537 locAssociationData->setLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL, reducedNeighbourRoleSetCombinationLabelSet);
2538 updateIndexedAssociationCount(locAssociationData, neighbourRoleSetCombinationLabelSet, CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL, ontologyData);
2539
2540
2541 CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* reducedArrayIndexData = getIndividualNeighbourArrayIndexExtensionData(reducedNeighbourRoleSetCombinationLabelSet, ontologyData);
2542 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* reducedArray = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
2543 reducedArray->initNeighbourArray(reducedArrayIndexData);
2544 // copy data into reduced array
2545 for (cint64 i = 0; i < reducedArrayIndexData->getArraySize(); ++i) {
2546 cint64 prevIndex = newArrayIndexData->getIndex(reducedArrayIndexData->getNeighbourRoleSetLabel(i));
2547 if (prevIndex >= 0) {
2548 reducedArray->at(i) = newArray->at(prevIndex);
2549 }
2550 }
2551 newArray = reducedArray;
2552 newArrayIndexData = reducedArrayIndexData;
2553
2554 }
2555
2556 }
2557 }
2558
2559
2560
2561 locAssociationData->setNeighbourRoleSetHash(newNeighbourRoleSetHash);
2562 locAssociationData->setRoleSetNeighbourArray(newArray);
2563 associationsUpdated = true;
2564
2565
2566 } else if (associationData && associationData->getNeighbourRoleSetHash()) {
2567
2568 if (detSameNeighbourCompletion) {
2569 linksUpdated = true;
2570 CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash* newNeighbourRoleSetHash = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
2571 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* newArray = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
2572 newArray->initNeighbourArray(associationData->getRoleSetNeighbourArray());
2573 newNeighbourRoleSetHash->initNeighbourRoleSetHash(associationData->getNeighbourRoleSetHash(), requiesDataCopying);
2574 if (requiesDataCopying) {
2575 CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* arrayIndexData = newArray->getIndexData();
2576 for (cint64 i = 0; i < arrayIndexData->getArraySize(); ++i) {
2577 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourData& neighbourData = newArray->at(i);
2578 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* firstNewIndiIdLinker = copyNeighbourIndividualIdLinkers(neighbourData.getIndividualIdLinker(), context, locAssociationData, i);
2579 neighbourData.setIndividualIdLinker(firstNewIndiIdLinker, false);
2580 }
2581 }
2582
2583 locAssociationData->setNeighbourRoleSetHash(newNeighbourRoleSetHash);
2584 locAssociationData->setRoleSetNeighbourArray(newArray);
2585 bool completed = completeSameAsNeighbours(locAssociationData, newCompletingDetSameNeighbours, newlyCompletedDetSameNeighbours, completeOnlyCompatibleNeighbourRoleLabels, ontologyData);
2586 if (completed || requiesDataCopying) {
2587 associationsUpdated = true;
2588 mTmpCompleteNeighbourSameIndiMergingSet.insert(locAssociationData->getAssociatedIndividualId());
2589 } else {
2590 locAssociationData->setNeighbourRoleSetHash(associationData->getNeighbourRoleSetHash());
2591 locAssociationData->setRoleSetNeighbourArray(associationData->getRoleSetNeighbourArray());
2592 }
2593
2594
2595 } else {
2596
2597 if (requiesDataCopying) {
2598 CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash* newNeighbourRoleSetHash = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
2599 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* newArray = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
2600 newArray->initNeighbourArray(associationData->getRoleSetNeighbourArray());
2601 newNeighbourRoleSetHash->initNeighbourRoleSetHash(associationData->getNeighbourRoleSetHash(), requiesDataCopying);
2602 CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* arrayIndexData = newArray->getIndexData();
2603 for (cint64 i = 0; i < arrayIndexData->getArraySize(); ++i) {
2604 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourData& neighbourData = newArray->at(i);
2605 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* firstNewIndiIdLinker = copyNeighbourIndividualIdLinkers(neighbourData.getIndividualIdLinker(), context, locAssociationData, i);
2606 neighbourData.setIndividualIdLinker(firstNewIndiIdLinker, false);
2607 }
2608 locAssociationData->setNeighbourRoleSetHash(newNeighbourRoleSetHash);
2609 locAssociationData->setRoleSetNeighbourArray(newArray);
2610
2611 } else {
2612
2613 locAssociationData->setNeighbourRoleSetHash(associationData->getNeighbourRoleSetHash());
2614 locAssociationData->setRoleSetNeighbourArray(associationData->getRoleSetNeighbourArray());
2615 }
2616
2617 }
2618 }
2619
2620
2621
2622 if (requiesDataCopying && !propCutRemovedNeighbourIndiLinkerCopied) {
2623 propCutRemovedNeighbourIndiLinkerCopied = true;
2624 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* prevRemNeighIndiLinker = locAssociationData->getPropagationCutRemovedNeighbourIndividualLinker();
2625 prevRemNeighIndiLinker = copyNeighbourIndividualIdLinkers(prevRemNeighIndiLinker, context, locAssociationData, -1);
2626 locAssociationData->setPropagationCutRemovedNeighbourIndividualLinker(prevRemNeighIndiLinker);
2627 }
2628
2629
2630 CBackendRepresentativeMemoryLabelCacheItem* detSameLabItem = locAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL);
2631 if (detSameLabItem) {
2632 mStatMaxSameAsMergedCount = qMax(mStatMaxSameAsMergedCount, detSameLabItem->getCacheValueCount());
2633 }
2634 if (locAssociationData->getNeighbourRoleSetHash()) {
2635 mStatMaxNeighbourLinksCount = qMax(mStatMaxNeighbourLinksCount, locAssociationData->getNeighbourRoleSetHash()->getNeighbourCount());
2636 }
2637 mStatMaxAssociationUpdateCount = qMax(mStatMaxAssociationUpdateCount, locAssociationData->getAssociationDataUpdateId());
2638
2639
2640 if (tempAssWriteDataLinkerIt->hasIndirectlyConnectedIndividualIntegration()) {
2641 CBackendRepresentativeMemoryCacheNominalIndividualIndirectConnectionData*& nomConnData = (*ontologyData->getNominaIIndividualdIndirectConnectionDataHash())[individualID];
2642 if (nomConnData || integratedIndirectlyConnectedIndividualsChangeId > 0) {
2643 if (integratedIndirectlyConnectedIndividualsChangeId != nomConnData->getLastChangeId()) {
2644 locAssociationData->setCompletelyHandled(false);
2645 associationsUpdated = true;
2646 }
2647 }
2648 }
2649
2650
2651 if (incompatibleChanges && locAssociationData->isCompletelyHandled() && (labelsUpdated || linksUpdated || tempAssWriteDataLinkerIt->requireSameAsNeighboursCompletion())) {
2652 // TODO: may optimize by checking whether something has actually been updated, e.g., labels, links, etc., but requires neighbours completion then
2653 mCheckIncompatibleIndiCount++;
2654 if (locAssociationData->hasRepresentativeSameIndividualMerging()) {
2655 markRepresentativeReferencedIndividualAssociationIncompletelyHandled(individualID, locAssociationData, ontologyData);
2656 } else {
2657 locAssociationData->setCompletelyHandled(false);
2658 associationsUpdated = true;
2659 }
2660 }
2661
2662 if (locAssociationData->isCompletelyPropagated() && !tempAssWriteDataLinkerIt->isCompletelyPropagated()) {
2663 locAssociationData->setCompletelyPropagated(false);
2664 associationsUpdated = true;
2665 }
2666
2667
2668 if (!locAssociationData->isCompletelyHandled() && locAssociationData->hasRepresentativeSameIndividualMerging()) {
2669 locAssociationData->setCompletelyHandled(true);
2670 markRepresentativeReferencedIndividualAssociationIncompletelyHandled(individualID, locAssociationData, ontologyData);
2671 associationsUpdated = true;
2672 }
2673
2674 if (detSameNeighbourCompletion) {
2675 if (referredLabels[CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL] != locAssociationData->getDeterministicMergedSameConsideredLabelCacheEntry()) {
2676
2677 CBackendRepresentativeMemoryLabelCacheItem* prevDetSameLabel = locAssociationData->getDeterministicMergedSameConsideredLabelCacheEntry();
2678 CBackendRepresentativeMemoryLabelCacheItem* newDetSameLabel = referredLabels[CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL];
2679 CBackendRepresentativeMemoryLabelCacheItem* reducedNewDetSameLabel = getReducedLabel(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL, newDetSameLabel, [&](const CCacheValue& cacheValue)-> bool {
2680 cint64 detSameHandledId = cacheValue.getTag();
2681 if (detSameHandledId == individualID || prevDetSameLabel && prevDetSameLabel->hasCachedTagValue(detSameHandledId) || newlyCompletedDetSameNeighbours.contains(detSameHandledId)) {
2682 return false;
2683 } else {
2684 return true;
2685 }
2686 }, ontologyData);
2687 locAssociationData->setDeterministicMergedSameConsideredLabelCacheEntry(reducedNewDetSameLabel);
2688 cint64 prevDetSameLabelCacheEntryId = -1;
2689 if (prevDetSameLabel) {
2690 prevDetSameLabelCacheEntryId = prevDetSameLabel->getCacheEntryID();
2691 }
2692 DEBUG_HANDLING_STRING(QString("Updating deterministically same handled label for %1 to label %2 from label %3").arg(locAssociationData->getAssociatedIndividualId()).arg(locAssociationData->getDeterministicMergedSameConsideredLabelCacheEntry()->getCacheEntryID()).arg(prevDetSameLabelCacheEntryId))
2693 associationsUpdated = true;
2694 }
2695 }
2696
2697
2698
2699 associationsUpdated |= storeIndividualIncompletelyMarked(locAssociationData, !locAssociationData->isCompletelyHandled(), ontologyData);
2700
2701
2702
2703 //if (mIncompletelyHandledIndiIdCount != mIncompletelyAssociatedIndividualSet.size()) {
2704 // bool debug = true;
2705 //}
2706 //if (locAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL) && locAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL)->getCacheValueCount() > 0) {
2707 // if (!locAssociationData->getRoleSetNeighbourArray()) {
2708 // bool debug = true;
2709 // }
2710 //}
2711
2712
2713 if (associationsUpdated) {
2714 if (locAssociationData->getPreviousData() && !locAssociationData->getPreviousData()->getPreviousData()) {
2715 ontologyData->incIndividualAssociationDataDirectUpdateCount();
2716 }
2717 setUpdatedIndividualAssociationData(individualID, locAssociationData, ontologyData);
2718 if (!locAssociationData->hasRepresentativeSameIndividualMerging()) {
2719 udateDeterministicSameAssociations(locAssociationData, ontologyData);
2720 }
2721 }
2722
2723 DEBUG_HANDLING_STRING(QString("Finishing updating data for %1").arg(locAssociationData->getAssociatedIndividualId()))
2724
2725 }
2726 return associationsUpdated;
2727 }
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763 bool CBackendRepresentativeMemoryCache::updateIndexedAssociationCount(CBackendRepresentativeMemoryCacheIndividualAssociationData* locAssociationData, CBackendRepresentativeMemoryLabelCacheItem* prevLabelItem, cint64 i, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
2764 bool updated = false;
2765 bool exactIndiAssocTracking = requiresIndividualAssociations(i);
2766
2767 CBackendRepresentativeMemoryLabelCacheItem* newLabelItem = locAssociationData->getLabelCacheEntry(i);
2768
2769 if (prevLabelItem != newLabelItem) {
2770 if (prevLabelItem) {
2771 updated = true;
2772 prevLabelItem->decIndividualAssociationCount();
2773 if (exactIndiAssocTracking && !mConfLateIndividualLabelAssociationIndexing) {
2774 CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData* indiAssoExtData = getIndividualAssociationsExtensionData(prevLabelItem, ontologyData);
2775 indiAssoExtData->removeIndividualIdAssociation(locAssociationData);
2776 }
2777 }
2778 if (newLabelItem) {
2779 updated = true;
2780 newLabelItem->incIndividualAssociationCount();
2781 if (exactIndiAssocTracking && !mConfLateIndividualLabelAssociationIndexing) {
2782 CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData* indiAssoExtData = getIndividualAssociationsExtensionData(newLabelItem, ontologyData);
2783 indiAssoExtData->addIndividualIdAssociation(locAssociationData);
2784 }
2785 }
2786 }
2787 return true;
2788 }
2789
2790
2791 bool CBackendRepresentativeMemoryCache::updateIndexedAssociationCount(CBackendRepresentativeMemoryCacheIndividualAssociationData* locAssociationData, CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData, cint64 i, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
2792 bool updated = false;
2793
2794
2795 bool exactIndiAssocTracking = requiresIndividualAssociations(i);
2796 bool sameIndiMergedChanged = false;
2797 if (!associationData || associationData->getRepresentativeSameIndividualId() != locAssociationData->getRepresentativeSameIndividualId()) {
2798 sameIndiMergedChanged = true;
2799 }
2800
2801 CBackendRepresentativeMemoryLabelCacheItem* prevLabelItem = nullptr;
2802 CBackendRepresentativeMemoryLabelCacheItem* newLabelItem = locAssociationData->getLabelCacheEntry(i);
2803 if (associationData) {
2804 prevLabelItem = associationData->getLabelCacheEntry(i);
2805 }
2806
2807 if (prevLabelItem != newLabelItem || sameIndiMergedChanged) {
2808 if (prevLabelItem) {
2809 updated = true;
2810 prevLabelItem->decIndividualAssociationCount();
2811 if (exactIndiAssocTracking && !mConfLateIndividualLabelAssociationIndexing) {
2812 CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData* indiAssoExtData = getIndividualAssociationsExtensionData(prevLabelItem, ontologyData);
2813 indiAssoExtData->removeIndividualIdAssociation(associationData);
2814 }
2815 }
2816 if (newLabelItem) {
2817 updated = true;
2818 newLabelItem->incIndividualAssociationCount();
2819 if (exactIndiAssocTracking && !mConfLateIndividualLabelAssociationIndexing) {
2820 CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData* indiAssoExtData = getIndividualAssociationsExtensionData(newLabelItem, ontologyData);
2821 indiAssoExtData->addIndividualIdAssociation(locAssociationData);
2822 }
2823 }
2824 }
2825 return true;
2826 }
2827
2828
2829 bool CBackendRepresentativeMemoryCache::udateDeterministicSameAssociations(CBackendRepresentativeMemoryCacheIndividualAssociationData* locNeighbourAssociationData, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
2830 bool updated = false;
2831 DEBUG_HANDLING_STRING(QString("Updating deterministic same references for %1").arg(locNeighbourAssociationData->getAssociatedIndividualId()))
2832 CBackendRepresentativeMemoryLabelCacheItem* detSameLabItem = locNeighbourAssociationData->getDeterministicMergedSameConsideredLabelCacheEntry();
2833 if (detSameLabItem) {
2834 for (CBackendRepresentativeMemoryLabelValueLinker* detSameValueLinkerIt = detSameLabItem->getCacheValueLinker(); detSameValueLinkerIt; detSameValueLinkerIt = detSameValueLinkerIt->getNext()) {
2835 cint64 detSameNeighbourId = detSameValueLinkerIt->getCacheValue().getTag();
2836
2837 if (detSameNeighbourId != locNeighbourAssociationData->getAssociatedIndividualId()) {
2838 CBackendRepresentativeMemoryCacheIndividualAssociationData* detSameNeighbourAssociationData = nullptr;
2839 if (ontologyData->getIndividualIdAssoiationDataVector() && detSameNeighbourId < ontologyData->getIndividualIdAssoiationDataVectorSize()) {
2840 detSameNeighbourAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[detSameNeighbourId];
2841 }
2842
2843 if (detSameNeighbourAssociationData && detSameNeighbourAssociationData->getDeterministicSameIndividualId() == locNeighbourAssociationData->getAssociatedIndividualId()) {
2844 if (checkRequiresDeterministicSameAsAssociationUpdateInstallation(detSameNeighbourAssociationData, detSameNeighbourId, locNeighbourAssociationData, locNeighbourAssociationData->getAssociatedIndividualId(), ontologyData)) {
2845 updated |= installDeterministicSameAsAssociationUpdate(detSameNeighbourAssociationData, detSameNeighbourId, locNeighbourAssociationData, locNeighbourAssociationData->getAssociatedIndividualId(), ontologyData);
2846 }
2847 }
2848 }
2849 }
2850 }
2851 return updated;
2852 }
2853
2854
2855 bool CBackendRepresentativeMemoryCache::completeSameAsNeighbours(CBackendRepresentativeMemoryCacheIndividualAssociationData* locAssociationData, QSet<cint64> newCompletingDetSameNeighbours, QSet<cint64>& newlyCompletedDetSameNeighbours, bool completeOnlyCompatibleNeighbourRoleLabels, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
2856 bool completed = false;
2857 DEBUG_HANDLING_STRING(QString("Completing links for %1").arg(locAssociationData->getAssociatedIndividualId()))
2858 cint64 linksCompleted = 0;
2859 CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
2860 CBackendRepresentativeMemoryLabelCacheItem* detSameLabItem = locAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL);
2861 CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash* neighbourRoleSetLabelItemHash = locAssociationData->getNeighbourRoleSetHash();
2862 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* neighbourArray = locAssociationData->getRoleSetNeighbourArray();
2863 CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* arrayIndexData = neighbourArray->getIndexData();
2864 if (detSameLabItem) {
2865 #ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
2866 locAssociationData->mDebugNeighbourCompletionPerformed = true;
2867 #endif
2868 for (CBackendRepresentativeMemoryLabelValueLinker* detSameValueLinkerIt = detSameLabItem->getCacheValueLinker(); detSameValueLinkerIt; detSameValueLinkerIt = detSameValueLinkerIt->getNext()) {
2869 cint64 detSameId = detSameValueLinkerIt->getCacheValue().getTag();
2870 if (detSameId != locAssociationData->getAssociatedIndividualId() && newCompletingDetSameNeighbours.contains(detSameId)) {
2871 CBackendRepresentativeMemoryCacheIndividualAssociationData* detSameAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[detSameId];
2872 if (!detSameAssociationData->hasDeterministicSameIndividualMerging()) {
2873 DEBUG_HANDLING_STRING(QString("Trying links completion for %1 with %2").arg(locAssociationData->getAssociatedIndividualId()).arg(detSameId))
2874 bool completedDetSameId = true;
2875 bool oneChangedDetSameIdNeighbourLabelCompletion = false;
2876 bool oneIncompatibleDetSameIdNeighbourLabelCompletion = false;
2877 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* detSameNeighbourArray = detSameAssociationData->getRoleSetNeighbourArray();
2878 if (detSameNeighbourArray) {
2879 CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* detSameArrayIndexData = detSameNeighbourArray->getIndexData();
2880 for (cint64 i = 0; i < detSameArrayIndexData->getArraySize() && completedDetSameId; ++i) {
2881 CBackendRepresentativeMemoryLabelCacheItem* detSameNeighbourRoleSetLabelItem = detSameArrayIndexData->getNeighbourRoleSetLabel(i);
2882 cint64 arrayIndex = arrayIndexData->getIndex(detSameNeighbourRoleSetLabelItem);
2883 if (arrayIndex >= 0) {
2884 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourData& neighbourData = neighbourArray->at(arrayIndex);
2885 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* neighbourIndiLinker = detSameNeighbourArray->at(i).getIndividualIdLinker();
2886 for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* neighbourIndiLinkerIt = neighbourIndiLinker; neighbourIndiLinkerIt && completedDetSameId; neighbourIndiLinkerIt = neighbourIndiLinkerIt->getNext()) {
2887 cint64 neighbourIndiId = neighbourIndiLinkerIt->getIndividualId();
2888 CBackendRepresentativeMemoryLabelCacheItem* neighbourRoleSetLabelItem = neighbourRoleSetLabelItemHash->getNeighbourRoleSetLabel(neighbourIndiId);
2889 if (!neighbourRoleSetLabelItem) {
2890 neighbourRoleSetLabelItemHash->setNeighbourRoleSetLabel(neighbourIndiId, detSameNeighbourRoleSetLabelItem);
2891 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newLinker = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
2892 ++mStatCreatedNeighbourLinks;
2893 ++linksCompleted;
2894 newLinker->initIndividualIdLinker(neighbourIndiId);
2895 neighbourData.addIndividualIdLinker(newLinker, true);
2896 } else if (neighbourRoleSetLabelItem != detSameNeighbourRoleSetLabelItem) {
2897 oneChangedDetSameIdNeighbourLabelCompletion = true;
2898 ++mStatChangedLabelNeighbourCompletionCount;
2899 if (completeOnlyCompatibleNeighbourRoleLabels) {
2900 completedDetSameId = false;
2901 ++mStatIncompatibleLabelNeighbourCompletionCount;
2902 oneIncompatibleDetSameIdNeighbourLabelCompletion = true;
2903 }
2904 }
2905 }
2906 }
2907 }
2908 }
2909 if (completedDetSameId) {
2910 DEBUG_HANDLING_STRING(QString("Links completed for %1 with %2").arg(locAssociationData->getAssociatedIndividualId()).arg(detSameId))
2911 newlyCompletedDetSameNeighbours.insert(detSameId);
2912 completed = true;
2913 ++mStatNeighbourCompletionDetSameSuccededCount;
2914 }
2915 if (oneChangedDetSameIdNeighbourLabelCompletion) {
2916 ++mStatNeighbourCompletionDetSameChangedCount;
2917 }
2918 if (oneIncompatibleDetSameIdNeighbourLabelCompletion) {
2919 ++mStatNeighbourCompletionDetSameIncompatibleCount;
2920 }
2921 if (!oneChangedDetSameIdNeighbourLabelCompletion) {
2922 ++mStatNeighbourCompletionDetSameUnchangedCount;
2923 }
2924
2925 } else {
2926 newlyCompletedDetSameNeighbours.insert(detSameId);
2927 }
2928 }
2929 }
2930 }
2931 DEBUG_HANDLING_STRING(QString("Completed %2 links for %1").arg(locAssociationData->getAssociatedIndividualId()).arg(linksCompleted))
2932 return completed;
2933 }
2934
2935
2936
2937
2938 bool CBackendRepresentativeMemoryCache::completeDeterministicSameAsMergingInformation(QHash<cint64, cint64>& tmpDetSameMergingCompletionReferenceHash, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
2939 bool updated = false;
2940 cint64 completedDetSameLabelItems = 0;
2941 cint64 completedDetSameIntegratedIndividuals = 0;
2942 cint64 completedIndividualUpdates = 0;
2943
2944 DEBUG_HANDLING_STRING(QString("Starting deterministic same mergings completions for %1 entries").arg(tmpDetSameMergingCompletionReferenceHash.size()))
2945
2946 QSet<cint64> mergedHandledIndiIdSet;
2947 for (QHash<cint64, cint64>::const_iterator it = tmpDetSameMergingCompletionReferenceHash.constBegin(), itEnd = tmpDetSameMergingCompletionReferenceHash.constEnd(); it != itEnd; ++it) {
2948 cint64 refIndiId = it.key();
2949 cint64 destIndiId = it.value();
2950 DEBUG_HANDLING_STRING(QString("Completing deterministic same mergings for %1 to %2").arg(refIndiId).arg(destIndiId))
2951 if (!mergedHandledIndiIdSet.contains(destIndiId)) {
2952 mergedHandledIndiIdSet.insert(destIndiId);
2953
2954 ++completedDetSameLabelItems;
2955
2956 CBackendRepresentativeMemoryCacheIndividualAssociationData* refIndiAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[destIndiId];
2957 CBackendRepresentativeMemoryLabelCacheItem* detSameLabItem = refIndiAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL);
2958 if (detSameLabItem) {
2959 QSet<CBackendRepresentativeMemoryLabelCacheItem*> mergingDetSameLabelItemSet;
2960 QList<CBackendRepresentativeMemoryLabelCacheItem*> processingDetSameLabelItemList;
2961 processingDetSameLabelItemList.append(detSameLabItem);
2962 mergingDetSameLabelItemSet.insert(detSameLabItem);
2963
2964 cint64 minRepIndiId = destIndiId;
2965
2966 while (!processingDetSameLabelItemList.isEmpty()) {
2967 CBackendRepresentativeMemoryLabelCacheItem* procDetSameLabItem = processingDetSameLabelItemList.takeFirst();
2968
2969 for (CBackendRepresentativeMemoryLabelValueLinker* detSameValueLinkerIt = procDetSameLabItem->getCacheValueLinker(); detSameValueLinkerIt; detSameValueLinkerIt = detSameValueLinkerIt->getNext()) {
2970 cint64 detSameId = detSameValueLinkerIt->getCacheValue().getTag();
2971 mergedHandledIndiIdSet.insert(detSameId);
2972 minRepIndiId = qMin(detSameId, minRepIndiId);
2973 ++completedDetSameIntegratedIndividuals;
2974
2975 CBackendRepresentativeMemoryCacheIndividualAssociationData* sameIndiAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[detSameId];
2976 CBackendRepresentativeMemoryLabelCacheItem* otherDetSameLabItem = sameIndiAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL);
2977 if (otherDetSameLabItem && otherDetSameLabItem != procDetSameLabItem && !mergingDetSameLabelItemSet.contains(otherDetSameLabItem)) {
2978 mergingDetSameLabelItemSet.insert(otherDetSameLabItem);
2979 processingDetSameLabelItemList.append(otherDetSameLabItem);
2980 }
2981
2982
2983 cint64 otherDestIndiId = tmpDetSameMergingCompletionReferenceHash.value(detSameId);
2984 CBackendRepresentativeMemoryCacheIndividualAssociationData* otherDestIndiAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[otherDestIndiId];
2985 CBackendRepresentativeMemoryLabelCacheItem* otherDestDetSameLabItem = otherDestIndiAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL);
2986 if (otherDestDetSameLabItem && otherDestDetSameLabItem != procDetSameLabItem && !mergingDetSameLabelItemSet.contains(otherDestDetSameLabItem)) {
2987 mergingDetSameLabelItemSet.insert(otherDestDetSameLabItem);
2988 processingDetSameLabelItemList.append(otherDestDetSameLabItem);
2989 }
2990 }
2991 }
2992
2993
2994
2995 CBackendRepresentativeMemoryLabelCacheItem* mergedDetSameLabItem = detSameLabItem;
2996 for (CBackendRepresentativeMemoryLabelCacheItem* mergingLabelItem : mergingDetSameLabelItemSet) {
2997 if (mergingLabelItem != detSameLabItem) {
2998 mergedDetSameLabItem = getAdditionMergedLabel(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL, mergingLabelItem, mergedDetSameLabItem, ontologyData);
2999 }
3000 }
3001
3002 if (mergedDetSameLabItem) {
3003 DEBUG_HANDLING_STRING(QString("Found new deterministic same mergings for %1 to %2 stored in label item %3").arg(refIndiId).arg(destIndiId).arg(mergedDetSameLabItem->getCacheEntryID()))
3004 }
3005
3006 for (CBackendRepresentativeMemoryLabelValueLinker* detSameValueLinkerIt = mergedDetSameLabItem->getCacheValueLinker(); detSameValueLinkerIt; detSameValueLinkerIt = detSameValueLinkerIt->getNext()) {
3007 cint64 detSameId = detSameValueLinkerIt->getCacheValue().getTag();
3008 CBackendRepresentativeMemoryCacheIndividualAssociationData* sameIndiAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[detSameId];
3009 if (sameIndiAssociationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL) != mergedDetSameLabItem || sameIndiAssociationData->getRepresentativeSameIndividualId() != minRepIndiId || sameIndiAssociationData->getAssociatedIndividualId() == minRepIndiId && sameIndiAssociationData->isCompletelyHandled()) {
3010
3011 bool incrementUpdateIdForDeterministicSameAsCompletion = mConfIncrementUpdateIdForDeterministicSameAsCompletion;
3012 if (sameIndiAssociationData->getAssociatedIndividualId() == minRepIndiId) {
3013 incrementUpdateIdForDeterministicSameAsCompletion = true;
3014 }
3015
3016 CBackendRepresentativeMemoryCacheIndividualAssociationData* locSameIndiAssociationData = createLocalizedIndividualAssociationData(sameIndiAssociationData->getAssociatedIndividualId(), sameIndiAssociationData, ontologyData, false, incrementUpdateIdForDeterministicSameAsCompletion);
3017 CBackendRepresentativeMemoryCacheContext* context = getIndividualAssociationDataMemoryContext(locSameIndiAssociationData, ontologyData);
3018
3019 locSameIndiAssociationData->setCacheUpdateId(mNextIndiUpdateId++);
3020 locSameIndiAssociationData->setNeighbourRoleSetHash(sameIndiAssociationData->getNeighbourRoleSetHash());
3021 locSameIndiAssociationData->setRoleSetNeighbourArray(sameIndiAssociationData->getRoleSetNeighbourArray());
3022
3023 #ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
3024 locSameIndiAssociationData->mDebugUpdateFunction = QString("completeDeterministicSameAsMergingInformation");
3025 locSameIndiAssociationData->mDebugSameIndividualsCompletion = true;
3026 locSameIndiAssociationData->mDebugUpdateRecomputationId = mCurrentUpdateHandlingRecomputationId;
3027 #endif
3028 DEBUG_HANDLING_STRING(QString("Updating new deterministic same mergings for %1").arg(detSameId))
3029
3030
3031 locSameIndiAssociationData->setRepresentativeSameIndividualId(minRepIndiId);
3032 if (locSameIndiAssociationData->hasRepresentativeSameIndividualMerging()) {
3033 if (!sameIndiAssociationData || !sameIndiAssociationData->hasRepresentativeSameIndividualMerging()) {
3034 mStatDetSameRepresentativeMergingCount++;
3035 ontologyData->incIndividualAssociationMergingCount();
3036 }
3037 }
3038
3039 //locSameIndiAssociationData->setDeterministicSameIndividualId(minRepIndiId);
3040 locSameIndiAssociationData->setLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL, mergedDetSameLabItem);
3041 updateIndexedAssociationCount(locSameIndiAssociationData, sameIndiAssociationData, CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL, ontologyData);
3042 if (detSameId != minRepIndiId) {
3043 locSameIndiAssociationData->setCompletelyHandled(true);
3044 } else {
3045 locSameIndiAssociationData->setCompletelyHandled(false);
3046 }
3047
3048 ++mUpdatedIndiCount;
3049 ++completedIndividualUpdates;
3050 ontologyData->setMaxIndividualAssociationDataUpdateCount(qMax(locSameIndiAssociationData->getAssociationDataUpdateId(), ontologyData->getMaxIndividualAssociationDataUpdateCount()));
3051
3052 storeIndividualIncompletelyMarked(locSameIndiAssociationData, !locSameIndiAssociationData->isCompletelyHandled(), ontologyData);
3053 setUpdatedIndividualAssociationData(locSameIndiAssociationData->getAssociatedIndividualId(), locSameIndiAssociationData, ontologyData);
3054
3055 updated = true;
3056 }
3057 }
3058
3059
3060
3061 }
3062
3063 }
3064 }
3065 LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Completed %1 deterministic same individual labels over %2 individuals with %3 updated association data.").arg(completedDetSameLabelItems).arg(completedDetSameIntegratedIndividuals).arg(completedIndividualUpdates), this);
3066
3067 return updated;
3068 }
3069
3070
3071
3072 bool CBackendRepresentativeMemoryCache::completeNeighboursForSameAsMerging(QSet<cint64>& tmpCompleteNeighbourSameIndiMergingSet, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
3073 CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
3074 cint64 checkingNeighbours = 0;
3075 cint64 completedNeighbours = 0;
3076 cint64 completedNeighbourLinks = 0;
3077 bool updated = false;
3078 QHash<cint64, CBackendRepresentativeMemoryCacheIndividualAssociationData*> tmpLocalIndiAssocDataHash;
3079 for (cint64 mergeIndiId : mTmpCompleteNeighbourSameIndiMergingSet) {
3080 CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData = ontologyData->getIndividualIdAssoiationDataVector()[mergeIndiId];
3081 CBackendRepresentativeMemoryLabelCacheItem* detSameLabItem = associationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL);
3082 if (detSameLabItem) {
3083 CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash* neighbourRoleSetLabelItemHash = associationData->getNeighbourRoleSetHash();
3084 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* neighbourArray = associationData->getRoleSetNeighbourArray();
3085 CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* arrayIndexData = neighbourArray->getIndexData();
3086
3087 for (cint64 i = 0; i < arrayIndexData->getArraySize(); ++i) {
3088 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* neighbourIndiLinker = neighbourArray->at(i).getIndividualIdLinker();
3089 for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* neighbourIndiLinkerIt = neighbourIndiLinker; neighbourIndiLinkerIt; neighbourIndiLinkerIt = neighbourIndiLinkerIt->getNext()) {
3090 cint64 neighbourIndiId = neighbourIndiLinkerIt->getIndividualId();
3091 ++checkingNeighbours;
3092
3093 CBackendRepresentativeMemoryCacheIndividualAssociationData* neighbourAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[neighbourIndiId];
3094
3095 if (!neighbourAssociationData->hasDeterministicSameIndividualMerging()) {
3096
3097 CBackendRepresentativeMemoryCacheIndividualAssociationData* locNeighbourAssociationData = nullptr;
3098 CBackendRepresentativeMemoryLabelCacheItem* sameNeighbourRoleSetLabelItem = nullptr;
3099 cint64 sameNeighbourRoleSetLabelArrayId = -1;
3100 for (CBackendRepresentativeMemoryLabelValueLinker* detSameValueLinkerIt = detSameLabItem->getCacheValueLinker(); detSameValueLinkerIt && !sameNeighbourRoleSetLabelItem; detSameValueLinkerIt = detSameValueLinkerIt->getNext()) {
3101 cint64 detSameId = detSameValueLinkerIt->getCacheValue().getTag();
3102
3103 if (neighbourAssociationData->getNeighbourRoleSetHash()) {
3104 CBackendRepresentativeMemoryLabelCacheItem* neighbourRoleSetLabelItem = neighbourAssociationData->getNeighbourRoleSetHash()->getNeighbourRoleSetLabel(detSameId);
3105 if (neighbourRoleSetLabelItem) {
3106 sameNeighbourRoleSetLabelItem = neighbourRoleSetLabelItem;
3107 sameNeighbourRoleSetLabelArrayId = neighbourAssociationData->getRoleSetNeighbourArray()->getIndexData()->getIndex(sameNeighbourRoleSetLabelItem);
3108 }
3109 }
3110 }
3111
3112 if (sameNeighbourRoleSetLabelItem && sameNeighbourRoleSetLabelArrayId >= 0) {
3113 for (CBackendRepresentativeMemoryLabelValueLinker* detSameValueLinkerIt = detSameLabItem->getCacheValueLinker(); detSameValueLinkerIt; detSameValueLinkerIt = detSameValueLinkerIt->getNext()) {
3114 cint64 detSameId = detSameValueLinkerIt->getCacheValue().getTag();
3115
3116 CBackendRepresentativeMemoryLabelCacheItem* neighbourRoleSetLabelItem = neighbourAssociationData->getNeighbourRoleSetHash()->getNeighbourRoleSetLabel(detSameId);
3117 if (!neighbourRoleSetLabelItem) {
3118
3119 if (!locNeighbourAssociationData) {
3120 CBackendRepresentativeMemoryCacheIndividualAssociationData*& tmpLlocNeighbourAssociationData = tmpLocalIndiAssocDataHash[neighbourIndiId];
3121 if (!tmpLlocNeighbourAssociationData) {
3122 tmpLlocNeighbourAssociationData = createLocalizedIndividualAssociationData(neighbourAssociationData->getAssociatedIndividualId(), neighbourAssociationData, ontologyData, false);
3123 bool requiresDataCopying = false;
3124 CBackendRepresentativeMemoryCacheContext* context = getIndividualAssociationDataMemoryContext(tmpLlocNeighbourAssociationData, ontologyData, &requiresDataCopying);
3125 #ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
3126 tmpLlocNeighbourAssociationData->mDebugUpdateFunction = QString("completeNeighboursForSameAsMerging");
3127 tmpLlocNeighbourAssociationData->mDebugNeighbourCompletionExtended = true;
3128 tmpLlocNeighbourAssociationData->mDebugUpdateRecomputationId = mCurrentUpdateHandlingRecomputationId;
3129 #endif
3130
3131
3132 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* newArray = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
3133 CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash* newNeighbourRoleSetHash = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
3134 newNeighbourRoleSetHash->initNeighbourRoleSetHash(neighbourAssociationData->getNeighbourRoleSetHash(), requiresDataCopying);
3135 newArray->initNeighbourArray(neighbourAssociationData->getRoleSetNeighbourArray());
3136 tmpLlocNeighbourAssociationData->setNeighbourRoleSetHash(newNeighbourRoleSetHash);
3137 tmpLlocNeighbourAssociationData->setRoleSetNeighbourArray(newArray);
3138 tmpLlocNeighbourAssociationData->setCacheUpdateId(mNextIndiUpdateId++);
3139 ontologyData->setMaxIndividualAssociationDataUpdateCount(qMax(tmpLlocNeighbourAssociationData->getAssociationDataUpdateId(), ontologyData->getMaxIndividualAssociationDataUpdateCount()));
3140 ++mUpdatedIndiCount;
3141 completedNeighbours++;
3142 }
3143 locNeighbourAssociationData = tmpLlocNeighbourAssociationData;
3144 }
3145
3146 CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash* locNeighbourRoleSetLabelItemHash = locNeighbourAssociationData->getNeighbourRoleSetHash();
3147 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* locArray = locNeighbourAssociationData->getRoleSetNeighbourArray();
3148 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newLinker = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
3149 ++mStatCreatedNeighbourLinks;
3150 newLinker->initIndividualIdLinker(detSameId);
3151 locArray->at(sameNeighbourRoleSetLabelArrayId).addIndividualIdLinker(newLinker);
3152 locNeighbourRoleSetLabelItemHash->setNeighbourRoleSetLabel(detSameId, sameNeighbourRoleSetLabelItem);
3153 completedNeighbourLinks++;
3154
3155 }
3156 }
3157 }
3158 }
3159 }
3160
3161 }
3162 }
3163 }
3164 for (QHash<cint64, CBackendRepresentativeMemoryCacheIndividualAssociationData*>::const_iterator it = tmpLocalIndiAssocDataHash.constBegin(), itEnd = tmpLocalIndiAssocDataHash.constEnd(); it != itEnd; ++it) {
3165 CBackendRepresentativeMemoryCacheIndividualAssociationData* locNeighbourAssociationData = it.value();
3166 storeIndividualIncompletelyMarked(locNeighbourAssociationData, !locNeighbourAssociationData->isCompletelyHandled(), ontologyData);
3167 setUpdatedIndividualAssociationData(locNeighbourAssociationData->getAssociatedIndividualId(), locNeighbourAssociationData, ontologyData);
3168
3169 updated |= udateDeterministicSameAssociations(locNeighbourAssociationData, ontologyData);
3170 }
3171 LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Completed %1 neighbouring links for %2 neighbours (%4 checked) of %3 same as merged individuals.").arg(completedNeighbourLinks).arg(completedNeighbours).arg(mTmpCompleteNeighbourSameIndiMergingSet.size()).arg(checkingNeighbours), this);
3172 return updated;
3173 }
3174
3175
3176 bool CBackendRepresentativeMemoryCache::storeIndividualIncompletelyMarked(CBackendRepresentativeMemoryCacheIndividualAssociationData* locAssociationData, bool markIncompletelyHandled, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
3177 bool associationsUpdated = false;
3178 cint64 individualID = locAssociationData->getAssociatedIndividualId();
3179 if (!markIncompletelyHandled) {
3180 if (locAssociationData->isIncompletelyMarked()) {
3181 ontologyData->decIncompletelyHandledIndividualIdCount();
3182 //if (!mIncompletelyAssociatedIndividualSet.contains(CIndividualReference(individualID))) {
3183 // bool debug = true;
3184 //}
3185 //mIncompletelyAssociatedIndividualSet.remove(CIndividualReference(individualID));
3186 locAssociationData->setIncompletelyMarked(false);
3187 associationsUpdated = true;
3188
3189 if (locAssociationData->hasProblematicLevel()) {
3190 CCACHINGSET<cint64>* propIndiSet = ontologyData->getProblematicIncompletelyHandledIndividualSet();
3191 if (propIndiSet) {
3192 propIndiSet->remove(individualID);
3193 }
3194 }
3195 }
3196 } else {
3197 ontologyData->setLastMinIncompletelyHandledIndvidualiId(qMin(ontologyData->getLastMinIncompletelyHandledIndvidualiId(), individualID));
3198 if (!locAssociationData->isIncompletelyMarked()) {
3199 ontologyData->incIncompletelyHandledIndividualIdCount();
3200 //if (mIncompletelyAssociatedIndividualSet.contains(CIndividualReference(individualID))) {
3201 // bool debug = true;
3202 //}
3203 //mIncompletelyAssociatedIndividualSet.insert(CIndividualReference(individualID));
3204 locAssociationData->setIncompletelyMarked(true);
3205 associationsUpdated = true;
3206
3207 if (locAssociationData->hasProblematicLevel()) {
3208 CCACHINGSET<cint64>* propIndiSet = ontologyData->getProblematicIncompletelyHandledIndividualSet();
3209 if (!propIndiSet) {
3210 CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
3211 propIndiSet = CObjectParameterizingAllocator < CCACHINGSET<cint64>, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
3212 }
3213 propIndiSet->insert(individualID);
3214 }
3215 }
3216 }
3217 return associationsUpdated;
3218 }
3219
3220
3221
3222
3223 bool CBackendRepresentativeMemoryCache::installNominalIndirectConncetionUpdates(CBackendRepresentativeMemoryCacheTemporaryNominalIndirectConnectionDataLinker* tempNomIndirectConnDataLinker, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
3224 CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
3225 bool connectionDataUpdated = false;
3226
3227 for (CBackendRepresentativeMemoryCacheTemporaryNominalIndirectConnectionDataLinker* tempNomIndirectConnDataLinkerIt = tempNomIndirectConnDataLinker; tempNomIndirectConnDataLinkerIt; tempNomIndirectConnDataLinkerIt = tempNomIndirectConnDataLinkerIt->getNext()) {
3228 cint64 nomIndiId = tempNomIndirectConnDataLinkerIt->getIndividualID();
3229 CXLinker<cint64>* indirectlyConnectedIndividualIdLinker = tempNomIndirectConnDataLinkerIt->getIndirectlyConnectedIndividualIdLinker();
3230 cint64 lastChangeIntegrationId = tempNomIndirectConnDataLinkerIt->getLastChangeIntegrationId();
3231
3232 CBackendRepresentativeMemoryCacheNominalIndividualIndirectConnectionData*& nomConnData = (*ontologyData->getNominaIIndividualdIndirectConnectionDataHash())[nomIndiId];
3233 CBackendRepresentativeMemoryCacheNominalIndividualIndirectConnectionData* prevNomConnData = nomConnData;
3234
3235
3236 CBackendRepresentativeMemoryCacheIndividualAssociationData* nominalAssociationData = nullptr;
3237 if (ontologyData->getIndividualIdAssoiationDataVector() && nomIndiId < ontologyData->getIndividualIdAssoiationDataVectorSize()) {
3238 nominalAssociationData = ontologyData->getIndividualIdAssoiationDataVector()[nomIndiId];
3239 }
3240 if (nominalAssociationData && nominalAssociationData->getLastIntegratedIndirectlyConnectedIndividualsChangeId() != lastChangeIntegrationId && lastChangeIntegrationId > 0 ||
3241 nominalAssociationData && prevNomConnData && lastChangeIntegrationId == 0 && nominalAssociationData->hasIndirectlyConnectedIndividualIntegration() && prevNomConnData->getLastChangeId() != nominalAssociationData->getLastIntegratedIndirectlyConnectedIndividualsChangeId()) {
3242 // mark as incompletely handled
3243 connectionDataUpdated |= markRepresentativeReferencedIndividualAssociationIncompletelyHandled(nomIndiId, nominalAssociationData, ontologyData);
3244 }
3245
3246
3247 CBackendRepresentativeMemoryCacheNominalIndividualIndirectConnectionData* newNomConnData = CObjectAllocator< CBackendRepresentativeMemoryCacheNominalIndividualIndirectConnectionData >::allocateAndConstruct(context->getMemoryAllocationManager());
3248 newNomConnData->initNominalIndividualIndirectConnectionData(prevNomConnData);
3249 nomConnData = newNomConnData;
3250 newNomConnData->setLastChangeId(mNextNomConnUpdateId++);
3251
3252 for (CXLinker<cint64>* indirectlyConnectedIndividualIdLinkerIt = indirectlyConnectedIndividualIdLinker; indirectlyConnectedIndividualIdLinkerIt; indirectlyConnectedIndividualIdLinkerIt = indirectlyConnectedIndividualIdLinkerIt->getNext()) {
3253 cint64 indiId = indirectlyConnectedIndividualIdLinkerIt->getData();
3254 // avoid duplicates
3255
3256 bool alreadyPresent = false;
3257 CBackendRepresentativeMemoryCacheIndividualAssociationData* associationData = nullptr;
3258 if (ontologyData->getIndividualIdAssoiationDataVector() && indiId < ontologyData->getIndividualIdAssoiationDataVectorSize()) {
3259 associationData = ontologyData->getIndividualIdAssoiationDataVector()[indiId];
3260 }
3261
3262 CBackendRepresentativeMemoryLabelCacheItem* indiConnNomIndiIdSetLabel = nullptr;
3263 if (associationData) {
3264 if (associationData->getCacheUpdateId() < mTmpIndiAssocPrevUpdateId) {
3265 indiConnNomIndiIdSetLabel = associationData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::INDIRECTLY_CONNECTED_NOMINAL_INDIVIDUAL_SET_LABEL);
3266 } else {
3267 indiConnNomIndiIdSetLabel = mTmpIndiIndirectlyConnNomLabelItemHash.value(indiId);
3268 }
3269 }
3270
3271 if (indiConnNomIndiIdSetLabel && indiConnNomIndiIdSetLabel->hasCachedTagValue(nomIndiId)) {
3272 alreadyPresent = true;
3273 }
3274
3275
3276 if (!alreadyPresent) {
3277 CXLinker<cint64>* indiLinker = CObjectAllocator< CXLinker<cint64> >::allocateAndConstruct(context->getMemoryAllocationManager());
3278 indiLinker->initLinker(indiId);
3279 nomConnData->addIndirectlyConnectedIndividualIdLinker(indiLinker);
3280 connectionDataUpdated = true;
3281 }
3282
3283 }
3284 }
3285
3286 return connectionDataUpdated;
3287 }
3288
3289
3290
3291 void CBackendRepresentativeMemoryCache::setUpdatedIndividualAssociationData(cint64 individualID, CBackendRepresentativeMemoryCacheIndividualAssociationData* locAssociationData, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
3292 CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
3293 CBackendRepresentativeMemoryCacheIndividualAssociationData** indiIdAssoDataVector = ontologyData->getIndividualIdAssoiationDataVector();
3294 cint64 indiIdAssoDataVectorSize = ontologyData->getIndividualIdAssoiationDataVectorSize();
3295 if (individualID >= indiIdAssoDataVectorSize) {
3296 cint64 prevSize = indiIdAssoDataVectorSize;
3297 CBackendRepresentativeMemoryCacheIndividualAssociationData** prevIndiIdAssoDataVector = indiIdAssoDataVector;
3298 indiIdAssoDataVectorSize = qMax(indiIdAssoDataVectorSize * 10, individualID);
3299 indiIdAssoDataVector = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualAssociationData* >::allocateAndConstructArray(context->getMemoryAllocationManager(), indiIdAssoDataVectorSize);
3300 for (cint64 i = 0; i < prevSize; ++i) {
3301 indiIdAssoDataVector[i] = prevIndiIdAssoDataVector[i];
3302 }
3303 for (cint64 i = prevSize; i < indiIdAssoDataVectorSize; ++i) {
3304 indiIdAssoDataVector[i] = nullptr;
3305 }
3306 ontologyData->setIndividualIdAssoiationDataVector(indiIdAssoDataVectorSize, indiIdAssoDataVector);
3307 }
3308 ontologyData->updateMaxStoredIndvidualiId(individualID);
3309 if (!indiIdAssoDataVector[individualID]) {
3310 ontologyData->incIndividualAssociationsCount();
3311 }
3312 if (locAssociationData->getPreviousData() && !locAssociationData->getPreviousData()->getPreviousData()) {
3313 ontologyData->incIndividualAssociationDataUpdateCount();
3314 }
3315 indiIdAssoDataVector[individualID] = locAssociationData;
3316 //mIndiIdAssoDataHash->insert(individualID, locAssociationData);
3317 DEBUG_HANDLING_STRING(QString("Stored update data for %1").arg(locAssociationData->getAssociatedIndividualId()))
3318 }
3319
3320
3321 bool CBackendRepresentativeMemoryCache::getIncompletlyAssociationCachedIndividuals(cint64 ontologyIdentifier, CBackendIndividualRetrievalComputationUpdateCoordinationHash* prevCoordHash, CBackendIndividualRetrievalComputationUpdateCoordinationHash* newCoordHash, bool allIndividualsAdded, bool refillRetrievalCoordHash, cint64 limit, CCallbackData* callbackData) {
3322 if (!callbackData) {
3323 CBlockingCallbackData blockingCallbackData;
3324 if (mConfDirectUpdateSynchronization) {
3325 mDirectUpdateSyncMutex.lock();
3326 CRetrieveIncompletelyAssociationCachedEvent* procEvent = new CRetrieveIncompletelyAssociationCachedEvent(&blockingCallbackData, ontologyIdentifier, prevCoordHash, newCoordHash, allIndividualsAdded, refillRetrievalCoordHash, limit);
3327 processCustomsEvents(CRetrieveIncompletelyAssociationCachedEvent::EVENTTYPE, procEvent);
3328 mDirectUpdateSyncMutex.unlock();
3329 } else {
3330 postEvent(new CRetrieveIncompletelyAssociationCachedEvent(&blockingCallbackData, ontologyIdentifier, prevCoordHash, newCoordHash, allIndividualsAdded, refillRetrievalCoordHash, limit));
3331 }
3332 blockingCallbackData.waitForCallback();
3333 } else {
3334 if (mConfDirectUpdateSynchronization) {
3335 mDirectUpdateSyncMutex.lock();
3336 CRetrieveIncompletelyAssociationCachedEvent* procEvent = new CRetrieveIncompletelyAssociationCachedEvent(callbackData, ontologyIdentifier, prevCoordHash, newCoordHash, allIndividualsAdded, refillRetrievalCoordHash, limit);
3337 processCustomsEvents(CRetrieveIncompletelyAssociationCachedEvent::EVENTTYPE, procEvent);
3338 mDirectUpdateSyncMutex.unlock();
3339 } else {
3340 postEvent(new CRetrieveIncompletelyAssociationCachedEvent(callbackData, ontologyIdentifier, prevCoordHash, newCoordHash, allIndividualsAdded, refillRetrievalCoordHash, limit));
3341 }
3342 }
3343 return true;
3344 }
3345
3346 bool CBackendRepresentativeMemoryCache::initializeIndividualsAssociationCaching(cint64 ontologyIdentifier, cint64 individualCount) {
3347 if (mConfDirectUpdateSynchronization) {
3348 mDirectUpdateSyncMutex.lock();
3349 CInitializeIndividualAssociationsCacheEvent* procEvent = new CInitializeIndividualAssociationsCacheEvent(ontologyIdentifier, individualCount);
3350 processCustomsEvents(CInitializeIndividualAssociationsCacheEvent::EVENTTYPE, procEvent);
3351 mDirectUpdateSyncMutex.unlock();
3352 } else {
3353 postEvent(new CInitializeIndividualAssociationsCacheEvent(ontologyIdentifier, individualCount));
3354 }
3355 return true;
3356 }
3357
3358 bool CBackendRepresentativeMemoryCache::reportMaximumHandledRecomputationId(cint64 ontologyIdentifier, cint64 maximumRecomputationId) {
3359 if (mConfDirectUpdateSynchronization) {
3360 mDirectUpdateSyncMutex.lock();
3361 CReportMaximumHandledRecomputationIdsEvent* procEvent = new CReportMaximumHandledRecomputationIdsEvent(ontologyIdentifier, maximumRecomputationId);
3362 processCustomsEvents(CReportMaximumHandledRecomputationIdsEvent::EVENTTYPE, procEvent);
3363 mDirectUpdateSyncMutex.unlock();
3364 } else {
3365 postEvent(new CReportMaximumHandledRecomputationIdsEvent(ontologyIdentifier, maximumRecomputationId));
3366 }
3367 return true;
3368 }
3369
3370 bool CBackendRepresentativeMemoryCache::requiresIndividualAssociations(cint64 labelType) {
3371 if (labelType == CBackendRepresentativeMemoryLabelCacheItem::FULL_CONCEPT_SET_LABEL || labelType == CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL || labelType == CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_COMBINED_EXISTENTIAL_INSTANTIATED_ROLE_SET_LABEL ||
3372 labelType == CBackendRepresentativeMemoryLabelCacheItem::NONDETERMINISTIC_COMBINED_EXISTENTIAL_INSTANTIATED_ROLE_SET_LABEL || labelType == CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL || labelType == CBackendRepresentativeMemoryLabelCacheItem::NONDETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL ||
3373 labelType == CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_COMBINED_DATA_INSTANTIATED_ROLE_SET_LABEL || labelType == CBackendRepresentativeMemoryLabelCacheItem::NONDETERMINISTIC_COMBINED_DATA_INSTANTIATED_ROLE_SET_LABEL) {
3374 return true;
3375 }
3376 return false;
3377 }
3378
3379
3380 CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData* CBackendRepresentativeMemoryCache::getIndividualAssociationsExtensionData(CBackendRepresentativeMemoryLabelCacheItem* labelItem, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
3381 CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
3382 CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData* extensionData = (CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData*)labelItem->getExtensionData(CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData::INDIVIDUAL_ASSOCIATION_MAP);
3383 if (!extensionData) {
3384 extensionData = CObjectParameterizingAllocator< CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
3385 labelItem->setExtensionData(CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData::INDIVIDUAL_ASSOCIATION_MAP, extensionData);
3386 }
3387 return extensionData;
3388 }
3389
3390
3391 CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* CBackendRepresentativeMemoryCache::getIndividualNeighbourArrayIndexExtensionData(CBackendRepresentativeMemoryLabelCacheItem* labelItem, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
3392 CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
3393 CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* extensionData = (CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData*)labelItem->getExtensionData(CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData::INDIVIDUAL_NEIGHBOUR_ARRAY_INDEX);
3394 if (!extensionData) {
3395 extensionData = CObjectParameterizingAllocator< CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
3396 labelItem->setExtensionData(CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData::INDIVIDUAL_NEIGHBOUR_ARRAY_INDEX, extensionData);
3397 extensionData->initNeighbourArrayIndexData(labelItem);
3398 }
3399 return extensionData;
3400 }
3401
3402
3403 CBackendRepresentativeMemoryLabelCacheItemTagLabelResolvingExtensionData* CBackendRepresentativeMemoryCache::getNeighbourArrayRoleTagResolvingLabelExtensionData(CBackendRepresentativeMemoryLabelCacheItem* labelItem, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
3404 CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
3405 CBackendRepresentativeMemoryLabelCacheItemTagLabelResolvingExtensionData* extensionData = (CBackendRepresentativeMemoryLabelCacheItemTagLabelResolvingExtensionData*)labelItem->getExtensionData(CBackendRepresentativeMemoryLabelCacheItemTagLabelResolvingExtensionData::TAG_RESOLVING_HASH);
3406 if (!extensionData) {
3407 extensionData = CObjectParameterizingAllocator< CBackendRepresentativeMemoryLabelCacheItemTagLabelResolvingExtensionData, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
3408 labelItem->setExtensionData(CBackendRepresentativeMemoryLabelCacheItemTagLabelResolvingExtensionData::TAG_RESOLVING_HASH, extensionData);
3409 extensionData->initTagLabelResolvingExtensionData();
3410
3411 CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* indexExtensionData = getIndividualNeighbourArrayIndexExtensionData(labelItem, ontologyData);
3412 for (cint64 i = 0; i < indexExtensionData->getArraySize(); ++i) {
3413 CBackendRepresentativeMemoryLabelCacheItem* neighbourRoleSetLabelItem = indexExtensionData->getNeighbourRoleSetLabel(i);
3414
3415 for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = neighbourRoleSetLabelItem->getCacheValueLinker(); labelValueLinkerIt; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
3416 CCacheValue cacheValue = labelValueLinkerIt->getData();
3417
3418
3419 CCacheValue::CACHEVALUEIDENTIFIER identifier = cacheValue.getCacheValueIdentifier();
3420 bool nondeterministc = identifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_ROLE || identifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_INVERSEDROLE || identifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_ASSERTEDROLE ||
3421 identifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_INVERSEDASSERTEDROLE || identifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_NOMINAL_CONNECTED_ROLE || identifier == CCacheValue::CACHEVALTAGAND_NONDETERMINISTIC_INVERSED_NOMINAL_CONNECTED_ROLE;
3422
3423
3424 CBackendRepresentativeMemoryLabelCacheItemTagLabelResolvingDataLinker* linker = CObjectAllocator< CBackendRepresentativeMemoryLabelCacheItemTagLabelResolvingDataLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
3425 linker->initTagLabelResolvingData(neighbourRoleSetLabelItem, i, !nondeterministc);
3426 extensionData->appendTagLabelResolvingDataLinker(cacheValue.getTag(), linker);
3427 }
3428 }
3429
3430 }
3431 return extensionData;
3432 }
3433
3434
3435 cint64 CBackendRepresentativeMemoryCache::getMinimumSlotReferreringInstalledValidRecomputationId(CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
3436 cint64 minInstalledValidRecId = CINT64_MAX;
3437 cint64 ontologyIdentifier = ontologyData->getOntologyIdentifer();
3438
3439 for (CBackendRepresentativeMemoryCacheSlotItem* slotLinkerIt = mSlotLinker; slotLinkerIt; slotLinkerIt = slotLinkerIt->getNext()) {
3440 CBackendRepresentativeMemoryCacheOntologyData* referredOntologyData = slotLinkerIt->getOntologyData(ontologyIdentifier);
3441 cint64 minValidRecId = referredOntologyData->getMinimumValidRecomputationId();
3442 minInstalledValidRecId = qMin(minInstalledValidRecId, minValidRecId);
3443 }
3444 return minInstalledValidRecId;
3445 }
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484 bool CBackendRepresentativeMemoryCache::processCustomsEvents(QEvent::Type type, CCustomEvent *event) {
3485 if (CThread::processCustomsEvents(type, event)) {
3486 return true;
3487
3488 }
3489 else if (type == CReportMaximumHandledRecomputationIdsEvent::EVENTTYPE) {
3490 CReportMaximumHandledRecomputationIdsEvent* rmhrie = (CReportMaximumHandledRecomputationIdsEvent*)event;
3491 cint64 maxHandledRecomId = rmhrie->getMaximumHandledRecomputationId();
3492 cint64 ontologyIdentifier = rmhrie->getOntologyIdentifier();
3493
3494 CBackendRepresentativeMemoryCacheOntologyData* ontologyData = mOntologyIdentifierDataHash->value(ontologyIdentifier);
3495 if (ontologyData) {
3496 ontologyData->setNextUpdateMinimumValidRecomputationId(maxHandledRecomId+1);
3497 LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Received report of handled recomputation id %1.").arg(maxHandledRecomId), this);
3498 }
3499 ++mStatReportedMaximumHandledRecomputationIdCount;
3500
3501 return true;
3502
3503 } else if (type == CInitializeIndividualAssociationsCacheEvent::EVENTTYPE) {
3504 CInitializeIndividualAssociationsCacheEvent* iiace = (CInitializeIndividualAssociationsCacheEvent*)event;
3505
3506 cint64 indiCount = iiace->getIndividualCount();
3507 cint64 ontologyIdentifier = iiace->getOntologyIdentifier();
3508
3509 prepareOntologyDataUpdate(ontologyIdentifier, indiCount);
3510
3511 return true;
3512
3513 } else if (type == CRetrieveIncompletelyAssociationCachedEvent::EVENTTYPE) {
3514 CRetrieveIncompletelyAssociationCachedEvent* iace = (CRetrieveIncompletelyAssociationCachedEvent*)event;
3515 cint64 limit = iace->getRetrievalLimit();
3516 cint64 ontologyIdentifier = iace->getOntologyIdentifier();
3517
3518 bool allIndividualsAdded = iace->hasAllIndividualsAdded();
3519 bool refillRetrievalCoordHash = iace->hasRefillRetrievalCoordHashOrdered();
3520
3521
3522 ++mCheckingRemainingIncompletelyHandledCount;
3523 bool updateSlot = false;
3524
3525 CBackendRepresentativeMemoryCacheOntologyData* ontologyData = mOntologyIdentifierDataHash->value(ontologyIdentifier);
3526 if (ontologyData && !ontologyData->isAssociationCompleted()) {
3527 ontologyData->incIncompletelyHandledIndividualsRetrievalCount();
3528 cint64 count = 0;
3529 cint64 prevLastMinIncompletelyHandledIndiId = ontologyData->getLastMinIncompletelyHandledIndvidualiId();
3530
3531 CBackendIndividualRetrievalComputationUpdateCoordinationHash* lastRetrievalHash = iace->getLastIndividualsRetrievalCoordinationHash();
3532 CBackendIndividualRetrievalComputationUpdateCoordinationHash* newRetrievalHash = iace->getNewIndividualsRetrievalCoordinationHash();
3533
3534
3535
3536
3537
3538 newRetrievalHash->setApproximateTotalIncompletelyHandledCount(ontologyData->getIncompletelyHandledIndividualIdCount());
3539 if (lastRetrievalHash) {
3540
3541 cint64 corr = lastRetrievalHash->getApproximateCorrectionIncompletelyHandledCount();
3542 //LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("last corr %1").arg(corr), this);
3543 corr += ontologyData->getIncompletelyHandledIndividualIdCount() - lastRetrievalHash->getApproximateRemainingIncompletelyHandledCount();
3544 //LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("remain diff %1").arg(ontologyData->getIncompletelyHandledIndividualIdCount() - lastRetrievalHash->getApproximateRemainingIncompletelyHandledCount()), this);
3545 corr += lastRetrievalHash->getApproximateTotalIncompletelyHandledCount() - ontologyData->getIncompletelyHandledIndividualIdCount();
3546 //LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("last diff %1").arg(lastRetrievalHash->getApproximateTotalIncompletelyHandledCount() - ontologyData->getIncompletelyHandledIndividualIdCount()), this);
3547 newRetrievalHash->setApproximateCorrectionIncompletelyHandledCount(corr);
3548 //LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("new corr %1").arg(corr), this);
3549 }
3550
3551
3552
3553 if (lastRetrievalHash && !refillRetrievalCoordHash) {
3554 for (CBackendIndividualRetrievalComputationUpdateCoordinationHash::const_iterator it = lastRetrievalHash->constBegin(), itEnd = lastRetrievalHash->constEnd(); it != itEnd; ++it) {
3555 cint64 indiId = it.key();
3556 CBackendIndividualRetrievalComputationUpdateCoordinationHashData* refData = it.value();
3557 bool keepIndi = false;
3558 CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssData = ontologyData->getIndividualIdAssoiationDataVector()[indiId];
3559 if (!refData->isComputationIntegrated() && !refData->isComputationOrdered()) {
3560 keepIndi = true;
3561 //newRetrievalHash->incHashRemainingCount();
3562 //if (!indiAssData->isIncompletelyMarked()) {
3563 // newRetrievalHash->incApproximateCorrectionIncompletelyHandledCount();
3564 //}
3565 } else if (!refData->isProcessed()) {
3566 //newRetrievalHash->incHashComputationCount();
3567 keepIndi = true;
3568 } else {
3569 if (indiAssData->getCacheTouchId() == refData->getAssociationUpdateId()) {
3570 //newRetrievalHash->incHashProcessedCount();
3571 keepIndi = true;
3572 }
3573 }
3574 if (keepIndi) {
3575 refData->incUsageCount();
3576 newRetrievalHash->insert(indiId, refData);
3577
3578 } else {
3579 //newRetrievalHash->incHashRemovedCount();
3580 }
3581 }
3582 }
3583
3584
3585
3586 if (ontologyData->isBasicPrecomputationMode()) {
3587
3588
3589 if (newRetrievalHash) {
3590 cint64 pos = ontologyData->getBasicPrecompuationRetrievalIndividualIdPosition();
3591 for (; pos < ontologyData->getBasicPrecomputationIndividualIdAssoiationDataVectorSize() && count < ontologyData->getIncompletelyHandledIndividualIdCount() && (count < limit || limit < 0); ++pos) {
3592 CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssData = ontologyData->getIndividualIdAssoiationDataVector()[pos];
3593 if (indiAssData && !indiAssData->isCompletelyHandled() && indiAssData->getAssociationDataUpdateId() <= 1) {
3594
3595 CBackendIndividualRetrievalComputationUpdateCoordinationHashData*& refData = (*newRetrievalHash)[pos];
3596 if (!refData) {
3597 refData = newRetrievalHash->createCoordinationData();
3598 refData->setAssociationUpdateId(indiAssData->getCacheTouchId());
3599 refData->incUsageCount();
3600 newRetrievalHash->incHashRemainingCount();
3601 count++;
3602 }
3603
3604 }
3605 }
3606 ontologyData->setBasicPrecompuationRetrievalIndividualIdPosition(pos);
3607 }
3608
3609 if (newRetrievalHash->isEmpty()) {
3610 // deactivate basic precompuation mode
3611 ontologyData->setBasicPrecomputationMode(false);
3612 updateSlot = true;
3613 newRetrievalHash->setBasicPrecomputationFinished(true);
3614 } else {
3615 newRetrievalHash->setBasicPrecomputationMode(true);
3616 }
3617
3618 }
3619 if (!ontologyData->isBasicPrecomputationMode()) {
3620
3621 CCACHINGSET<cint64>* propIndiSet = ontologyData->getProblematicIncompletelyHandledIndividualSet();
3622 if (propIndiSet) {
3623 for (CCACHINGSET<cint64>::const_iterator it = propIndiSet->constBegin(), itEnd = propIndiSet->constEnd(); it != itEnd && count < ontologyData->getIncompletelyHandledIndividualIdCount() && (count < limit || limit < 0); ++it) {
3624 cint64 indiId = *it;
3625 CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssData = ontologyData->getIndividualIdAssoiationDataVector()[indiId];
3626
3627 CBackendIndividualRetrievalComputationUpdateCoordinationHashData*& refData = (*newRetrievalHash)[indiId];
3628 if (!refData) {
3629 refData = newRetrievalHash->createCoordinationData();
3630 refData->setAssociationUpdateId(indiAssData->getCacheTouchId());
3631 refData->incUsageCount();
3632 newRetrievalHash->incHashRemainingCount();
3633 count++;
3634 }
3635
3636 }
3637 }
3638
3639
3640
3641 if (newRetrievalHash) {
3642 for (cint64 i = ontologyData->getLastMinIncompletelyHandledIndvidualiId(); i < ontologyData->getIndividualIdAssoiationDataVectorSize() && count < ontologyData->getIncompletelyHandledIndividualIdCount() && (count < limit || limit < 0); ++i) {
3643 CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssData = ontologyData->getIndividualIdAssoiationDataVector()[i];
3644 if (indiAssData && !indiAssData->isCompletelyHandled()) {
3645
3646 CBackendIndividualRetrievalComputationUpdateCoordinationHashData*& refData = (*newRetrievalHash)[i];
3647 if (!refData) {
3648 refData = newRetrievalHash->createCoordinationData();
3649 refData->setAssociationUpdateId(indiAssData->getCacheTouchId());
3650 refData->incUsageCount();
3651 newRetrievalHash->incHashRemainingCount();
3652 count++;
3653 }
3654
3655 } else if (i == ontologyData->getLastMinIncompletelyHandledIndvidualiId()) {
3656 ontologyData->incLastMinIncompletelyHandledIndvidualiId();
3657 }
3658 }
3659
3660 /*if (limit < 0 || limit > mIncompletelyAssociatedIndividualSet.size()) {
3661 *individualSet = mIncompletelyAssociatedIndividualSet;
3662 } else {
3663 cint64 count = 0;
3664 for (QSet<CIndividualReference>::const_iterator it = mIncompletelyAssociatedIndividualSet.constBegin(), itEnd = mIncompletelyAssociatedIndividualSet.constEnd(); it != itEnd && count < limit; ++it) {
3665 individualSet->insert(*it);
3666 ++count;
3667 }
3668 }*/
3669
3670
3671 if (newRetrievalHash->size() <= 0 && ontologyData->getIncompletelyHandledIndividualIdCount() > 0) {
3672 cint64 foundIncompleteCount = 0;
3673 cint64 prevIncompleteCount = ontologyData->getIncompletelyHandledIndividualIdCount();
3674 for (cint64 i = 0; i < ontologyData->getIndividualIdAssoiationDataVectorSize() && count < ontologyData->getIncompletelyHandledIndividualIdCount() && (count < limit || limit < 0); ++i) {
3675 CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssData = ontologyData->getIndividualIdAssoiationDataVector()[i];
3676 if (indiAssData && !indiAssData->isCompletelyHandled()) {
3677
3678 if (foundIncompleteCount <= 0) {
3679 LOG(WARN, "::Konclude::RepresentativeBackendCache", logTr("Corrected first insufficiently handled individual from %2 to %1.").arg(ontologyData->getLastMinIncompletelyHandledIndvidualiId()).arg(i), this);
3680 ontologyData->setLastMinIncompletelyHandledIndvidualiId(i);
3681 }
3682 foundIncompleteCount++;
3683 CBackendIndividualRetrievalComputationUpdateCoordinationHashData*& refData = (*newRetrievalHash)[i];
3684 if (!refData) {
3685 refData = newRetrievalHash->createCoordinationData();
3686 refData->setAssociationUpdateId(indiAssData->getCacheTouchId());
3687 refData->incUsageCount();
3688 newRetrievalHash->incHashRemainingCount();
3689 count++;
3690 }
3691
3692 }
3693 }
3694 if (foundIncompleteCount <= 0) {
3695 ontologyData->setIncompletelyHandledIndividualIdCount(0);
3696 }
3697 LOG(WARN, "::Konclude::RepresentativeBackendCache", logTr("Corrected insufficiently handled individuals from %2 to %1.").arg(foundIncompleteCount).arg(prevIncompleteCount), this);
3698
3699 }
3700
3701 }
3702 }
3703
3704
3705 if (allIndividualsAdded) {
3706 if (!ontologyData->isFirstIncompletelyHandledIndividualsRetrieved()) {
3707 if (!ontologyData->isSlotUpdateIntegrated()) {
3708 updateSlot = true;
3709 }
3710 }
3711 ontologyData->setFirstIncompletelyHandledIndividualsRetrieved(true);
3712 }
3713
3714 bool forceCompletion = false;
3715 if (mConfMaxIncompletelyHandledIndividualsRetrievalCount > 0 && ontologyData->getIncompletelyHandledIndividualsRetrievalCount() > mConfMaxIncompletelyHandledIndividualsRetrievalCount) {
3716 LOG(WARN, "::Konclude::RepresentativeBackendCache", logTr("Forcing association completion with reached maximum of %1 retrievals.").arg(mConfMaxIncompletelyHandledIndividualsRetrievalCount), this);
3717 forceCompletion = true;
3718 count = 0;
3719 if (newRetrievalHash) {
3720 newRetrievalHash->clear();
3721 }
3722 }
3723 checkAssociationComplete(ontologyData, forceCompletion);
3724
3725 LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Retrieval %3 returning %1 new of %2 insufficiently handled individuals for reprocessing.").arg(count).arg(ontologyData->getIncompletelyHandledIndividualIdCount()).arg(ontologyData->getIncompletelyHandledIndividualsRetrievalCount()), this);
3726
3727 //if (mIncompletelyHandledIndiIdCount != mIncompletelyAssociatedIndividualSet.size()) {
3728 // bool debug = true;
3729 //}
3730
3731 //if (mIncompletelyHandledIndiIdCount > 0 && individualSet->isEmpty()) {
3732 // cint64 count = 0;
3733 // for (cint64 i = 0; i < mIndiIdAssoDataVectorSize && count < mIncompletelyHandledIndiIdCount && count < limit; ++i) {
3734 // CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssData = mIndiIdAssoDataVector[i];
3735 // if (indiAssData && !indiAssData->isCompletelyHandled()) {
3736 // individualSet->insert(i);
3737 // count++;
3738 // }
3739 // }
3740
3741 // count = count;
3742 // bool debug = true;
3743 //}
3744
3745
3746
3747 }
3748
3749
3750 CCallbackData* callbackData = iace->getCallback();
3751 if (callbackData) {
3752 callbackData->doCallback();
3753 }
3754
3755
3756 if (updateSlot) {
3757 ontologyData->setNextSlotUpdateWaitingCount(mSlotUpdateWaitingIncreaseCount);
3758 createReaderSlotUpdate(ontologyData, &mContext);
3759 cleanUnusedSlots(&mContext);
3760 }
3761 return true;
3762
3763
3764 }
3765 else if (type == CWriteBackendAssociationCachedEvent::EVENTTYPE) {
3766 CWriteBackendAssociationCachedEvent* wcde = (CWriteBackendAssociationCachedEvent*)event;
3767 CMemoryPool* memoryPools = wcde->getMemoryPools();
3768 CBackendRepresentativeMemoryCacheWriteData* newWriteData = (CBackendRepresentativeMemoryCacheWriteData*)wcde->getWriteData();
3769 cint64 dataWriteCount = newWriteData->getCount();
3770
3771 CBackendRepresentativeMemoryCacheWriteData* dataLinkerIt = newWriteData;
3772
3773 cint64 ontologyIdentifier = dataLinkerIt->getOntologyIdentifier();
3774
3775 CBackendRepresentativeMemoryCacheOntologyData* ontologyData = mOntologyIdentifierDataHash->value(ontologyIdentifier);
3776 mLastHandledOntologyContext = ontologyData;
3777
3778 if (mStatCollectStatistics) {
3779 mPendingUpdateCount.deref();
3780 }
3781 mCurrentUpdateHandlingRecomputationId = -1;
3782
3783 if (!ontologyData || !ontologyData->isAssociationCompleted()) {
3784 ontologyData->incCacheDataUpdateWritingCount();
3785 ++mWriteDataCount;
3786 ontologyData = prepareOntologyDataUpdate(ontologyIdentifier);
3787 cint64 prevUpdatedIndiCount = mUpdatedIndiCount;
3788 cint64 prevAssociationUpdatedIndiCount = mAssociationUpdatedIndiCount;
3789 bool allCachingSuccess = true;
3790 bool oneCachingSuccess = false;
3791 bool oneCachingExpSuccess = false;
3792 while (dataLinkerIt) {
3793 mCurrentUpdateHandlingRecomputationId = -1;
3794 if (dataLinkerIt->getCacheWriteDataType() == CBackendRepresentativeMemoryCacheWriteData::BACKENDASSOCIATIONWRITEDATATYPE) {
3795 CBackendRepresentativeMemoryCacheLabelAssociationWriteData* baAsWrDa = (CBackendRepresentativeMemoryCacheLabelAssociationWriteData*)dataLinkerIt;
3796
3797 mCurrentUpdateHandlingRecomputationId = baAsWrDa->getRecompuationId();
3798
3799 CBackendRepresentativeMemoryCacheTemporaryAssociationWriteDataLinker* tempAssWriteDataLinker = baAsWrDa->getTemporaryAssociationWriteDataLinker();
3800 CBackendRepresentativeMemoryCacheTemporaryLabelWriteDataLinker* tempLabelWriteDataLinker = baAsWrDa->getTemporaryLabelWriteDataLinker();
3801 CBackendRepresentativeMemoryCacheTemporaryCardinalityWriteDataLinker* tempCardWriteDataLinker = baAsWrDa->getTemporaryCardinaltyWriteDataLinker();
3802 CBackendRepresentativeMemoryCacheTemporaryAssociationUseDataLinker* tempAssUseDataLinker = baAsWrDa->getTemporaryAssociationUseDataLinker();
3803 CBackendRepresentativeMemoryCacheTemporaryNominalIndirectConnectionDataLinker* tempNomIndirectConnDataLinker = baAsWrDa->getTemporaryNominalIndirectConnectionDataLinker();
3804 CBackendRepresentativeMemoryCacheTemporaryInvolvedIndividualDataLinker* tmpInvolvedIndiDataLinker = baAsWrDa->getTemporaryInvolvedIndividualIdDataLinker();
3805 CBackendRepresentativeMemoryCacheTemporaryPropagationCutDataLinker* tmpPropCutDataLinker = baAsWrDa->getTemporaryPropagationCutDataLinker();
3806
3807 installTemporaryLabels(tempLabelWriteDataLinker, ontologyData);
3808 installTemporaryCardinalities(tempCardWriteDataLinker, ontologyData);
3809
3810 mTmpIndiAssocPrevUpdateId = mNextIndiUpdateId;
3811 mTmpIndiIndirectlyConnNomLabelItemHash.clear();
3812
3813
3814 bool cached = false;
3815
3816 if (checkUpdateRejection(tempAssWriteDataLinker, ontologyData)) {
3817 cached |= handleUpdateRejection(tempAssWriteDataLinker, ontologyData);
3818 LOG(WARN, "::Konclude::RepresentativeBackendCache", logTr("Recjecting installation of update from recomputation id %1.").arg(mCurrentUpdateHandlingRecomputationId), this);
3819
3820 } else {
3821
3822 cached |= integratePropagationCut(tmpPropCutDataLinker, ontologyData);
3823 analyseDeterministicSameAsAssociationInstallation(tempAssWriteDataLinker, ontologyData);
3824 cached |= installAssociationUpdates(tempAssWriteDataLinker, ontologyData);
3825 cached |= installNominalIndirectConncetionUpdates(tempNomIndirectConnDataLinker, ontologyData);
3826 cached |= checkAssociationUsage(tempAssUseDataLinker, ontologyData);
3827 cached |= updateInvolvedIndividuals(tmpInvolvedIndiDataLinker, ontologyData);
3828 cached |= installDeterministicSameAsAssociationUpdates(tempAssWriteDataLinker, ontologyData);
3829 mDeterministicSameHandlingInstallationDataHash.clear();
3830 mPropagationCutIndiSet.clear();
3831
3832
3833 if (!mTmpDetSameMergingCompletionReferenceHash.isEmpty()) {
3834 cached |= completeDeterministicSameAsMergingInformation(mTmpDetSameMergingCompletionReferenceHash, ontologyData);
3835 mTmpDetSameMergingCompletionReferenceHash.clear();
3836 }
3837 if (!mTmpCompleteNeighbourSameIndiMergingSet.isEmpty()) {
3838 cached |= completeNeighboursForSameAsMerging(mTmpCompleteNeighbourSameIndiMergingSet, ontologyData);
3839 mTmpCompleteNeighbourSameIndiMergingSet.clear();
3840 }
3841
3842 if (mTmpPropCutIndiArrayNeighboursHandlingDataHash) {
3843 for (QHash<cint64, CPropagationCutNeighbourArrayHandlingData*>::const_iterator it = mTmpPropCutIndiArrayNeighboursHandlingDataHash->constBegin(), itEnd = mTmpPropCutIndiArrayNeighboursHandlingDataHash->constBegin(); it != itEnd; ++it) {
3844 delete it.value();
3845 }
3846 delete mTmpPropCutIndiArrayNeighboursHandlingDataHash;
3847 mTmpPropCutIndiArrayNeighboursHandlingDataHash = nullptr;
3848 }
3849 }
3850
3851 allCachingSuccess &= cached;
3852 oneCachingSuccess |= cached;
3853 oneCachingExpSuccess |= cached;
3854 }
3855 dataLinkerIt = (CBackendRepresentativeMemoryCacheWriteData*)dataLinkerIt->getNext();
3856 }
3857
3858 if (!oneCachingSuccess) {
3859 ++mEmptyWriteDataCount;
3860 }
3861
3862 bool forceCompletion = false;
3863 if (mConfMaxCacheDataUpdateWritingCount > 0 && ontologyData->getCacheDataUpdateWritingCount() > mConfMaxCacheDataUpdateWritingCount) {
3864 LOG(WARN, "::Konclude::RepresentativeBackendCache", logTr("Forcing association completion with reached maximum of %1 updates.").arg(mConfMaxCacheDataUpdateWritingCount), this);
3865 forceCompletion = true;
3866 }
3867 checkAssociationComplete(ontologyData, forceCompletion);
3868
3869 bool basicPrecomputationModeActivated = false;
3870 if (checkBasicPrecompuationModeActivation(ontologyData)) {
3871 if (activateBasicPrecompuationMode(ontologyData)) {
3872 basicPrecomputationModeActivated = true;
3873 }
3874 }
3875
3876 bool slotUpdate = false;
3877 if (ontologyData->getNextSlotUpdateWaitingCount() <= 0 || ontologyData->isAssociationCompleted() || basicPrecomputationModeActivated) {
3878 deleteExpiredIndividualAssociationMemoryContexts(ontologyData, &mContext);
3879 createReaderSlotUpdate(ontologyData, &mContext);
3880 cleanUnusedSlots(&mContext);
3881 ontologyData->setNextSlotUpdateWaitingCount(mSlotUpdateWaitingIncreaseCount++);
3882 if (mSlotUpdateWaitingMaxCount >= 0 && mSlotUpdateWaitingIncreaseCount > mSlotUpdateWaitingMaxCount) {
3883 mSlotUpdateWaitingIncreaseCount = mSlotUpdateWaitingMaxCount;
3884 }
3885 slotUpdate = true;
3886 } else {
3887 ontologyData->setNextSlotUpdateWaitingCount(ontologyData->getNextSlotUpdateWaitingCount() - 1);
3888 }
3889
3890 cint64 updatedIndiCount = mUpdatedIndiCount - prevUpdatedIndiCount;
3891 cint64 directUpdatedIndiCount = mAssociationUpdatedIndiCount - prevAssociationUpdatedIndiCount;
3892 QString updateType = QString("intern");
3893 if (slotUpdate) {
3894 updateType = QString("published");
3895 }
3896 QString pendingString;
3897 if (mStatCollectStatistics) {
3898 pendingString = QString(", %1 pending").arg(mPendingUpdateCount);
3899 }
3900 LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Update %1 (%5%6) for representative backend cache, updated %2 (%7 direct) of overall %3 individual associations, %4 remaining insufficiently handled individuals.")
3901 .arg(mWriteDataCount).arg(updatedIndiCount).arg(ontologyData->getIndividualAssociationsCount()).arg(ontologyData->getIncompletelyHandledIndividualIdCount()).arg(updateType).arg(pendingString).arg(directUpdatedIndiCount), this);
3902
3903 if (mStatCollectStatistics) {
3904 LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Overall %1 updated invididual associations (max %5), %2 created neighbour links (%3 only adding link updates, %4 changed/removed link udpates, %6 max links).")
3905 .arg(mUpdatedIndiCount).arg(mStatCreatedNeighbourLinks).arg(mStatAddingNeighbourLinksAssociationUpdateCount).arg(mStatUpdatedOrRemovedNeighbourLinksAssociationUpdateCount).arg(mStatMaxAssociationUpdateCount).arg(mStatMaxNeighbourLinksCount), this);
3906
3907
3908
3909 LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Successfully installed %1 deterministic same association data (of %6 deterministic merges), failed installation for %2 (%3 due to incompletely handled destination, %5 due to merged destination, %4 due to different update id).")
3910 .arg(mStatDetSameAssociationInstallCount).arg(mStatDetSameAssociationFailedCount).arg(mStatDetSameAssociationIncompleteHandledDestFailedCount).arg(mStatDetSameAssociationDifferentUpdateIdFailedCount).arg(mStatDetSameAssociationRepMergedDestFailedCount).arg(mStatDetSameRepresentativeMergingCount), this);
3911
3912 QStringList labelsStringList;
3913 for (cint64 i = 0; i < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_TYPE_COUNT; ++i) {
3914 QString labelName = getRepresentativeCacheLabelItemString(i);
3915 QString labelString = QString("\t%2 of label type %1 (max size %3, all value %4)").arg(labelName).arg(mStatLabelTypeCount[i]).arg(mStatLabelTypeMaxValueCount[i]).arg(mStatLabelTypeAllValueCount[i]);
3916 labelsStringList.append(labelString);
3917 }
3918
3919 LOG(INFO, "::Konclude::RepresentativeBackendCache", QString("Containing %2 labels, %3 max label size, %4 max same merged individuals,\r\n%1").arg(labelsStringList.join(",\r\n"))
3920 .arg(mStatLabelCount).arg(mStatMaxLabelValueCount).arg(mStatMaxSameAsMergedCount), this);
3921
3922
3923
3924 }
3925
3926
3927 } else {
3928 LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Ignoring update since individual label association is already complete."), this);
3929 }
3930
3931
3932 //mContext.releaseTemporaryMemoryPools(memoryPools);
3933 mCacheStat.setMemoryConsumption(mContext.getMemoryConsumption());
3934
3935 //if (ontologyData->isFirstIncompletelyHandledIndividualsRetrieved() && (mWriteDataCount % 50 == 0)) {
3936 // writeStringifiedRepresentativeCacheToFile();
3937 //}
3938
3939 if (mLimitRemainingWritePending) {
3940 mRemainingWritePendingSemaphore.release();
3941 }
3942
3943 #ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
3944 if (mConfDebugWriteRepresentativeCache && (mWriteDataCount <= 50 || mWriteDataCount % 500 == 0)) {
3945 //writeStringifiedRepresentativeCacheToFile();
3946 }
3947 #endif
3948
3949
3950 return true;
3951 }
3952 return false;
3953 }
3954
3955
3956
3957
3958 bool CBackendRepresentativeMemoryCache::checkBasicPrecompuationModeActivation(CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
3959 if (!ontologyData->isAssociationCompleted() && !ontologyData->hasBasicPrecomputationModeActivation() && ontologyData->getIncompletelyHandledIndividualIdCount() > 0 && mConfBasicPrecomputationModeActivationUpdateMergesRatio > 0) {
3960 double basicIndiUpdateCount = ontologyData->getIndividualAssociationDataDirectUpdateCount();
3961 double indiMergesCount = ontologyData->getIndividualAssociationMergingCount();
3962
3963 if (indiMergesCount > 0) {
3964 double ratio = indiMergesCount / basicIndiUpdateCount;
3965 if (ratio > mConfBasicPrecomputationModeActivationUpdateMergesRatio) {
3966 return true;
3967 }
3968 }
3969 }
3970 return false;
3971 }
3972
3973
3974
3975 bool CBackendRepresentativeMemoryCache::activateBasicPrecompuationMode(CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
3976 if (!ontologyData->isBasicPrecomputationMode() && !ontologyData->hasBasicPrecomputationModeActivation()) {
3977 ontologyData->setBasicPrecomputationModeActivation(true);
3978 ontologyData->setBasicPrecomputationMode(true);
3979
3980 CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
3981
3982 cint64 basicIndiUpdateCount = ontologyData->getIndividualAssociationDataDirectUpdateCount();
3983 cint64 indiMergesCount = ontologyData->getIndividualAssociationMergingCount();
3984
3985 CBackendRepresentativeMemoryCacheIndividualAssociationData** indiIdAssoDataVector = ontologyData->getIndividualIdAssoiationDataVector();
3986 cint64 indiIdAssoDataVectorSize = ontologyData->getIndividualIdAssoiationDataVectorSize();
3987 CBackendRepresentativeMemoryCacheIndividualAssociationData** basicPrecompIndiIdAssoDataVector = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualAssociationData* >::allocateAndConstructArray(context->getMemoryAllocationManager(), indiIdAssoDataVectorSize);
3988 for (cint64 i = 0; i < indiIdAssoDataVectorSize; ++i) {
3989 basicPrecompIndiIdAssoDataVector[i] = indiIdAssoDataVector[i];
3990 }
3991 ontologyData->setBasicPrecomputationIndividualIdAssoiationDataVector(indiIdAssoDataVectorSize, basicPrecompIndiIdAssoDataVector);
3992
3993 LOG(INFO, "::Konclude::RepresentativeBackendCache", QString("Activated basic precompuation mode for %1 merges at %2 updates\r\n").arg(indiMergesCount).arg(basicIndiUpdateCount), this);
3994
3995 return true;
3996 }
3997 return false;
3998 }
3999
4000
4001
4002 void CBackendRepresentativeMemoryCache::indexIndividualLabelAssociations(CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
4003 cint64 indexingCount = 0;
4004 for (cint64 i = 0; i < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_ASSOCIATABLE_TYPE_COUNT; ++i) {
4005 if (requiresIndividualAssociations(i)) {
4006 indexingCount++;
4007 }
4008 }
4009 ontologyData->setIndividualLabelAssociationIndexingCount(indexingCount);
4010 cint64 indiCount = ontologyData->getMaxStoredIndvidualiId();
4011 cint64 vecSize = ontologyData->getIndividualIdAssoiationDataVectorSize();
4012 LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Starting concurrent indexing of %1 label types for %2 individuals.").arg(indexingCount).arg(indiCount), this);
4013
4014 for (cint64 i = 0; i < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_ASSOCIATABLE_TYPE_COUNT; ++i) {
4015 bool exactIndiAssocTracking = requiresIndividualAssociations(i);
4016 if (exactIndiAssocTracking) {
4017 CConcurrentTaskScheduler::run([=] {
4018
4019
4020 CMemoryPoolNewAllocationIncreasingContext newAllocContext;
4021
4022 for (cint64 indiId = 0; indiId <= indiCount && indiId < vecSize; ++indiId) {
4023 CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssData = ontologyData->getIndividualIdAssoiationDataVector()[indiId];
4024 if (indiAssData) {
4025
4026 CBackendRepresentativeMemoryLabelCacheItem* labelItem = indiAssData->getLabelCacheEntry(i);
4027 if (labelItem) {
4028 CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData* indiAssoExtData = (CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData*)labelItem->getExtensionData(CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData::INDIVIDUAL_ASSOCIATION_MAP);
4029 if (!indiAssoExtData) {
4030 indiAssoExtData = CObjectParameterizingAllocator< CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData, CMemoryPoolNewAllocationIncreasingContext* >::allocateAndConstructAndParameterize(newAllocContext.getMemoryAllocationManager(), &newAllocContext);
4031 labelItem->setExtensionData(CBackendRepresentativeMemoryLabelCacheItemIndividualAssociationMapExtensionData::INDIVIDUAL_ASSOCIATION_MAP, indiAssoExtData);
4032 }
4033 indiAssoExtData->addIndividualIdAssociation(indiAssData);
4034 }
4035 }
4036 }
4037 CMemoryPool* memoryPools = newAllocContext.takeMemoryPools();
4038 cint64 remainingCount = ontologyData->updateIndividualLabelAssociationIndexed(true, memoryPools);
4039 if (remainingCount <= 0) {
4040 LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Finished indexing all label types for %1 individuals.").arg(indiCount), this);
4041 } else {
4042 LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Finished indexing %1 label types.").arg(indexingCount - remainingCount), this);
4043 }
4044
4045
4046 });
4047 }
4048 }
4049
4050 if (mConfWaitIndividualLabelAssociationIndexed) {
4051 LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Waiting indexing finishing."), this);
4052 ontologyData->waitIndividualLabelAssociationIndexed();
4053 }
4054
4055 }
4056
4057
4058
4059
4060
4061 bool CBackendRepresentativeMemoryCache::integratePropagationCut(CBackendRepresentativeMemoryCacheTemporaryPropagationCutDataLinker* tmpPropCutDataLinker, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
4062 bool updated = false;
4063 if (tmpPropCutDataLinker) {
4064 mLastHandledPropCutDataLinker = tmpPropCutDataLinker;
4065 cint64 propCutIndiCount = 0;
4066 cint64 propCutInfluencedIndiCount = 0;
4067 cint64 propCutAllNeighbourRelationsInfluencingCount = 0;
4068 cint64 propCutNeighbourNonDetermininisticRelationInfluencingCount = 0;
4069 cint64 propCutNeighbourRelationPropagatedConceptMissingInfluencingCount = 0;
4070 cint64 propCutNeighbourDirectConceptInfluenceUpdatingCount = 0;
4071 cint64 propCutNeighbourRelationUnspecifiedInfluencingCount = 0;
4072 CBackendRepresentativeMemoryCacheContext* context = ontologyData->getOntologyContext();
4073 CBackendRepresentativeMemoryCacheIndividualAssociationData** indiAssDataVector = ontologyData->getIndividualIdAssoiationDataVector();
4074
4075 QSet<cint64> allNeighbourRelationsConsideredIndiSet;
4076 QHash<cint64, cint64> incompletelyUpdatingNeighbourPropCutIndisHash;
4077 QSet<QPair<cint64, cint64>> incompletelyUpdatingNeighbourPropCutIndiPairSet;
4078
4079 QSet<cint64> expandedIndiSet;
4080 CXLinker<cint64>* lastInvestigatedExpandedIndiLinker = nullptr;
4081
4082 QHash<cint64, cint64> propagationCutIndiReferredNeighbourArrayIdHash;
4083
4084 for (CBackendRepresentativeMemoryCacheTemporaryPropagationCutDataLinker* tmpPropCutDataLinkerIt = tmpPropCutDataLinker; tmpPropCutDataLinkerIt; tmpPropCutDataLinkerIt = tmpPropCutDataLinkerIt->getNext()) {
4085 CXLinker<cint64>* expandedIndiLinker = tmpPropCutDataLinkerIt->getExpandedIndividualIdsLinker();
4086 if (lastInvestigatedExpandedIndiLinker != expandedIndiLinker) {
4087 lastInvestigatedExpandedIndiLinker = expandedIndiLinker;
4088 expandedIndiSet.clear();
4089 for (CXLinker<cint64>* expandedIndiLinkerIt = expandedIndiLinker; expandedIndiLinkerIt; expandedIndiLinkerIt = expandedIndiLinkerIt->getNext()) {
4090 cint64 expIndiId = expandedIndiLinkerIt->getData();
4091 expandedIndiSet.insert(expIndiId);
4092 }
4093 }
4094 cint64 propCutIndiId = tmpPropCutDataLinkerIt->getIndividualID();
4095
4096 mPropagationCutIndiSet.insert(propCutIndiId);
4097 cint64 assocUpdateId = tmpPropCutDataLinkerIt->getAssociationUpdateId();
4098 cint64 visitingCursor = tmpPropCutDataLinkerIt->getNeighbourPropagationCutCursor();
4099 CCacheValue propConValue = tmpPropCutDataLinkerIt->getConceptPropagationValue();
4100 bool missingNondeterministicExpansionPropagation = tmpPropCutDataLinkerIt->isMissingNondeterministicExpansionPropagation();
4101 bool allNeighbourRelationsConsiderationRequired = false;
4102 CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssocData = indiAssDataVector[propCutIndiId];
4103 if (indiAssocData->getAssociationDataUpdateId() != assocUpdateId) {
4104 allNeighbourRelationsConsiderationRequired = true;
4105 }
4106 if (propConValue.getIdentification() == 0) {
4107 allNeighbourRelationsConsiderationRequired = true;
4108 }
4109 if (!allNeighbourRelationsConsiderationRequired || !allNeighbourRelationsConsideredIndiSet.contains(propCutIndiId)) {
4110 if (allNeighbourRelationsConsiderationRequired) {
4111 if (!missingNondeterministicExpansionPropagation) {
4112 allNeighbourRelationsConsideredIndiSet.insert(propCutIndiId);
4113 }
4114
4115
4116 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* neighArray = indiAssocData->getRoleSetNeighbourArray();
4117 for (cint64 i = 0; i < neighArray->getIndexData()->getArraySize(); ++i) {
4118 bool hasNonExpanded = false;
4119 neighArray->at(i).visitNeighbourIndividualIdsFromCursor([&](cint64 neighbourId, cint64 nextCursor)->bool {
4120 if (!expandedIndiSet.contains(neighbourId)) {
4121 hasNonExpanded = true;
4122 CBackendRepresentativeMemoryCacheIndividualAssociationData* neighbourIndiAssocData = indiAssDataVector[neighbourId];
4123 // set as incompletely handled and add non-deterministic top property link
4124
4125 bool neighbourPropagationAffected = false;
4126 if (missingNondeterministicExpansionPropagation) {
4127 CBackendRepresentativeMemoryLabelCacheItem* neighbourConSetLabel = neighbourIndiAssocData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::FULL_CONCEPT_SET_LABEL);
4128 CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData* cardExtData = (CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData*)neighbourConSetLabel->getExtensionData(CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData::CARDINALITY_HASH);
4129 if (cardExtData || neighbourConSetLabel->hasNondeterministicElements()) {
4130 neighbourPropagationAffected = true;
4131 }
4132 } else {
4133 neighbourPropagationAffected = true;
4134 }
4135 if (neighbourPropagationAffected && !incompletelyUpdatingNeighbourPropCutIndiPairSet.contains(QPair<cint64, cint64>(neighbourId, propCutIndiId))) {
4136 propCutAllNeighbourRelationsInfluencingCount++;
4137 incompletelyUpdatingNeighbourPropCutIndiPairSet.insert(QPair<cint64, cint64>(neighbourId, propCutIndiId));
4138 incompletelyUpdatingNeighbourPropCutIndisHash.insertMulti(neighbourId, propCutIndiId);
4139 }
4140 }
4141 return true;
4142 }, 0);
4143 if (hasNonExpanded) {
4144 bool hasNonDetElements = neighArray->getIndexData()->getNeighbourRoleSetLabel(i)->hasNondeterministicElements();
4145 if (!hasNonDetElements) {
4146 propagationCutIndiReferredNeighbourArrayIdHash.insertMulti(propCutIndiId, i);
4147 }
4148 }
4149 }
4150
4151
4152 } else {
4153 cint64 arrayIndexId = tmpPropCutDataLinkerIt->getArrayIndex();
4154 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* neighArray = indiAssocData->getRoleSetNeighbourArray();
4155 bool hasNonExpanded = false;
4156 bool hasNonDetElements = neighArray->getIndexData()->getNeighbourRoleSetLabel(arrayIndexId)->hasNondeterministicElements();
4157 neighArray->at(arrayIndexId).visitNeighbourIndividualIdsFromCursor([&](cint64 neighbourId, cint64 nextCursor)->bool {
4158 if (!expandedIndiSet.contains(neighbourId)) {
4159 hasNonExpanded = true;
4160 CBackendRepresentativeMemoryCacheIndividualAssociationData* neighbourIndiAssocData = indiAssDataVector[neighbourId];
4161 bool neighbourPropagationAffected = false;
4162 if (hasNonDetElements) {
4163 neighbourPropagationAffected = true;
4164 ++propCutNeighbourNonDetermininisticRelationInfluencingCount;
4165 } else if (propConValue.getIdentification() != 0) {
4166 CBackendRepresentativeMemoryLabelCacheItem* neighbourConSetLabel = neighbourIndiAssocData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::FULL_CONCEPT_SET_LABEL);
4167 CBackendRepresentativeMemoryLabelValueLinker* cacheValueLinker = neighbourConSetLabel->getTagCacheValueHash(false)->value(propConValue.getTag());
4168 if (!cacheValueLinker) {
4169 // only for deterministically propagated concepts
4170 if (mConfPropagationCutPropagatedConceptDirectInstallation && (propConValue.getCacheValueIdentifier() == CCacheValue::CACHEVALTAGANDCONCEPT || propConValue.getCacheValueIdentifier() == CCacheValue::CACHEVALTAGANDNEGATEDCONCEPT)) {
4171 updatePropagationCutIndividualIncompletelyHandled(neighbourId, propConValue, ontologyData);
4172 propCutNeighbourDirectConceptInfluenceUpdatingCount++;
4173 ++propCutInfluencedIndiCount;
4174 updated = true;
4175 } else {
4176 neighbourPropagationAffected = true;
4177 propCutNeighbourRelationPropagatedConceptMissingInfluencingCount++;
4178 }
4179 }
4180 } else if (missingNondeterministicExpansionPropagation) {
4181 CBackendRepresentativeMemoryLabelCacheItem* neighbourConSetLabel = neighbourIndiAssocData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::FULL_CONCEPT_SET_LABEL);
4182 CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData* cardExtData = (CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData*)neighbourConSetLabel->getExtensionData(CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData::CARDINALITY_HASH);
4183 if (cardExtData || neighbourConSetLabel->hasNondeterministicElements()) {
4184 propCutNeighbourRelationUnspecifiedInfluencingCount++;
4185 neighbourPropagationAffected = true;
4186 }
4187 } else {
4188 propCutNeighbourRelationUnspecifiedInfluencingCount++;
4189 neighbourPropagationAffected = true;
4190 }
4191
4192 if (neighbourPropagationAffected) {
4193 // set as incompletely handled and add non-deterministic top property link
4194 if (!incompletelyUpdatingNeighbourPropCutIndiPairSet.contains(QPair<cint64, cint64>(neighbourId, propCutIndiId))) {
4195 incompletelyUpdatingNeighbourPropCutIndiPairSet.insert(QPair<cint64, cint64>(neighbourId, propCutIndiId));
4196 incompletelyUpdatingNeighbourPropCutIndisHash.insertMulti(neighbourId, propCutIndiId);
4197 }
4198 }
4199
4200 }
4201 return true;
4202 }, visitingCursor);
4203 if (!hasNonDetElements && hasNonExpanded) {
4204 propagationCutIndiReferredNeighbourArrayIdHash.insertMulti(propCutIndiId, arrayIndexId);
4205 }
4206 }
4207
4208 }
4209 }
4210
4211
4212 propCutIndiCount = mPropagationCutIndiSet.count();
4213
4214
4215 cint64 lastPropCutIndiId = -1;
4216 for (QHash<cint64, cint64>::const_iterator it = propagationCutIndiReferredNeighbourArrayIdHash.constBegin(), itEnd = propagationCutIndiReferredNeighbourArrayIdHash.constEnd(); it != itEnd; ++it) {
4217 cint64 propCutIndiId = it.key();
4218 if (lastPropCutIndiId != propCutIndiId) {
4219 lastPropCutIndiId = propCutIndiId;
4220
4221
4222
4223 QList<cint64> referredNeighbourArrayIdList = propagationCutIndiReferredNeighbourArrayIdHash.values(propCutIndiId);
4224 QSet<cint64> referredNeighbourArrayIdSet = referredNeighbourArrayIdList.toSet();
4225 CBackendRepresentativeMemoryCacheIndividualAssociationData* propCutIndiAssocData = indiAssDataVector[propCutIndiId];
4226 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* neighArray = propCutIndiAssocData->getRoleSetNeighbourArray();
4227 CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* arrayIndexData = neighArray->getIndexData();
4228 CPropagationCutNeighbourArrayHandlingData* handlingData = nullptr;
4229 for (cint64 i = 0; i < arrayIndexData->getArraySize(); ++i) {
4230 bool hasExpandableNeighbours = referredNeighbourArrayIdSet.contains(i);
4231 CBackendRepresentativeMemoryLabelCacheItem* neighbourLabelItem = neighArray->getIndexData()->getNeighbourRoleSetLabel(i);
4232 bool hasNonDetElements = neighbourLabelItem->hasNondeterministicElements();
4233 if (!hasExpandableNeighbours) {
4234 neighArray->at(i).visitNeighbourIndividualIdsFromCursor([&](cint64 neighbourId, cint64 nextCursor)->bool {
4235 if (!expandedIndiSet.contains(neighbourId)) {
4236 hasExpandableNeighbours = true;
4237 return false;
4238 }
4239 return true;
4240 }, 0);
4241 }
4242
4243
4244 if (hasExpandableNeighbours) {
4245 if (!mTmpPropCutIndiArrayNeighboursHandlingDataHash) {
4246 mTmpPropCutIndiArrayNeighboursHandlingDataHash = new QHash<cint64, CPropagationCutNeighbourArrayHandlingData *>();
4247 }
4248 if (!handlingData) {
4249 CPropagationCutNeighbourArrayHandlingData*& tmpHandlingData = (*mTmpPropCutIndiArrayNeighboursHandlingDataHash)[propCutIndiId];
4250 if (!tmpHandlingData) {
4251 tmpHandlingData = new CPropagationCutNeighbourArrayHandlingData();
4252 }
4253 handlingData = tmpHandlingData;
4254 }
4255 if (handlingData) {
4256 if (!hasNonDetElements) {
4257 handlingData->mReaddingArrayPosSet.insert(i);
4258 } else if (neighbourLabelItem->getCacheValueCount() > 1 || neighbourLabelItem->getCacheValueLinker()->getCacheValue().getTag() != 1) {
4259 bool allNondeterministic = true;
4260 for (CBackendRepresentativeMemoryLabelValueLinker* cacheValueLinkerIt = neighbourLabelItem->getCacheValueLinker(); cacheValueLinkerIt && allNondeterministic; cacheValueLinkerIt = cacheValueLinkerIt->getNext()) {
4261 if (!isCacheValueRoleNondeterministic(cacheValueLinkerIt->getCacheValue())) {
4262 allNondeterministic = false;
4263 }
4264 }
4265 if (allNondeterministic) {
4266 handlingData->mRemovalArrayPosSet.insert(i);
4267 } else {
4268 handlingData->mReductionArrayPosSet.insert(i);
4269 }
4270 }
4271 }
4272 }
4273 }
4274 }
4275 }
4276
4277
4278 if (!incompletelyUpdatingNeighbourPropCutIndisHash.isEmpty()) {
4279
4280 CBackendRepresentativeMemoryLabelCacheItem* propMarkLabelItem = ontologyData->getPrioritizedPropagationMarkedNeighbourLabelItem();
4281 CCacheValue propMarkRoleCacheValue = tmpPropCutDataLinker->getPropagationMarkingRoleValue();
4282 if (!propMarkLabelItem) {
4283 CBackendRepresentativeMemoryLabelSignatureResolveCacheItem& sigResolCacheItem = (*ontologyData->getSignatureLabelItemHash(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_LABEL))[propMarkRoleCacheValue.getTag()];
4284 for (CBackendRepresentativeMemoryLabelCacheItem* labelItemLinkerIt = sigResolCacheItem.getLabelItems(); labelItemLinkerIt && !propMarkLabelItem; labelItemLinkerIt = labelItemLinkerIt->getNext()) {
4285 if (labelItemLinkerIt->getCacheValueCount() == 1) {
4286 if (labelItemLinkerIt->getCacheValueLinker() && labelItemLinkerIt->getCacheValueLinker()->getCacheValue() == propMarkRoleCacheValue) {
4287 propMarkLabelItem = labelItemLinkerIt;
4288 }
4289 }
4290 }
4291 if (!propMarkLabelItem) {
4292 propMarkLabelItem = CObjectParameterizingAllocator< CBackendRepresentativeMemoryLabelCacheItem, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
4293 propMarkLabelItem->initCacheEntry(propMarkRoleCacheValue.getTag(), ontologyData->getNextEntryID(), CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_LABEL);
4294
4295 CCACHINGHASH<cint64, CBackendRepresentativeMemoryLabelValueLinker*>* labelValueHash = propMarkLabelItem->getTagCacheValueHash(true);
4296 CBackendRepresentativeMemoryLabelValueLinker* newValueLinker = CObjectAllocator< CBackendRepresentativeMemoryLabelValueLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
4297 newValueLinker->initLabelValueLinker(propMarkRoleCacheValue);
4298 labelValueHash->insert(propMarkRoleCacheValue.getTag(), newValueLinker);
4299 propMarkLabelItem->addCacheValueLinker(newValueLinker);
4300
4301 sigResolCacheItem.appendLabelItem(propMarkLabelItem);
4302 ontologyData->setPrioritizedPropagationMarkedNeighbourLabelItem(propMarkLabelItem);
4303 }
4304 }
4305
4306
4307 cint64 lastNeighbour = -1;
4308 for (QHash<cint64, cint64>::const_iterator it = incompletelyUpdatingNeighbourPropCutIndisHash.constBegin(), itEnd = incompletelyUpdatingNeighbourPropCutIndisHash.constEnd(); it != itEnd; ++it) {
4309 cint64 neighbourId = it.key();
4310 if (lastNeighbour != neighbourId) {
4311 lastNeighbour = neighbourId;
4312 QList<cint64> propCutIndiIds = incompletelyUpdatingNeighbourPropCutIndisHash.values(neighbourId);
4313 if (updatePropagationCutIndividualIncompletelyHandled(neighbourId, propCutIndiIds, propMarkLabelItem, ontologyData)) {
4314 ++propCutInfluencedIndiCount;
4315 updated = true;
4316 }
4317 }
4318 }
4319
4320 }
4321
4322 QString propCutIndisString = QString("%1 individuals").arg(propCutIndiCount);
4323 bool debuggingIndi = false;
4324 if (mStatCollectStatistics && propCutIndiCount < 5) {
4325 QStringList propCutIndStringList;
4326 for (cint64 propCutIndiId : mPropagationCutIndiSet) {
4327 QString indiString = QString::number(propCutIndiId);
4328 #ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
4329 CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssocData = indiAssDataVector[propCutIndiId];
4330 if (indiAssocData->mDebugIndi) {
4331 QString indiName = CIRIName::getRecentIRIName(indiAssocData->mDebugIndi->getIndividualNameLinker());
4332 indiString = QString("%2 with id %1").arg(propCutIndiId).arg(indiName);
4333 }
4334 #endif
4335 propCutIndStringList.append(indiString);
4336 }
4337 propCutIndisString += QString(" (%1)").arg(propCutIndStringList.join(", "));
4338 }
4339
4340
4341 #ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
4342 if (debuggingIndi) {
4343 QStringList modelStringList = getRepresentativeCacheString();
4344 modelStringList.append(QString("\r\n\r\nReceived propagation cut for %1 influencing %2 neighbour individuals (%3 from all relations, %4 due to non deterministic relations, %5 due to propagated concept missing, %7 direct added propagated concept, %6 unspecified)\r\n")
4345 .arg(propCutIndisString).arg(propCutInfluencedIndiCount).arg(propCutAllNeighbourRelationsInfluencingCount).arg(propCutNeighbourNonDetermininisticRelationInfluencingCount).arg(propCutNeighbourRelationPropagatedConceptMissingInfluencingCount).arg(propCutNeighbourRelationUnspecifiedInfluencingCount).arg(propCutNeighbourDirectConceptInfluenceUpdatingCount));
4346
4347 for (QHash<cint64, cint64>::const_iterator it = incompletelyUpdatingNeighbourPropCutIndisHash.constBegin(), itEnd = incompletelyUpdatingNeighbourPropCutIndisHash.constEnd(); it != itEnd; ++it) {
4348 modelStringList.append(QString("propagation cut individual %1 influenced %2\r\n").arg(it.key()).arg(it.value()));
4349 }
4350
4351 QFile modelFile(QString("./Debugging/RepresentativeCache/individual-label-association-data-%1-propagation-cut.txt").arg(mWriteDataCount));
4352 if (modelFile.open(QIODevice::WriteOnly)) {
4353 for (const QString& modelString : modelStringList) {
4354 QByteArray data = modelString.toUtf8();
4355 modelFile.write(data);
4356 }
4357 modelFile.close();
4358 }
4359 }
4360 #endif
4361
4362 LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Received propagation cut for %1 influencing %2 neighbour individuals (%3 from all relations, %4 due to non deterministic relations, %5 due to propagated concept missing, %7 direct added propagated concept, %6 unspecified).")
4363 .arg(propCutIndisString).arg(propCutInfluencedIndiCount).arg(propCutAllNeighbourRelationsInfluencingCount).arg(propCutNeighbourNonDetermininisticRelationInfluencingCount).arg(propCutNeighbourRelationPropagatedConceptMissingInfluencingCount).arg(propCutNeighbourRelationUnspecifiedInfluencingCount).arg(propCutNeighbourDirectConceptInfluenceUpdatingCount), this);
4364 }
4365 return updated;
4366 }
4367
4368
4369
4370 bool CBackendRepresentativeMemoryCache::updatePropagationCutIndividualIncompletelyHandled(cint64 neighbourId, const CCacheValue& propConValue, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
4371 CBackendRepresentativeMemoryCacheIndividualAssociationData** indiAssDataVector = ontologyData->getIndividualIdAssoiationDataVector();
4372 CBackendRepresentativeMemoryCacheIndividualAssociationData* neighbourIndiAssocData = indiAssDataVector[neighbourId];
4373
4374 CBackendRepresentativeMemoryCacheIndividualAssociationData* locNeighbourAssociationData = createLocalizedIndividualAssociationData(neighbourIndiAssocData->getAssociatedIndividualId(), neighbourIndiAssocData, ontologyData, false);
4375 CBackendRepresentativeMemoryCacheContext* context = getIndividualAssociationDataMemoryContext(locNeighbourAssociationData, ontologyData);
4376 #ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
4377 locNeighbourAssociationData->mDebugUpdateFunction = QString("updatePropagationCutIndividualIncompletelyHandled");
4378 locNeighbourAssociationData->mDebugUpdateRecomputationId = mCurrentUpdateHandlingRecomputationId;
4379 #endif
4380
4381 locNeighbourAssociationData->setNeighbourRoleSetHash(neighbourIndiAssocData->getNeighbourRoleSetHash());
4382 locNeighbourAssociationData->setRoleSetNeighbourArray(neighbourIndiAssocData->getRoleSetNeighbourArray());
4383
4384 CBackendRepresentativeMemoryLabelCacheItem* conSetLabel = neighbourIndiAssocData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::FULL_CONCEPT_SET_LABEL);
4385
4386 CBackendRepresentativeMemoryLabelCacheItem* newConSetLabel = getExtendedLabel(CBackendRepresentativeMemoryLabelCacheItem::FULL_CONCEPT_SET_LABEL, conSetLabel, propConValue, ontologyData);
4387 locNeighbourAssociationData->setLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::FULL_CONCEPT_SET_LABEL, newConSetLabel);
4388 locNeighbourAssociationData->setCacheUpdateId(mNextIndiUpdateId++);
4389 ontologyData->setMaxIndividualAssociationDataUpdateCount(qMax(locNeighbourAssociationData->getAssociationDataUpdateId(), ontologyData->getMaxIndividualAssociationDataUpdateCount()));
4390
4391 ++mUpdatedIndiCount;
4392
4393 if (locNeighbourAssociationData->isCompletelyHandled() && locNeighbourAssociationData->hasRepresentativeSameIndividualMerging()) {
4394 markRepresentativeReferencedIndividualAssociationIncompletelyHandled(neighbourId, locNeighbourAssociationData, ontologyData);
4395 } else if (locNeighbourAssociationData->isCompletelyHandled()) {
4396 locNeighbourAssociationData->setCompletelyHandled(false);
4397 storeIndividualIncompletelyMarked(locNeighbourAssociationData, true, ontologyData);
4398 }
4399 setUpdatedIndividualAssociationData(locNeighbourAssociationData->getAssociatedIndividualId(), locNeighbourAssociationData, ontologyData);
4400
4401 return true;
4402 }
4403
4404
4405
4406 bool CBackendRepresentativeMemoryCache::updatePropagationCutIndividualIncompletelyHandled(cint64 neighbourId, const QList<cint64>& propCutIndiIdList, CBackendRepresentativeMemoryLabelCacheItem* propMarkLabelItem, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
4407 CBackendRepresentativeMemoryCacheIndividualAssociationData** indiAssDataVector = ontologyData->getIndividualIdAssoiationDataVector();
4408 CBackendRepresentativeMemoryCacheIndividualAssociationData* neighbourIndiAssocData = indiAssDataVector[neighbourId];
4409 if (!neighbourIndiAssocData->hasRepresentativeSameIndividualMerging()) {
4410 CBackendRepresentativeMemoryCacheIndividualAssociationData* locNeighbourAssociationData = createLocalizedIndividualAssociationData(neighbourIndiAssocData->getAssociatedIndividualId(), neighbourIndiAssocData, ontologyData, false);
4411 bool requiresDataCopying = false;
4412 CBackendRepresentativeMemoryCacheContext* context = getIndividualAssociationDataMemoryContext(locNeighbourAssociationData, ontologyData, &requiresDataCopying);
4413 #ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
4414 locNeighbourAssociationData->mDebugUpdateFunction = QString("updatePropagationCutIndividualIncompletelyHandled");
4415 locNeighbourAssociationData->mDebugUpdateRecomputationId = mCurrentUpdateHandlingRecomputationId;
4416 #endif
4417 CBackendRepresentativeMemoryLabelCacheItem* neighbourRoleSetCompLabel = neighbourIndiAssocData->getLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL);
4418 CBackendRepresentativeMemoryLabelValueLinker* propMarkTagCacheValueLinker = neighbourRoleSetCompLabel->getTagCacheValueHash()->value(propMarkLabelItem->getCacheValueLinker()->getData().getTag());
4419 CBackendRepresentativeMemoryLabelCacheItem* newNeighbourRoleSetCompLabel = neighbourRoleSetCompLabel;
4420 if (!propMarkTagCacheValueLinker || propMarkTagCacheValueLinker->getData() != propMarkLabelItem->getCacheValueLinker()->getData()) {
4421 CCacheValue extendingCacheValue;
4422 extendingCacheValue.initCacheValue(propMarkLabelItem->getCacheEntryID(), (cint64)propMarkLabelItem, CCacheValue::CACHE_VALUE_TAG_AND_ENTRY);
4423 newNeighbourRoleSetCompLabel = getExtendedLabel(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL, neighbourRoleSetCompLabel, extendingCacheValue, ontologyData);
4424 }
4425
4426 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* newArray = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
4427 CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash* newNeighbourRoleSetHash = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
4428 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* prevArrayData = neighbourIndiAssocData->getRoleSetNeighbourArray();
4429 CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* prevArrayIndexData = prevArrayData->getIndexData();
4430 CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* newArrayIndexData = prevArrayIndexData;
4431 if (newNeighbourRoleSetCompLabel == neighbourRoleSetCompLabel) {
4432 newArray->initNeighbourArray(neighbourIndiAssocData->getRoleSetNeighbourArray());
4433 } else {
4434 locNeighbourAssociationData->setLabelCacheEntry(CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL, newNeighbourRoleSetCompLabel);
4435 updateIndexedAssociationCount(locNeighbourAssociationData, neighbourIndiAssocData, CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL, ontologyData);
4436 newArrayIndexData = getIndividualNeighbourArrayIndexExtensionData(newNeighbourRoleSetCompLabel, ontologyData);
4437 newArray->initNeighbourArray(newArrayIndexData);
4438
4439 for (cint64 i = 0; i < newArrayIndexData->getArraySize(); ++i) {
4440 CBackendRepresentativeMemoryLabelCacheItem* referredLabelCacheItem = newArrayIndexData->getNeighbourRoleSetLabel(i);
4441 cint64 prevIndex = prevArrayIndexData->getIndex(referredLabelCacheItem);
4442 if (prevIndex >= 0) {
4443 newArray->at(i) = prevArrayData->at(prevIndex);
4444 }
4445 }
4446 }
4447 QSet<cint64> existingPropCutIndiSet;
4448 cint64 propMarkIndex = newArrayIndexData->getIndex(propMarkLabelItem);
4449 newArray->at(propMarkIndex).visitNeighbourIndividualIds([&](cint64 indi)->bool {
4450 existingPropCutIndiSet.insert(indi);
4451 return true;
4452 });
4453
4454 for (cint64 propCutIndi : propCutIndiIdList) {
4455 if (!existingPropCutIndiSet.contains(propCutIndi)) {
4456 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newLinker = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
4457 ++mStatCreatedNeighbourLinks;
4458 newLinker->initIndividualIdLinker(propCutIndi);
4459 newArray->at(propMarkIndex).addIndividualIdLinker(newLinker);
4460 existingPropCutIndiSet.insert(propCutIndi);
4461 }
4462 }
4463
4464 newNeighbourRoleSetHash->initNeighbourRoleSetHash(neighbourIndiAssocData->getNeighbourRoleSetHash(), requiresDataCopying);
4465
4466 locNeighbourAssociationData->setNeighbourRoleSetHash(newNeighbourRoleSetHash);
4467 locNeighbourAssociationData->setRoleSetNeighbourArray(newArray);
4468
4469 locNeighbourAssociationData->setCacheUpdateId(mNextIndiUpdateId++);
4470 ontologyData->setMaxIndividualAssociationDataUpdateCount(qMax(locNeighbourAssociationData->getAssociationDataUpdateId(), ontologyData->getMaxIndividualAssociationDataUpdateCount()));
4471
4472 ++mUpdatedIndiCount;
4473
4474 locNeighbourAssociationData->setCompletelyHandled(false);
4475 storeIndividualIncompletelyMarked(locNeighbourAssociationData, true, ontologyData);
4476 setUpdatedIndividualAssociationData(locNeighbourAssociationData->getAssociatedIndividualId(), locNeighbourAssociationData, ontologyData);
4477
4478 return true;
4479 }
4480
4481 return false;
4482 }
4483
4484
4485
4486 bool CBackendRepresentativeMemoryCache::updateInvolvedIndividuals(CBackendRepresentativeMemoryCacheTemporaryInvolvedIndividualDataLinker* tmpInvolvedIndiDataLinker, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
4487 bool updated = false;
4488 if (tmpInvolvedIndiDataLinker) {
4489 QSet<cint64> newInvolvedIndiSet;
4490 QSet<cint64> addedInvolvedIndiStatusSet;
4491 cint64 involvedIndiCount = 0;
4492 cint64 newInvolvedIndiCount = 0;
4493 cint64 involvedIndiUpdatingCount = 0;
4494 CBackendRepresentativeMemoryCacheIndividualAssociationData** indiAssDataVector = ontologyData->getIndividualIdAssoiationDataVector();
4495 for (CXLinker<cint64>* indiLinkerIt = tmpInvolvedIndiDataLinker->getInvolvedIndividualIdsLinker(); indiLinkerIt; indiLinkerIt = indiLinkerIt->getNext()) {
4496 cint64 indiId = indiLinkerIt->getData();
4497 newInvolvedIndiSet.insert(indiId);
4498
4499 CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssocData = indiAssDataVector[indiId];
4500 bool addedInvolvedStatus = false;
4501 if (!indiAssocData->hasProblematicLevel()) {
4502 ontologyData->incInvolvedIndividualCount();
4503 addedInvolvedStatus = true;
4504 addedInvolvedIndiStatusSet.insert(indiId);
4505 ++newInvolvedIndiCount;
4506 }
4507 indiAssocData->incProblematicLevel();
4508 ++involvedIndiCount;
4509 }
4510
4511
4512 QSet<cint64> neighbourInvolvedIndiArrayUpdateSet;
4513 for (CXLinker<cint64>* indiLinkerIt = tmpInvolvedIndiDataLinker->getInvolvedIndividualIdsLinker(); indiLinkerIt; indiLinkerIt = indiLinkerIt->getNext()) {
4514 cint64 indiId = indiLinkerIt->getData();
4515 if (addedInvolvedIndiStatusSet.contains(indiId)) {
4516 CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssocData = indiAssDataVector[indiId];
4517 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* roleAssNeighArray = indiAssocData->getRoleSetNeighbourArray();
4518 CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* arrayIndexData = roleAssNeighArray->getIndexData();
4519 if (arrayIndexData) {
4520 for (cint64 i = 0; i < arrayIndexData->getArraySize(); ++i) {
4521 for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* neighbourIndiLinkerIt = roleAssNeighArray->at(i).getIndividualIdLinker(); neighbourIndiLinkerIt; neighbourIndiLinkerIt = neighbourIndiLinkerIt->getNext()) {
4522 cint64 neighbourIndiId = neighbourIndiLinkerIt->getIndividualId();
4523 neighbourInvolvedIndiArrayUpdateSet.insert(neighbourIndiId);
4524 }
4525 }
4526 }
4527 for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* neighbourIndiLinkerIt = indiAssocData->getPropagationCutRemovedNeighbourIndividualLinker(); neighbourIndiLinkerIt; neighbourIndiLinkerIt = neighbourIndiLinkerIt->getNext()) {
4528 cint64 neighbourIndiId = neighbourIndiLinkerIt->getIndividualId();
4529 neighbourInvolvedIndiArrayUpdateSet.insert(neighbourIndiId);
4530 }
4531 }
4532 }
4533
4534
4535 for (QSet<cint64>::const_iterator it = neighbourInvolvedIndiArrayUpdateSet.constBegin(), itEnd = neighbourInvolvedIndiArrayUpdateSet.constEnd(); it != itEnd; ++it) {
4536 cint64 indiId = *it;
4537 ++involvedIndiUpdatingCount;
4538
4539 CBackendRepresentativeMemoryCacheIndividualAssociationData* prevIndiAssocData = indiAssDataVector[indiId];
4540 CBackendRepresentativeMemoryCacheIndividualAssociationData* locAssociationData = createLocalizedIndividualAssociationData(prevIndiAssocData->getAssociatedIndividualId(), prevIndiAssocData, ontologyData, false);
4541 bool requiresDataCopying = false;
4542 CBackendRepresentativeMemoryCacheContext* context = getIndividualAssociationDataMemoryContext(locAssociationData, ontologyData, &requiresDataCopying);
4543 #ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
4544 locAssociationData->mDebugUpdateFunction = QString("updateInvolvedIndividuals");
4545 locAssociationData->mDebugUpdateRecomputationId = mCurrentUpdateHandlingRecomputationId;
4546 #endif
4547 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* prevArrayData = prevIndiAssocData->getRoleSetNeighbourArray();
4548 CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* prevArrayIndexData = prevArrayData->getIndexData();
4549
4550 CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* newArrayIndexData = prevArrayIndexData;
4551
4552
4553 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* newArray = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
4554 CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash* newNeighbourRoleSetHash = CObjectParameterizingAllocator< CBackendRepresentativeMemoryCacheIndividualNeighbourRoleSetHash, CBackendRepresentativeMemoryCacheContext* >::allocateAndConstructAndParameterize(context->getMemoryAllocationManager(), context);
4555 newArray->initNeighbourArray(newArrayIndexData);
4556 newNeighbourRoleSetHash->initNeighbourRoleSetHash(prevIndiAssocData->getNeighbourRoleSetHash(), requiresDataCopying);
4557
4558 for (cint64 i = 0; i < newArrayIndexData->getArraySize(); ++i) {
4559
4560 QSet<cint64> updatingInolvedNeighbourIndiSet;
4561 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* firstNewLinker = nullptr;
4562 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* lastNewLinker = nullptr;
4563
4564 prevArrayData->at(i).visitNeighbourIndividualIds([&](cint64 neighbourIndiId)->bool {
4565
4566 if (!addedInvolvedIndiStatusSet.contains(neighbourIndiId)) {
4567 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newLinker = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
4568 ++mStatCreatedNeighbourLinks;
4569 newLinker->initIndividualIdLinker(neighbourIndiId);
4570 if (lastNewLinker) {
4571 lastNewLinker->append(newLinker);
4572 lastNewLinker = newLinker;
4573 } else {
4574 firstNewLinker = newLinker;
4575 lastNewLinker = newLinker;
4576 }
4577 } else {
4578 updatingInolvedNeighbourIndiSet.insert(neighbourIndiId);
4579 }
4580 return true;
4581 });
4582
4583
4584 // add problematic/involved neighbour ids to the beginning
4585 for (QSet<cint64>::const_iterator updateIt = updatingInolvedNeighbourIndiSet.constBegin(), updateItEnd = updatingInolvedNeighbourIndiSet.constEnd(); updateIt != updateItEnd; ++updateIt) {
4586 cint64 updatingIndiId = *updateIt;
4587
4588 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newLinker = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
4589 ++mStatCreatedNeighbourLinks;
4590 newLinker->initIndividualIdLinker(updatingIndiId);
4591 firstNewLinker = newLinker->append(firstNewLinker);
4592 }
4593 if (firstNewLinker) {
4594 newArray->at(i).addIndividualIdLinker(firstNewLinker);
4595 }
4596
4597 }
4598
4599 locAssociationData->setNeighbourRoleSetHash(newNeighbourRoleSetHash);
4600 locAssociationData->setRoleSetNeighbourArray(newArray);
4601
4602 locAssociationData->setProblematicLeveledNeigbour(true);
4603
4604 locAssociationData->setCacheUpdateId(mNextIndiUpdateId++);
4605 ontologyData->setMaxIndividualAssociationDataUpdateCount(qMax(locAssociationData->getAssociationDataUpdateId(), ontologyData->getMaxIndividualAssociationDataUpdateCount()));
4606
4607 ++mUpdatedIndiCount;
4608
4609 storeIndividualIncompletelyMarked(locAssociationData, !locAssociationData->isCompletelyHandled(), ontologyData);
4610 setUpdatedIndividualAssociationData(locAssociationData->getAssociatedIndividualId(), locAssociationData, ontologyData);
4611
4612 updated = true;
4613 }
4614
4615 LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Received clash influence for %1 individuals (new for %2 individuals), reordered neighbours for %3 individuals.").arg(involvedIndiCount).arg(newInvolvedIndiCount).arg(involvedIndiUpdatingCount), this);
4616
4617 }
4618 return updated;
4619 }
4620
4621
4622
4623
4624 void CBackendRepresentativeMemoryCache::writeStringifiedRepresentativeCacheToFile() {
4625 QStringList modelStringList = getRepresentativeCacheString();
4626 QFile modelFileLatest("./Debugging/RepresentativeCache/individual-label-association-data.txt");
4627 if (modelFileLatest.open(QIODevice::WriteOnly)) {
4628 for (const QString& modelString : modelStringList) {
4629 QByteArray data = modelString.toUtf8();
4630 modelFileLatest.write(data);
4631 }
4632 modelFileLatest.close();
4633 }
4634 QFile modelFile(QString("./Debugging/RepresentativeCache/individual-label-association-data-%1.txt").arg(mWriteDataCount));
4635 if (modelFile.open(QIODevice::WriteOnly)) {
4636 for (const QString& modelString : modelStringList) {
4637 QByteArray data = modelString.toUtf8();
4638 modelFile.write(data);
4639 }
4640 modelFile.close();
4641 }
4642
4643 #ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
4644 QFile handlingFile(QString("./Debugging/RepresentativeCache/handling-data.txt"));
4645 if (handlingFile.open(QIODevice::WriteOnly)) {
4646 for (const QString& modelString : mDebugHandlingStrings) {
4647 QByteArray data = modelString.toUtf8();
4648 handlingFile.write(data);
4649 handlingFile.write(QString("\r\n").toUtf8());
4650 }
4651 handlingFile.close();
4652 }
4653 #endif
4654 }
4655
4656
4657
4658 QString CBackendRepresentativeMemoryCache::getRepresentativeCacheLabelItemString(cint64 labelType) {
4659 QString labelTypeString;
4660 if (labelType == CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_CONCEPT_SET_LABEL) {
4661 labelTypeString = "DETERMINISTIC_CONCEPT_SET_LABEL";
4662 }
4663 else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::NONDETERMINISTIC_CONCEPT_SET_LABEL) {
4664 labelTypeString = "NONDETERMINISTIC_CONCEPT_SET_LABEL";
4665 }
4666 else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::FULL_CONCEPT_SET_LABEL) {
4667 labelTypeString = "FULL_CONCEPT_SET_LABEL";
4668 }
4669 else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_COMBINED_EXISTENTIAL_INSTANTIATED_ROLE_SET_LABEL) {
4670 labelTypeString = "DETERMINISTIC_COMBINED_EXISTENTIAL_INSTANTIATED_ROLE_SET_LABEL";
4671 }
4672 else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::NONDETERMINISTIC_COMBINED_EXISTENTIAL_INSTANTIATED_ROLE_SET_LABEL) {
4673 labelTypeString = "NONDETERMINISTIC_COMBINED_EXISTENTIAL_INSTANTIATED_ROLE_SET_LABEL";
4674 }
4675 else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_COMBINED_NEIGHBOUR_INSTANTIATED_ROLE_SET_LABEL) {
4676 labelTypeString = "DETERMINISTIC_COMBINED_NEIGHBOUR_INSTANTIATED_ROLE_SET_LABEL";
4677 }
4678 else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::NONDETERMINISTIC_COMBINED_NEIGHBOUR_INSTANTIATED_ROLE_SET_LABEL) {
4679 labelTypeString = "NONDETERMINISTIC_COMBINED_NEIGHBOUR_INSTANTIATED_ROLE_SET_LABEL";
4680 }
4681 else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL) {
4682 labelTypeString = "NEIGHBOUR_INSTANTIATED_ROLE_SET_COMBINATION_LABEL";
4683 }
4684 else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL) {
4685 labelTypeString = "DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL";
4686 }
4687 else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::NONDETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL) {
4688 labelTypeString = "NONDETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL";
4689 }
4690 else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_DIFFRENT_INDIVIDUAL_SET_LABEL) {
4691 labelTypeString = "DETERMINISTIC_DIFFRENT_INDIVIDUAL_SET_LABEL";
4692 }
4693 else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::NONDETERMINISTIC_DIFFRENT_INDIVIDUAL_SET_LABEL) {
4694 labelTypeString = "NONDETERMINISTIC_DIFFRENT_INDIVIDUAL_SET_LABEL";
4695 }
4696 else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::INDIRECTLY_CONNECTED_NOMINAL_INDIVIDUAL_SET_LABEL) {
4697 labelTypeString = "INDIRECTLY_CONNECTED_NOMINAL_INDIVIDUAL_SET_LABEL";
4698 }
4699 else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::NEIGHBOUR_INSTANTIATED_ROLE_SET_LABEL) {
4700 labelTypeString = "NEIGHBOUR_INSTANTIATED_ROLE_SET_LABEL";
4701 }
4702 else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_COMBINED_DATA_INSTANTIATED_ROLE_SET_LABEL) {
4703 labelTypeString = "DETERMINISTIC_COMBINED_DATA_INSTANTIATED_ROLE_SET_LABEL";
4704 }
4705 else if (labelType == CBackendRepresentativeMemoryLabelCacheItem::NONDETERMINISTIC_COMBINED_DATA_INSTANTIATED_ROLE_SET_LABEL) {
4706 labelTypeString = "NONDETERMINISTIC_COMBINED_DATA_INSTANTIATED_ROLE_SET_LABEL";
4707 }
4708 return labelTypeString;
4709 }
4710
4711 QStringList CBackendRepresentativeMemoryCache::getRepresentativeCacheString() {
4712 QStringList cacheStringList;
4713 for (CCACHINGHASH<cint64, CBackendRepresentativeMemoryCacheOntologyData*>::const_iterator it = mOntologyIdentifierDataHash->constBegin(), itEnd = mOntologyIdentifierDataHash->constEnd(); it != itEnd; ++it) {
4714 cint64 ontId = it.key();
4715 CBackendRepresentativeMemoryCacheOntologyData* ontologyData = it.value();
4716 cacheStringList.append(QString("Ontology: %1\r\n\r\n\r\n").arg(ontId));
4717 QStringList ontologyIdDataStringList = getRepresentativeCacheString(ontologyData);
4718 cacheStringList += ontologyIdDataStringList;
4719 cacheStringList.append(QString("\r\n\r\n\r\n\r\n\r\n\r\n"));
4720 }
4721 return cacheStringList;
4722 }
4723
4724
4725
4726
4727 void CBackendRepresentativeMemoryCache::debugCheckWriteMaxSameMergedIndividualsToFile(CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
4728
4729 CBackendRepresentativeMemoryLabelCacheItem* maxSameMergedItem = nullptr;
4730 cint64 maxSameMergedCount = 0;
4731
4732 CCACHINGHASH<cint64, CBackendRepresentativeMemoryLabelSignatureResolveCacheItem>* sigLabelItemHash = ontologyData->getSignatureLabelItemHash(CBackendRepresentativeMemoryLabelCacheItem::DETERMINISTIC_SAME_INDIVIDUAL_SET_LABEL);
4733 for (CCACHINGHASH<cint64, CBackendRepresentativeMemoryLabelSignatureResolveCacheItem>::const_iterator it = sigLabelItemHash->constBegin(), itEnd = sigLabelItemHash->constEnd(); it != itEnd; ++it) {
4734
4735 CBackendRepresentativeMemoryLabelSignatureResolveCacheItem sigResolveItem = it.value();
4736 for (CBackendRepresentativeMemoryLabelCacheItem* labelItemLinkerIt = sigResolveItem.getLabelItems(); labelItemLinkerIt; labelItemLinkerIt = labelItemLinkerIt->getNext()) {
4737 cint64 itemId = labelItemLinkerIt->getCacheEntryID();
4738 if (labelItemLinkerIt->getCacheValueCount() > maxSameMergedCount) {
4739 maxSameMergedCount = labelItemLinkerIt->getCacheValueCount();
4740 maxSameMergedItem = labelItemLinkerIt;
4741 }
4742 }
4743
4744 }
4745
4746
4747 if (maxSameMergedItem) {
4748 QSet<QString> indiSet;
4749 CBackendRepresentativeMemoryLabelValueLinker* labelValueLinker = maxSameMergedItem->getCacheValueLinker();
4750 for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = labelValueLinker; labelValueLinkerIt; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
4751 CCacheValue cacheValue = labelValueLinkerIt->getCacheValue();
4752 cint64 tag = cacheValue.getTag();
4753 CIndividual* individual = (CIndividual*)mDebugOntology->getABox()->getIndividualVector()->getData(tag);
4754 QString indiName = CIRIName::getRecentIRIName(individual->getNameLinker());
4755 indiSet.insert(indiName);
4756 }
4757
4758
4759
4760
4761
4762 QSet<QString> indi1Set = indiSet;
4763
4764 QSet<QString> indi2Set;
4765 QFile file2("./Debugging/RepresentativeCache/max-deterministically-merged-data.txt");
4766 cint64 count2 = 0;
4767 if (file2.open(QIODevice::ReadOnly)) {
4768 while (!file2.atEnd()) {
4769 QString line = file2.readLine().trimmed();
4770 indi2Set.insert(line);
4771 }
4772 }
4773 file2.close();
4774
4775 for (QString indi2 : indi2Set) {
4776 if (!indi1Set.contains(indi2)) {
4777 bool debug = true;
4778 }
4779 }
4780
4781 for (QString indi1 : indi1Set) {
4782 if (!indi2Set.contains(indi1)) {
4783 bool debug = true;
4784 }
4785 }
4786
4787 bool writeFile = false;
4788 if (writeFile) {
4789 QFile sameAsMergedFile("./Debugging/RepresentativeCache/max-deterministically-merged-data.txt");
4790 if (sameAsMergedFile.open(QIODevice::WriteOnly)) {
4791 for (const QString& indiString : indiSet) {
4792 QByteArray data = indiString.toUtf8();
4793 sameAsMergedFile.write(data);
4794 sameAsMergedFile.write("\r\n");
4795 }
4796 }
4797 sameAsMergedFile.close();
4798 }
4799
4800
4801
4802 }
4803
4804 }
4805
4806
4807 QStringList CBackendRepresentativeMemoryCache::getRepresentativeCacheString(CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
4808 QStringList cacheStringList;
4809 QStringList labelStringList;
4810 for (cint64 labelType = 0; labelType < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_TYPE_COUNT; ++labelType) {
4811 QString labelTypeString = getRepresentativeCacheLabelItemString(labelType);
4812 labelTypeString = QString("\r\n\r\n%1:\r\n").arg(labelTypeString);
4813 CCACHINGHASH<cint64, CBackendRepresentativeMemoryLabelSignatureResolveCacheItem>* sigLabelItemHash = ontologyData->getSignatureLabelItemHash(labelType);
4814 for (CCACHINGHASH<cint64, CBackendRepresentativeMemoryLabelSignatureResolveCacheItem>::const_iterator it = sigLabelItemHash->constBegin(), itEnd = sigLabelItemHash->constEnd(); it != itEnd; ++it) {
4815 cint64 signature = it.key();
4816 CBackendRepresentativeMemoryLabelSignatureResolveCacheItem sigResolveItem = it.value();
4817 for (CBackendRepresentativeMemoryLabelCacheItem* labelItemLinkerIt = sigResolveItem.getLabelItems(); labelItemLinkerIt; labelItemLinkerIt = labelItemLinkerIt->getNext()) {
4818 cint64 itemId = labelItemLinkerIt->getCacheEntryID();
4819 QString labelString = QString("Label %1 (with signature %2, association count %3): ").arg(itemId).arg(signature).arg(labelItemLinkerIt->getIndividualAssociationCount());
4820 CBackendRepresentativeMemoryLabelValueLinker* labelValueLinker = labelItemLinkerIt->getCacheValueLinker();
4821 for (CBackendRepresentativeMemoryLabelValueLinker* labelValueLinkerIt = labelValueLinker; labelValueLinkerIt; labelValueLinkerIt = labelValueLinkerIt->getNext()) {
4822 CCacheValue cacheValue = labelValueLinkerIt->getCacheValue();
4823 if (labelValueLinkerIt != labelValueLinker) {
4824 labelString += ", ";
4825 }
4826 labelString += QString("%1(%2)").arg(cacheValue.getTag()).arg((cint64)cacheValue.getCacheValueIdentifier());
4827 }
4828 if (labelType == CBackendRepresentativeMemoryLabelCacheItem::FULL_CONCEPT_SET_LABEL) {
4829 CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData* cardExtData = (CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData*)labelItemLinkerIt->getExtensionData(CBackendRepresentativeMemoryLabelCacheItemCardinalityExtensionData::CARDINALITY_HASH);
4830 if (cardExtData) {
4831 QString cardDataString;
4832 CCACHINGHASH<cint64, CBackendRepresentativeMemoryLabelCacheItemCardinalityData*>* roleTagCardDataHash = cardExtData->getRoleCardinalityDataHash();
4833 for (CCACHINGHASH<cint64, CBackendRepresentativeMemoryLabelCacheItemCardinalityData*>::const_iterator it = roleTagCardDataHash->constBegin(), itEnd = roleTagCardDataHash->constEnd(); it != itEnd; ++it) {
4834 cint64 roleTag = it.key();
4835 CBackendRepresentativeMemoryLabelCacheItemCardinalityData* cardData = it.value();
4836 if (!cardDataString.isEmpty()) {
4837 cardDataString += "; ";
4838 }
4839 cardDataString += QString("%1 -> # %2 with <= %3").arg(roleTag).arg(cardData->getExistentialMaxUsedCardinality()).arg(cardData->getMinimumRestrictingCardinality());
4840 }
4841 labelString += QString("\r\n\t\t%1").arg(cardDataString);
4842 }
4843 }
4844
4845
4846
4847
4848 labelTypeString += QString("%1\r\n").arg(labelString);
4849 }
4850 }
4851 labelStringList.append(labelTypeString);
4852 }
4853
4854
4855 QStringList indiStringList;
4856 for (cint64 i = 0; i <= ontologyData->getMaxStoredIndvidualiId() && i < ontologyData->getIndividualIdAssoiationDataVectorSize(); ++i) {
4857 //for (CCACHINGHASH<cint64, CBackendRepresentativeMemoryCacheIndividualAssociationData*>::const_iterator it = mIndiIdAssoDataHash->constBegin(), itEnd = mIndiIdAssoDataHash->constEnd(); it != itEnd; ++it) {
4858 cint64 indiId = i;
4859 CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssData = ontologyData->getIndividualIdAssoiationDataVector()[indiId];
4860 if (indiAssData) {
4861 //QString indiName = CIRIName::getRecentIRIName(indiAssData->mDebugIndi->getNameLinker());
4862 //if (i < 100 || indiName == "http://semantics.crl.ibm.com/univ-bench-dl.owl#Tennis" || indiName == "http://semantics.crl.ibm.com/univ-bench-dl.owl#BasketBall" || indiName == "http://semantics.crl.ibm.com/univ-bench-dl.owl#Baseball"
4863 // || indiName == "http://www.Department1.University0.edu/StudentWhoLikesTennisButDislikesBasketBallAndBaseBall" || indiName == "http://www.Department1.University0.edu/StudentWhoLikesBasketBallButDislikesBaseBall"
4864 // || indiName == "http://www.Department1.University0.edu/StudentWhoLikesTennisAndBasketBallAndBaseBallAndShouldHaveManyHobbies" || mDebugIndiId == i) {
4865
4866 if (indiAssData && indiAssData->getAssociationDataUpdateId() >= 0) {
4867 QString indiAssString = getIndividualAssociationDebugString(indiId, indiAssData, ontologyData, 0);
4868 indiStringList.append(indiAssString);
4869
4870 cint64 prevCount = 0;
4871 CBackendRepresentativeMemoryCacheIndividualAssociationData* prevIndiAssData = indiAssData->getPreviousData();
4872 while (prevIndiAssData && !indiAssData->getIndividualAssociationMemoryContext() && prevCount <= 10) {
4873 QString prevIndiAssString = getIndividualAssociationDebugString(indiId, prevIndiAssData, ontologyData, ++prevCount);
4874 indiStringList.append(prevIndiAssString);
4875
4876 prevIndiAssData = prevIndiAssData->getPreviousData();
4877 }
4878 }
4879 //}
4880 }
4881
4882 }
4883
4884 QString statusString = QString("\tcache writes: %1,\r\n\ttotal association changes: %2,\r\n\tincompletely handled individuals: %3,\r\n\tincompatible association changes: %4,\r\n\tincompatible association checks: %5,\r\n\tmax individual association changes: %6\r\n")
4885 .arg(mWriteDataCount).arg(mNextIndiUpdateId).arg(ontologyData->getIncompletelyHandledIndividualIdCount()).arg(mUpdateIncompatibleIndiCount).arg(mCheckIncompatibleIndiCount).arg(ontologyData->getMaxIndividualAssociationDataUpdateCount());
4886
4887 cacheStringList.append(QString("STATISTICS:\r\n\r\n%1\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n").arg(statusString));
4888 cacheStringList.append(QString("REPRESENATIVE LABELS:\r\n\r\n"));
4889 for (const QString& stringLine : labelStringList) {
4890 cacheStringList.append(QString("%1\r\n").arg(stringLine));
4891 }
4892 cacheStringList.append(QString("\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\nINDIVIDUAL LABEL ASSOCIATIONS:\r\n"));
4893 for (const QString& stringLine : indiStringList) {
4894 cacheStringList.append(QString("%1\r\n").arg(stringLine));
4895 }
4896
4897 cacheStringList.append(QString("\r\n"));
4898
4899 return cacheStringList;
4900 }
4901
4902
4903 QString CBackendRepresentativeMemoryCache::getIndividualAssociationDebugString(cint64 indiId, CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssData, CBackendRepresentativeMemoryCacheOntologyData* ontologyData, cint64 prevTab) {
4904 QString prevTabString;
4905 for (cint64 i = 0; i < prevTab; ++i) {
4906 prevTabString += QString("\t");
4907 }
4908 QString indiAssString = QString("\r\nIndividual %1:\r\n").arg(indiId);
4909 #ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
4910 if (indiAssData->mDebugIndi) {
4911 QString indiName = CIRIName::getRecentIRIName(indiAssData->mDebugIndi->getIndividualNameLinker());
4912 indiAssString = QString("\r\nIndividual (%2) %1:\r\n").arg(indiId).arg(indiName);
4913 }
4914
4915 #endif
4916 if (prevTab > 0) {
4917 indiAssString = QString("\r\n%3Individual %1 previous entry %2:\r\n").arg(indiId).arg(prevTab).arg(prevTabString);
4918 }
4919 prevTabString += QString("\t");
4920
4921
4922 QString updateString = QString("%5update ids: individual association change: %1, total association change: %2, total touch: %3, \r\n%5\t\tlast indirectly connected individuals integration: %4").arg(indiAssData->getAssociationDataUpdateId()).arg(indiAssData->getCacheUpdateId()).arg(indiAssData->getCacheTouchId()).arg(indiAssData->getLastIntegratedIndirectlyConnectedIndividualsChangeId()).arg(prevTabString);
4923 #ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
4924 updateString += QString(", update function: %2, depending recomputation id: %1").arg(indiAssData->mDebugUpdateRecomputationId).arg(indiAssData->mDebugUpdateFunction);
4925 #endif
4926 updateString += QString("\r\n");
4927
4928 indiAssString += updateString;
4929
4930
4931 QStringList statusStrings;
4932 if (indiAssData->isCompletelySaturated()) {
4933 statusStrings += "completely saturated";
4934 } else {
4935 statusStrings += "incompletely saturated";
4936 }
4937
4938 if (indiAssData->isCompletelyPropagated()) {
4939 statusStrings += "completely propagated";
4940 } else {
4941 statusStrings += "incompletely propagated";
4942 }
4943
4944 if (indiAssData->isCompletelyHandled()) {
4945 statusStrings += "completely handled";
4946 } else {
4947 statusStrings += "incompletely handled";
4948 }
4949
4950 if (indiAssData->hasIndirectlyConnectedIndividualIntegration()) {
4951 statusStrings += "indirectly connected individuals integration";
4952 }
4953
4954
4955 if (indiAssData->isIndirectlyConnectedNominalIndividual()) {
4956 statusStrings += "indirectly connected nominal";
4957 }
4958
4959 if (indiAssData->hasProblematicLevel()) {
4960 statusStrings += "self clashing involved";
4961 }
4962
4963 if (indiAssData->hasProblematicLeveledNeigbour()) {
4964 statusStrings += "neighbour clashing involved";
4965 }
4966
4967 #ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
4968 if (indiAssData->mDebugSameIndividualsCompletion) {
4969 statusStrings += "deterministic same individuals completion";
4970 }
4971 if (indiAssData->mDebugNeighbourCompletionPerformed) {
4972 statusStrings += "neighbours same individuals completed";
4973 }
4974 if (indiAssData->mDebugNeighbourCompletionRequested) {
4975 statusStrings += "neighbours same individuals completion requested";
4976 }
4977
4978 if (indiAssData->mDebugNeighbourCompletionExtended) {
4979 statusStrings += "neighbours same individuals extended";
4980 }
4981 #endif
4982
4983
4984 QString statusString = QString("%2status: %1\r\n").arg(statusStrings.join(", ")).arg(prevTabString);
4985 indiAssString += statusString;
4986
4987
4988 if (indiAssData->getRepresentativeSameIndividualId() != indiAssData->getAssociatedIndividualId()) {
4989 indiAssString += QString("%2representative same individual id: %1\r\n").arg(indiAssData->getRepresentativeSameIndividualId()).arg(prevTabString);
4990 }
4991 if (indiAssData->getDeterministicSameIndividualId() != indiAssData->getAssociatedIndividualId()) {
4992 indiAssString += QString("%2deterministic merged same individual id: %1\r\n").arg(indiAssData->getDeterministicSameIndividualId()).arg(prevTabString);
4993 }
4994 if (indiAssData->getDeterministicMergedSameConsideredLabelCacheEntry()) {
4995 indiAssString += QString("%2deterministic merged same considered individuals label: %1\r\n").arg(indiAssData->getDeterministicMergedSameConsideredLabelCacheEntry()->getCacheEntryID()).arg(prevTabString);
4996 }
4997
4998
4999 for (cint64 labelType = 0; labelType < CBackendRepresentativeMemoryLabelCacheItem::LABEL_CACHE_ITEM_ASSOCIATABLE_TYPE_COUNT; ++labelType) {
5000 CBackendRepresentativeMemoryLabelCacheItem* labelItem = indiAssData->getLabelCacheEntry(labelType);
5001 if (labelItem) {
5002 cint64 itemId = labelItem->getCacheEntryID();
5003 QString labelTypeString = getRepresentativeCacheLabelItemString(labelType);
5004 QString indiLabelAssString = QString("%3associated %1 label: %2\r\n").arg(labelTypeString).arg(itemId).arg(prevTabString);
5005 indiAssString += indiLabelAssString;
5006 }
5007 }
5008
5009
5010 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* roleSetNeighbourArray = indiAssData->getRoleSetNeighbourArray();
5011 if (roleSetNeighbourArray) {
5012 CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* indexData = roleSetNeighbourArray->getIndexData();
5013 for (cint64 i = 0; i < indexData->getArraySize(); ++i) {
5014 CBackendRepresentativeMemoryLabelCacheItem* neighbourRoleSetLabelItem = indexData->getNeighbourRoleSetLabel(i);
5015 QString roleSetNeighbourIndiIdsString;
5016 roleSetNeighbourArray->at(i).visitNeighbourIndividualIds([&](cint64 id)->bool {
5017 if (!roleSetNeighbourIndiIdsString.isEmpty()) {
5018 roleSetNeighbourIndiIdsString += ", ";
5019 }
5020 roleSetNeighbourIndiIdsString += QString::number(id);
5021 CBackendRepresentativeMemoryCacheIndividualAssociationData* neighIndiAssData = ontologyData->getIndividualIdAssoiationDataVector()[id];
5022 if (neighIndiAssData && neighIndiAssData->hasProblematicLevel()) {
5023 roleSetNeighbourIndiIdsString += "(ci)";
5024 }
5025 if (neighbourRoleSetLabelItem->getCacheValueCount() == 1 && neighbourRoleSetLabelItem->getCacheValueLinker()->getData().getTag() == 1) {
5026 roleSetNeighbourIndiIdsString += "(pc)";
5027 }
5028 return true;
5029 });
5030 QString indiArrayAssString = QString("%4neighbouring ids for label %1 (%3): %2\r\n").arg(neighbourRoleSetLabelItem->getCacheEntryID()).arg(roleSetNeighbourIndiIdsString).arg(roleSetNeighbourArray->at(i).getIndividualCount()).arg(prevTabString);
5031 indiAssString += indiArrayAssString;
5032 }
5033 }
5034
5035 CBackendRepresentativeMemoryCacheNominalIndividualIndirectConnectionData* indConnData = ontologyData->getNominaIIndividualdIndirectConnectionDataHash()->value(indiId);
5036 if (indConnData) {
5037 QStringList indConnIndiIds;
5038 for (CXLinker<cint64>* indiLinkerIt = indConnData->getIndirectlyConnectedIndividualIdLinker(); indiLinkerIt; indiLinkerIt = indiLinkerIt->getNext()) {
5039 cint64 indiId = indiLinkerIt->getData();
5040 indConnIndiIds.append(QString::number(indiId));
5041 }
5042 QString indiIndConnDatatring = QString("%3indirectly connected individuals with change id %1: %2\r\n").arg(indConnData->getLastChangeId()).arg(indConnIndiIds.join(", ")).arg(prevTabString);
5043 indiAssString += indiIndConnDatatring;
5044 }
5045
5046 CBackendRepresentativeMemoryCacheIndividualAssociationContext* memManCon = indiAssData->getIndividualAssociationMemoryContext();
5047 if (memManCon) {
5048 QString indiIndMemManCon = QString("memory management context: %1\r\n").arg((cint64)memManCon);
5049 indiAssString += indiIndMemManCon;
5050 }
5051
5052
5053 return indiAssString;
5054 }
5055
5056
5057
5058
5059
5060
5061
5062 bool CBackendRepresentativeMemoryCache::debugCheckPropagationCutRemainingNeighbours(CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssData, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
5063 bool error = false;
5064 #ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
5065 if (indiAssData && indiAssData->mDebugIndi && ontologyData->isFirstIncompletelyHandledIndividualsRetrieved()) {
5066
5067 QHash<cint64, cint64> prevNeighbourIndiArrayPosHash;
5068 CBackendRepresentativeMemoryCacheIndividualAssociationData* prevIndiAssData = indiAssData->getPreviousData();
5069 if (prevIndiAssData) {
5070
5071 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* roleSetNeighbourArray = prevIndiAssData->getRoleSetNeighbourArray();
5072 if (roleSetNeighbourArray) {
5073 CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* indexData = roleSetNeighbourArray->getIndexData();
5074 for (cint64 i = 0; i < indexData->getArraySize(); ++i) {
5075
5076 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiLinker = roleSetNeighbourArray->at(i).getIndividualIdLinker();
5077 bool firstArrayPosNeighbour = true;
5078 for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiLinkerIt = indiLinker; indiLinkerIt; indiLinkerIt = indiLinkerIt->getNext()) {
5079 cint64 neighbourIndiId = indiLinkerIt->getIndividualId();
5080 prevNeighbourIndiArrayPosHash.insert(neighbourIndiId, i);
5081 }
5082 }
5083 }
5084 }
5085
5086 QHash<cint64,cint64> newNeighbourIndiArrayPosHash;
5087 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* roleSetNeighbourArray = indiAssData->getRoleSetNeighbourArray();
5088 if (roleSetNeighbourArray) {
5089 CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* indexData = roleSetNeighbourArray->getIndexData();
5090 for (cint64 i = 0; i < indexData->getArraySize(); ++i) {
5091
5092 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiLinker = roleSetNeighbourArray->at(i).getIndividualIdLinker();
5093 for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiLinkerIt = indiLinker; indiLinkerIt; indiLinkerIt = indiLinkerIt->getNext()) {
5094 cint64 neighbourIndiId = indiLinkerIt->getIndividualId();
5095 newNeighbourIndiArrayPosHash.insert(neighbourIndiId, i);
5096 if (!prevNeighbourIndiArrayPosHash.contains(neighbourIndiId)) {
5097 bool debug = true;
5098 }
5099 }
5100 }
5101
5102 for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiLinkerIt = indiAssData->getPropagationCutRemovedNeighbourIndividualLinker(); indiLinkerIt; indiLinkerIt = indiLinkerIt->getNext()) {
5103 cint64 neighbourIndiId = indiLinkerIt->getIndividualId();
5104 newNeighbourIndiArrayPosHash.insert(neighbourIndiId, -1);
5105 if (!prevNeighbourIndiArrayPosHash.contains(neighbourIndiId)) {
5106 bool debug = true;
5107 error = true;
5108 }
5109 }
5110
5111 }
5112
5113
5114 for (QHash<cint64, cint64>::const_iterator it = prevNeighbourIndiArrayPosHash.constBegin(), itEnd = prevNeighbourIndiArrayPosHash.constEnd(); it != itEnd; ++it) {
5115 cint64 prevIndiId = it.key();
5116 cint64 prevIndiArrayPos = it.value();
5117 if (!newNeighbourIndiArrayPosHash.contains(prevIndiId)) {
5118 bool debug = true;
5119 error = true;
5120 }
5121 }
5122 }
5123 #endif
5124 return error;
5125 }
5126
5127
5128 bool CBackendRepresentativeMemoryCache::debugCheckSingleNeighbourInvolvedOccurrences(CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssData, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
5129 bool error = false;
5130 #ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
5131 if (indiAssData && indiAssData->mDebugIndi && ontologyData->isFirstIncompletelyHandledIndividualsRetrieved()) {
5132 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* roleSetNeighbourArray = indiAssData->getRoleSetNeighbourArray();
5133 auto hash = indiAssData->getNeighbourRoleSetHash();
5134 QSet<cint64> neighbourIndiSet;
5135 if (roleSetNeighbourArray) {
5136 CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* indexData = roleSetNeighbourArray->getIndexData();
5137 for (cint64 i = 0; i < indexData->getArraySize(); ++i) {
5138
5139 auto label = indexData->getNeighbourRoleSetLabel(i);
5140 if (label->getCacheValueCount() > 1 || label->getCacheValueLinker()->getCacheValue().getTag() != 1) {
5141 auto& data = roleSetNeighbourArray->at(i);
5142 CBackendRepresentativeMemoryLabelCacheItem* neighbourRoleSetLabelItem = indexData->getNeighbourRoleSetLabel(i);
5143 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiLinker = roleSetNeighbourArray->at(i).getIndividualIdLinker();
5144 bool firstArrayPosNeighbour = true;
5145 for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiLinkerIt = indiLinker; indiLinkerIt; indiLinkerIt = indiLinkerIt->getNext()) {
5146 cint64 neighbourIndiId = indiLinkerIt->getIndividualId();
5147 auto neighbourLabel = hash->getNeighbourRoleSetLabel(neighbourIndiId);
5148 if (neighbourLabel != label) {
5149 bool debug = true;
5150 mDebugIndiId = indiAssData->getAssociatedIndividualId();
5151 error = true;
5152 }
5153 if (neighbourIndiSet.contains(neighbourIndiId)) {
5154 bool debug = true;
5155 mDebugIndiId = indiAssData->getAssociatedIndividualId();
5156 error = true;
5157 }
5158 neighbourIndiSet.insert(neighbourIndiId);
5159 CBackendRepresentativeMemoryCacheIndividualAssociationData* neighIndiAssData = ontologyData->getIndividualIdAssoiationDataVector()[neighbourIndiId];
5160 if (neighIndiAssData && neighIndiAssData->hasProblematicLevel()) {
5161 if (!firstArrayPosNeighbour) {
5162 bool debug = true;
5163 mDebugIndiId = indiAssData->getAssociatedIndividualId();
5164 error = true;
5165 }
5166 } else {
5167 firstArrayPosNeighbour = false;
5168 }
5169 }
5170 }
5171 }
5172
5173 }
5174 }
5175 #endif
5176 return error;
5177 }
5178
5179
5180
5181
5182
5183 bool CBackendRepresentativeMemoryCache::debugCheckCorrectNeighbourOccurrence(CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssData, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
5184 bool missingOccurrence = false;
5185 #ifdef KONCLUDE_CACHE_DEBUGGING_DATA_GENERATION
5186 if (indiAssData && indiAssData->mDebugIndi && ontologyData->isFirstIncompletelyHandledIndividualsRetrieved()) {
5187 QString indiName = CIRIName::getRecentIRIName(indiAssData->mDebugIndi->getNameLinker());
5188 if (indiName == "http://www.Department1.University0.edu/StudentWhoLikesTennisButDislikesBasketBallAndBaseBall") {
5189 s1 = true;
5190 }
5191 if (indiName == "http://www.Department1.University0.edu/StudentWhoLikesTennisAndBasketBallAndBaseBallAndShouldHaveManyHobbies") {
5192 s2 = true;
5193 }
5194 if (indiName == "http://www.Department1.University0.edu/StudentWhoLikesBasketBallButDislikesBaseBall") {
5195 s3 = true;
5196 }
5197 if (indiName == "http://semantics.crl.ibm.com/univ-bench-dl.owl#Tennis" || indiName == "http://semantics.crl.ibm.com/univ-bench-dl.owl#BasketBall" || indiName == "http://semantics.crl.ibm.com/univ-bench-dl.owl#Baseball") {
5198 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* roleSetNeighbourArray = indiAssData->getRoleSetNeighbourArray();
5199 QSet<QString> requiredNeighbourNamesSet;
5200 if (indiName == "http://semantics.crl.ibm.com/univ-bench-dl.owl#Tennis") {
5201 if (s1) {
5202 requiredNeighbourNamesSet.insert("http://www.Department1.University0.edu/StudentWhoLikesTennisButDislikesBasketBallAndBaseBall");
5203 }
5204 if (s2) {
5205 requiredNeighbourNamesSet.insert("http://www.Department1.University0.edu/StudentWhoLikesTennisAndBasketBallAndBaseBallAndShouldHaveManyHobbies");
5206 }
5207 } else if (indiName == "http://semantics.crl.ibm.com/univ-bench-dl.owl#BasketBall") {
5208 if (s2) {
5209 requiredNeighbourNamesSet.insert("http://www.Department1.University0.edu/StudentWhoLikesTennisAndBasketBallAndBaseBallAndShouldHaveManyHobbies");
5210 }
5211 if (s3) {
5212 requiredNeighbourNamesSet.insert("http://www.Department1.University0.edu/StudentWhoLikesBasketBallButDislikesBaseBall");
5213 }
5214 } else if (indiName == "http://semantics.crl.ibm.com/univ-bench-dl.owl#Baseball") {
5215 if (s2) {
5216 requiredNeighbourNamesSet.insert("http://www.Department1.University0.edu/StudentWhoLikesTennisAndBasketBallAndBaseBallAndShouldHaveManyHobbies");
5217 }
5218 }
5219
5220 if (roleSetNeighbourArray && !requiredNeighbourNamesSet.isEmpty()) {
5221 CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* indexData = roleSetNeighbourArray->getIndexData();
5222 for (cint64 i = 0; i < indexData->getArraySize(); ++i) {
5223 auto& data = roleSetNeighbourArray->at(i);
5224 CBackendRepresentativeMemoryLabelCacheItem* neighbourRoleSetLabelItem = indexData->getNeighbourRoleSetLabel(i);
5225 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiLinker = roleSetNeighbourArray->at(i).getIndividualIdLinker();
5226 for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiLinkerIt = indiLinker; indiLinkerIt; indiLinkerIt = indiLinkerIt->getNext()) {
5227 cint64 neighbourIndiId = indiLinkerIt->getIndividualId();
5228 CBackendRepresentativeMemoryCacheIndividualAssociationData* neighIndiAssData = ontologyData->getIndividualIdAssoiationDataVector()[neighbourIndiId];
5229 if (neighIndiAssData && neighIndiAssData->mDebugIndi) {
5230 requiredNeighbourNamesSet.remove(CIRIName::getRecentIRIName(neighIndiAssData->mDebugIndi->getNameLinker()));
5231 }
5232 }
5233 }
5234 if (!requiredNeighbourNamesSet.isEmpty()) {
5235 bool debug = true;
5236 missingOccurrence = true;
5237 if (debug) {
5238 writeStringifiedRepresentativeCacheToFile();
5239 }
5240 }
5241 }
5242 }
5243 }
5244 #endif
5245 return missingOccurrence;
5246 }
5247
5248 bool CBackendRepresentativeMemoryCache::debugCheckNeighoursCorrectlyCounted(CBackendRepresentativeMemoryCacheIndividualAssociationData* indiAssData, CBackendRepresentativeMemoryCacheOntologyData* ontologyData) {
5249 if (indiAssData) {
5250 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* roleSetNeighbourArray = indiAssData->getRoleSetNeighbourArray();
5251 if (roleSetNeighbourArray) {
5252 CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* indexData = roleSetNeighbourArray->getIndexData();
5253 for (cint64 i = 0; i < indexData->getArraySize(); ++i) {
5254 auto& data = roleSetNeighbourArray->at(i);
5255 CBackendRepresentativeMemoryLabelCacheItem* neighbourRoleSetLabelItem = indexData->getNeighbourRoleSetLabel(i);
5256 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiLinker = roleSetNeighbourArray->at(i).getIndividualIdLinker();
5257 cint64 count = 0;
5258 if (!indiLinker) {
5259 bool debug = false;
5260 if (debug) {
5261 writeStringifiedRepresentativeCacheToFile();
5262 }
5263 }
5264 if (indiLinker) {
5265 count = indiLinker->getCount();
5266 }
5267 if (count != roleSetNeighbourArray->at(i).getIndividualCount()) {
5268 bool debug = false;
5269 if (debug) {
5270 writeStringifiedRepresentativeCacheToFile();
5271 }
5272 return false;
5273 }
5274 }
5275 }
5276 }
5277 return true;
5278 }
5279
5280 void CBackendRepresentativeMemoryCache::checkAssociationComplete(CBackendRepresentativeMemoryCacheOntologyData* ontologyData, bool forceCompletion) {
5281 if ((ontologyData->getIncompletelyHandledIndividualIdCount() <= 0 || forceCompletion) && !ontologyData->isAssociationCompleted() && ontologyData->isFirstIncompletelyHandledIndividualsRetrieved()) {
5282 cint64 ontologyIdentifier = ontologyData->getOntologyIdentifer();
5283 ontologyData->setAssociationCompleted(true);
5284 mFixedOntologyIdentifierDataHashLock.lockForWrite();
5285 ontologyData->incUsageCount();
5286 mFixedOntologyIdentifierDataHash.insert(ontologyIdentifier, ontologyData);
5287 mFixedOntologyIdentifierDataHashLock.unlock();
5288
5289 LOG(INFO, "::Konclude::RepresentativeBackendCache", logTr("Associating %1 individuals with %2 labels completed.").arg(ontologyData->getMaxStoredIndvidualiId()).arg(ontologyData->getNextEntryID(false)), this);
5290
5291 if (mConfLateIndividualLabelAssociationIndexing) {
5292 indexIndividualLabelAssociations(ontologyData);
5293 }
5294
5295 if (mConfDebugWriteRepresentativeCache) {
5296 writeStringifiedRepresentativeCacheToFile();
5297 }
5298 }
5299 }
5300
5301 bool CBackendRepresentativeMemoryCache::debugTestingPrioritizedExpansionLinkDuplicates(CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourArray* testingArray, CBackendRepresentativeMemoryLabelCacheItem* propMarkLabelItem) {
5302 if (testingArray) {
5303 CBackendRepresentativeMemoryLabelCacheItemIndividualRoleSetNeighbourArrayIndexExtensionData* testingArrayIndexData = testingArray->getIndexData();
5304 if (testingArrayIndexData) {
5305 cint64 testingPrioPropMarkIndex = testingArrayIndexData->getIndex(propMarkLabelItem);
5306 if (testingPrioPropMarkIndex >= 0) {
5307 QSet<cint64> neighbourIndiIdSet;
5308 testingArray->at(testingPrioPropMarkIndex).visitNeighbourIndividualIds([&](cint64 id)->bool {
5309 if (neighbourIndiIdSet.contains(id)) {
5310 bool debug = true;
5311 } else {
5312 neighbourIndiIdSet.insert(id);
5313 }
5314 return true;
5315 });
5316 }
5317 }
5318 }
5319 return true;
5320 }
5321
5322 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* CBackendRepresentativeMemoryCache::copyNeighbourIndividualIdLinkers(CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiIdLinker, CBackendRepresentativeMemoryCacheContext* context, CBackendRepresentativeMemoryCacheIndividualAssociationData* locAssociationData, cint64 pos) {
5323 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* firstNewIndiIdLinker = nullptr;
5324 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* lastNewIndiIdLinker = nullptr;
5325 cint64 copiedLinkerCount = 0;
5326 for (CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* indiIdLinkerIt = indiIdLinker; indiIdLinkerIt; indiIdLinkerIt = indiIdLinkerIt->getNext()) {
5327 cint64 neighbourId = indiIdLinkerIt->getIndividualId();
5328 CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker* newLinker = CObjectAllocator< CBackendRepresentativeMemoryCacheIndividualRoleSetNeighbourIndividualIdLinker >::allocateAndConstruct(context->getMemoryAllocationManager());
5329 newLinker->initIndividualIdLinker(neighbourId);
5330 ++mStatIndividualAssociationSeparateMemoryManagmentNeighbourLinkCopyingCount;
5331 ++copiedLinkerCount;
5332
5333 if (!lastNewIndiIdLinker) {
5334 firstNewIndiIdLinker = lastNewIndiIdLinker = newLinker;
5335 } else {
5336 lastNewIndiIdLinker->setNext(newLinker);
5337 lastNewIndiIdLinker = newLinker;
5338 }
5339
5340 }
5341 return firstNewIndiIdLinker;
5342 }
5343
5344
5345 }; // end namespace Cache
5346
5347 }; // end namespace Kernel
5348
5349 }; // end namespace Reasoner
5350
5351 }; // end namespace Konclude