/*
* Copyright (C) 2013-2015, 2019 by the Konclude Developer Team.
*
* This file is part of the reasoning system Konclude.
* For details and support, see .
*
* Konclude is free software: you can redistribute it and/or modify
* it under the terms of version 3 of the GNU Lesser General Public
* License (LGPLv3) as published by the Free Software Foundation.
*
* Konclude is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU (Lesser) General Public License for more details.
*
* You should have received a copy of the GNU (Lesser) General Public
* License along with Konclude. If not, see .
*
*/
#include "CTotallyPrecomputationThread.h"
#ifndef KONCLUDE_FORCE_STATISTIC_DEACTIVATED
#define KONCLUDE_PRECOMPUTATION_SATURATION_STATISTIC_OUTPUT
#endif
namespace Konclude {
namespace Reasoner {
namespace Consistiser {
CTotallyPrecomputationThread::CTotallyPrecomputationThread(CReasonerManager *reasonerManager) : CPrecomputationThread(reasonerManager),CLogIdentifier("::Konclude::Reasoner::Kernel::Precomputator",this) {
mReuseCompletionGraphCacheWriter = nullptr;
CCompletionGraphCache* complGraphCache = reasonerManager->getCompletionGraphCache();
if (complGraphCache) {
CReuseCompletionGraphCache* reuseComplGraphCache = dynamic_cast(complGraphCache);
if (reuseComplGraphCache) {
mReuseCompletionGraphCacheWriter = reuseComplGraphCache->createCacheWriter();
}
}
mBackendAssocCache = nullptr;
CBackendCache* backendCache = reasonerManager->getBackendAssociationCache();
if (backendCache) {
mBackendAssocCache = dynamic_cast(backendCache);
}
mConfPrecomputationIndividualsRetrievalWhileSaturation = false;
}
CTotallyPrecomputationThread::~CTotallyPrecomputationThread() {
}
COntologyPrecomputationItem* CTotallyPrecomputationThread::initializeOntologyPrecomputionItem(CConcreteOntology* ontology, CConfigurationBase* config) {
CTotallyOntologyPrecomputationItem* item = new CTotallyOntologyPrecomputationItem();
item->initTotallyPrecomputationItem(ontology,config);
if (config) {
bool configErrorFlag = false;
cint64 processorCount = 1;
QString processorCountString = CConfigDataReader::readConfigString(config,"Konclude.Calculation.ProcessorCount",QString(),&configErrorFlag);
if (!configErrorFlag) {
if (processorCountString == "AUTO") {
processorCount = CThread::idealThreadCount();
} else {
qint64 convertedWorkerCount = processorCountString.toInt(&configErrorFlag);
if (configErrorFlag) {
processorCount = convertedWorkerCount;
}
}
}
bool mulConfigErrorFlag = false;
cint64 multiplicator = CConfigDataReader::readConfigInteger(config,"Konclude.Calculation.Precomputation.TotalPrecomputor.MultipliedUnitsParallelCalculationCount",1,&mulConfigErrorFlag);
mConfMaxTestParallelCount = processorCount*multiplicator;
bool maxConfigErrorFlag = false;
cint64 maxParallel = CConfigDataReader::readConfigInteger(config,"Konclude.Calculation.Precomputation.TotalPrecomputor.MaximumParallelCalculationCount",1,&maxConfigErrorFlag);
if (!maxConfigErrorFlag) {
if (!mulConfigErrorFlag) {
mConfMaxTestParallelCount = qMin(mConfMaxTestParallelCount,maxParallel);
} else {
mConfMaxTestParallelCount = maxParallel;
}
}
} else {
mConfMaxTestParallelCount = 1;
}
mConfMaxTestBatchCreationCount = CConfigDataReader::readConfigInteger(config, "Konclude.Calculation.Precomputation.TotalPrecomputor.MaximumBatchJobCreationCount", -1);
mConfForceFullCompletionGraphConstruction = CConfigDataReader::readConfigBoolean(config,"Konclude.Calculation.Precomputation.ForceFullCompletionGraphConstruction",false);
mConfConditionalFullCompletionGraphConstruction = CConfigDataReader::readConfigBoolean(config, "Konclude.Calculation.Precomputation.ConditionalFullCompletionGraphConstruction", false);
mConfFullCGCExclusionConditionMaximumIndividualLimit = CConfigDataReader::readConfigInteger(config, "Konclude.Calculation.Precomputation.FullCompletionGraphConstruction.ExclusionCondition.MaximumIndividualLimit", 300000);
mConfFullCGCSuggestionConditionMaximumIndividualLimit = CConfigDataReader::readConfigInteger(config, "Konclude.Calculation.Precomputation.FullCompletionGraphConstruction.SuggestionCondition.MaximumIndividualLimit", 10000);
mConfFullCGCExclusionConditionMaximumIndividualConceptRatio = CConfigDataReader::readConfigInteger(config, "Konclude.Calculation.Precomputation.FullCompletionGraphConstruction.ExclusionCondition.MaximumIndividualConceptRatio", 5000) / 100;
mConfFullCGCSuggestionConditionMaximumIndividualConceptRatio = CConfigDataReader::readConfigInteger(config, "Konclude.Calculation.Precomputation.FullCompletionGraphConstruction.SuggestionCondition.MaximumIndividualConceptRatio", 500) / 100;
return item;
}
bool CTotallyPrecomputationThread::createNextTest() {
COntologyPrecomputationItem* loopOntPreCompItem = 0;
bool workTestCreated = false;
while (!workTestCreated && !mProcessingOntItemList.isEmpty()) {
COntologyPrecomputationItem* ontPreCompItem = mProcessingOntItemList.first();
if (ontPreCompItem == loopOntPreCompItem) {
// don't run into infinite loop without doing something
break;
}
CTotallyOntologyPrecomputationItem* totallyPreCompItem = (CTotallyOntologyPrecomputationItem*)ontPreCompItem;
bool failDebug = false;
if (!workTestCreated && totallyPreCompItem->isConsistenceStepRequired()) {
if (!totallyPreCompItem->isConsistenceStepFinished()) {
if (totallyPreCompItem->areConsistenceStepProcessingRequirementSatisfied()) {
if (!totallyPreCompItem->hasConceptSaturationPrecomputationCreated()) {
LOG(INFO,getLogDomain(),logTr("Preparing concept saturation."),this);
CConcreteOntology* ontology = totallyPreCompItem->getOntology();
totallyPreCompItem->setSaturationStepRunning(true);
createSaturationConstructionJob(totallyPreCompItem);
totallyPreCompItem->setSaturationComputationRunning(true);
totallyPreCompItem->setConceptSaturationPrecomputationCreated(true);
addIdentifiedRemainingConsistencyRequiredConcepts(totallyPreCompItem);
addRequiredSaturationIndividuals(totallyPreCompItem);
bool fullCompletionGraphConstruction = false;
bool forceCompletionGraphConstruction = false;
if (mConfForceFullCompletionGraphConstruction || !totallyPreCompItem->hasIndividualsSaturated()) {
forceCompletionGraphConstruction = fullCompletionGraphConstruction = true;
}
if (!fullCompletionGraphConstruction) {
forceCompletionGraphConstruction = fullCompletionGraphConstruction |= totallyPreCompItem->getOntology()->getStructureSummary()->hasGroundingOccurrence();
}
if (!mConfForceFullCompletionGraphConstruction && mConfConditionalFullCompletionGraphConstruction) {
cint64 indiCount = ontology->getABox()->getNextIndividualId();
cint64 conceptCount = ontology->getTBox()->getConceptCount();
double indiConRatio = (double)indiCount / (double)conceptCount;
// check whether no exclusion limit/condition is satisfied
if (indiCount < mConfFullCGCExclusionConditionMaximumIndividualLimit && indiConRatio < mConfFullCGCExclusionConditionMaximumIndividualConceptRatio) {
// check whether at least one suggestion condition is satisfied
if (indiCount < mConfFullCGCSuggestionConditionMaximumIndividualLimit || indiConRatio < mConfFullCGCSuggestionConditionMaximumIndividualConceptRatio) {
fullCompletionGraphConstruction = true;
LOG(INFO, getLogDomain(), logTr("Enabling full completion graph construction for %1 individuals with %2 individuals to concepts ratio.").arg(indiCount).arg(indiConRatio), this);
}
}
}
totallyPreCompItem->setFullCompletionGraphConstruction(fullCompletionGraphConstruction);
totallyPreCompItem->setForceCompletionGraphConstruction(forceCompletionGraphConstruction);
}
if (totallyPreCompItem->hasRemainingConsistencyRequiredSaturationConcepts() && !totallyPreCompItem->isSaturationComputationRunning()) {
if (saturateRemainingConsistencyRequiredConcepts(totallyPreCompItem)) {
totallyPreCompItem->setSaturationComputationRunning(true);
workTestCreated = true;
}
}
if (totallyPreCompItem->hasRemainingRequiredSaturationIndividuals() && !totallyPreCompItem->hasClashedSaturationIndividuals() && (!totallyPreCompItem->isSaturationComputationRunning() || totallyPreCompItem->hasIndividualSaturationRunning())) {
if (!isAllAssertionIndividualSaturationSufficient(totallyPreCompItem) || totallyPreCompItem->isIndividualStepRequired() || totallyPreCompItem->isFullCompletionGraphConstruction() || totallyPreCompItem->isForceCompletionGraphConstruction()) {
if (saturateRemainingRequiredSaturationIndividuals(totallyPreCompItem)) {
totallyPreCompItem->incIndividualSaturationRunningCount();
totallyPreCompItem->setSaturationComputationRunning(true);
totallyPreCompItem->setIndividualStepRunning(true);
workTestCreated = true;
}
}
}
if (!totallyPreCompItem->isSaturationComputationRunning()) {
if (totallyPreCompItem->failAfterConsistencyConceptSaturation()) {
totallyPreCompItem->getConsistencePrecomputationStep()->submitRequirementsUpdate(COntologyProcessingStatus::PSFAILED);
totallyPreCompItem->getConsistencePrecomputationStep()->setStepFinished(true);
failDebug |= true;
} else {
if (!isAllAssertionIndividualSaturationSufficient(totallyPreCompItem) || totallyPreCompItem->isIndividualStepRequired()) {
if (!totallyPreCompItem->hasAllIndividualPrecomputationCreated() && !totallyPreCompItem->hasClashedSaturationIndividuals() && !totallyPreCompItem->hasIndividualPrecomputationClashed() && !totallyPreCompItem->isSaturationComputationRunning() && !totallyPreCompItem->hasIndividualSaturationRunning()) {
if (createIndividualPrecomputationCheck(totallyPreCompItem)) {
totallyPreCompItem->setIndividualStepRunning(true);
totallyPreCompItem->setIndividualComputationRunning(true);
totallyPreCompItem->incIndividualComputationRunningCount();
workTestCreated = true;
} else {
if (totallyPreCompItem->hasAllIncompletelyHandledIndividualsRetrieved()) {
totallyPreCompItem->setAllIndividualPrecomputationCreated(true);
if (!totallyPreCompItem->isIndividualComputationRunning()) {
totallyPreCompItem->setIndividualPrecomputationChecked(true);
}
}
}
}
}
}
}
if (!failDebug && !totallyPreCompItem->hasConsistenceCheckCreated() && !totallyPreCompItem->isSaturationComputationRunning() && !totallyPreCompItem->isIndividualComputationRunning() && !totallyPreCompItem->isPrecompuationRetrievingIncompletelyHandledIndividuals()) {
LOG(INFO,getLogDomain(),logTr("Precompute ontology consistency."),this);
totallyPreCompItem->setConsistenceCheckCreated(true);
workTestCreated = createConsistencePrecomputationCheck(totallyPreCompItem);
}
if (totallyPreCompItem->hasConsistenceCheched()) {
LOG(INFO,getLogDomain(),logTr("Consistency computation completed."),this);
totallyPreCompItem->setSaturationStepRunning(false);
totallyPreCompItem->getConsistencePrecomputationStep()->setStepFinished(true);
COntologyProcessingStatistics* ontProcStats = totallyPreCompItem->getConsistencePrecomputationStep()->getProcessingStepData()->getProcessingStatistics(true);
ontProcStats->setProcessingCountStatisticValue("consistency-checking-time",totallyPreCompItem->getInitializationTime()->elapsed());
if (totallyPreCompItem->failAfterConsistencyChecking()) {
totallyPreCompItem->getConsistencePrecomputationStep()->submitRequirementsUpdate(COntologyProcessingStatus::PSFAILED);
failDebug |= true;
} else {
if (totallyPreCompItem->getOntology()->getConsistence()->isOntologyConsistent()) {
totallyPreCompItem->getConsistencePrecomputationStep()->submitRequirementsUpdate(COntologyProcessingStatus::PSSUCESSFULL);
} else {
totallyPreCompItem->getConsistencePrecomputationStep()->submitRequirementsUpdate(COntologyProcessingStatus::PSSUCESSFULL | COntologyProcessingStatus::PSINCONSITENT);
}
}
}
} else {
totallyPreCompItem->getConsistencePrecomputationStep()->submitRequirementsUpdate(COntologyProcessingStatus::PSFAILED | COntologyProcessingStatus::PSFAILEDREQUIREMENT);
totallyPreCompItem->getConsistencePrecomputationStep()->setStepFinished(true);
}
} else {
totallyPreCompItem->getConsistencePrecomputationStep()->submitRequirementsUpdate();
}
}
if (!workTestCreated && totallyPreCompItem->isIndividualStepRequired() && totallyPreCompItem->isConsistenceStepFinished()) {
if (!totallyPreCompItem->isIndividualStepFinished()) {
if (totallyPreCompItem->areIndividualStepProcessingRequirementSatisfied()) {
if (totallyPreCompItem->hasRemainingRequiredSaturationIndividuals() && !totallyPreCompItem->hasClashedSaturationIndividuals() && (!totallyPreCompItem->isSaturationComputationRunning() || totallyPreCompItem->hasIndividualSaturationRunning())) {
if (saturateRemainingRequiredSaturationIndividuals(totallyPreCompItem)) {
totallyPreCompItem->incIndividualSaturationRunningCount();
totallyPreCompItem->setSaturationComputationRunning(true);
totallyPreCompItem->setIndividualStepRunning(true);
workTestCreated = true;
}
}
if (!totallyPreCompItem->hasAllIndividualPrecomputationCreated() && !totallyPreCompItem->hasClashedSaturationIndividuals() && !totallyPreCompItem->hasIndividualPrecomputationClashed() && !totallyPreCompItem->isSaturationComputationRunning() && !totallyPreCompItem->hasIndividualSaturationRunning()) {
if (createIndividualPrecomputationCheck(totallyPreCompItem)) {
totallyPreCompItem->setIndividualStepRunning(true);
totallyPreCompItem->setIndividualComputationRunning(true);
totallyPreCompItem->incIndividualComputationRunningCount(1);
} else {
if (totallyPreCompItem->hasAllIncompletelyHandledIndividualsRetrieved()) {
totallyPreCompItem->setAllIndividualPrecomputationCreated(true);
if (!totallyPreCompItem->isIndividualComputationRunning()) {
totallyPreCompItem->setIndividualPrecomputationChecked(true);
}
}
}
}
if ((totallyPreCompItem->hasIndividualPrecomputationChecked() || (totallyPreCompItem->hasClashedSaturationIndividuals() || totallyPreCompItem->hasIndividualPrecomputationClashed())) && !totallyPreCompItem->isIndividualComputationRunning()) {
totallyPreCompItem->setIndividualStepRunning(false);
totallyPreCompItem->getIndividualPrecomputationStep()->setStepFinished(true);
if (totallyPreCompItem->hasClashedSaturationIndividuals() || totallyPreCompItem->hasIndividualPrecomputationClashed()) {
totallyPreCompItem->getIndividualPrecomputationStep()->submitRequirementsUpdate(COntologyProcessingStatus::PSFAILED);
} else {
totallyPreCompItem->getIndividualPrecomputationStep()->submitRequirementsUpdate(COntologyProcessingStatus::PSSUCESSFULL);
}
}
} else {
totallyPreCompItem->setIndividualStepRunning(false);
totallyPreCompItem->getIndividualPrecomputationStep()->setStepFinished(true);
totallyPreCompItem->getIndividualPrecomputationStep()->submitRequirementsUpdate(COntologyProcessingStatus::PSFAILED | COntologyProcessingStatus::PSFAILEDREQUIREMENT);
}
} else {
totallyPreCompItem->getIndividualPrecomputationStep()->submitRequirementsUpdate();
}
}
if (!workTestCreated && totallyPreCompItem->isOccurrenceStatisticsStepRequired() && totallyPreCompItem->isConsistenceStepFinished() && !totallyPreCompItem->isSaturationStepRunning() && !totallyPreCompItem->isIndividualComputationRunning() && !totallyPreCompItem->isSaturationComputationRunning()) {
if (!totallyPreCompItem->isOccurrenceStatisticsStepFinished()) {
if (totallyPreCompItem->areOccurrenceStatisticsStepProcessingRequirementSatisfied()) {
if (!totallyPreCompItem->getCalculationConfiguration()->isOccurrenceStatisticsCollectionActivated()) {
totallyPreCompItem->setSaturationOccurrenceStatisticsCollectingInitialized(true);
totallyPreCompItem->setSaturationOccurrenceStatisticsCollected(true);
}
if (!totallyPreCompItem->hasSaturationOccurrenceStatisticsCollectingInitialized()) {
// determine concept and role occurrences in saturation graph
LOG(INFO, getLogDomain(), logTr("Collecting remaining occurrence statistics from saturation for ontology '%1'.").arg(ontPreCompItem->getOntology()->getTerminologyName()), getLogObject());
createOccurrenceStatisticsCollectingSaturationProcessingJob(totallyPreCompItem);
totallyPreCompItem->setSaturationOccurrenceStatisticsCollectingInitialized(true);
}
if (totallyPreCompItem->hasSaturationOccurrenceStatisticsCollected()) {
totallyPreCompItem->getOccurrenceStatisticsPrecomputationStep()->setStepFinished(true);
totallyPreCompItem->getOccurrenceStatisticsPrecomputationStep()->submitRequirementsUpdate(COntologyProcessingStatus::PSSUCESSFULL);
}
} else {
totallyPreCompItem->getOccurrenceStatisticsPrecomputationStep()->submitRequirementsUpdate(COntologyProcessingStatus::PSFAILED | COntologyProcessingStatus::PSFAILEDREQUIREMENT);
totallyPreCompItem->getOccurrenceStatisticsPrecomputationStep()->setStepFinished(true);
}
} else {
totallyPreCompItem->getOccurrenceStatisticsPrecomputationStep()->submitRequirementsUpdate();
}
}
if (!workTestCreated && totallyPreCompItem->isSaturationStepRequired() && totallyPreCompItem->isConsistenceStepFinished() && !totallyPreCompItem->hasIndividualSaturationRunning() && !totallyPreCompItem->isIndividualStepRunning() && !totallyPreCompItem->isSaturationComputationRunning() && (!totallyPreCompItem->isOccurrenceStatisticsStepRequired() || totallyPreCompItem->isOccurrenceStatisticsStepFinished())) {
if (!totallyPreCompItem->isSaturationStepFinished()) {
if (totallyPreCompItem->areSaturationStepProcessingRequirementSatisfied()) {
if (totallyPreCompItem->hasRemainingRequiredSaturationConcepts() && !totallyPreCompItem->isSaturationComputationRunning()) {
if (saturateRemainingRequiredItems(totallyPreCompItem)) {
totallyPreCompItem->setSaturationStepRunning(true);
totallyPreCompItem->setSaturationComputationRunning(true);
workTestCreated = true;
}
}
if (totallyPreCompItem->hasConceptSaturationPrecomputationChecked() && !totallyPreCompItem->isSaturationComputationRunning()) {
totallyPreCompItem->setSaturationStepRunning(false);
if (totallyPreCompItem->requiresAllConceptsSaturation()) {
LOG(INFO,getLogDomain(),logTr("Concept saturation finished."),this);
totallyPreCompItem->getSaturationPrecomputationStep()->setStepFinished(true);
}
CPrecomputation* precomputation = totallyPreCompItem->getPrecomputation();
CSaturationData* saturationData = totallyPreCompItem->getSaturationData();
if (totallyPreCompItem->failAfterConceptSaturation()) {
totallyPreCompItem->getSaturationPrecomputationStep()->submitRequirementsUpdate(COntologyProcessingStatus::PSFAILED);
failDebug |= true;
} else {
if (!saturationData) {
LOG(ERROR,getLogDomain(),logTr("Unknow error in computation, tableau saturation for ontology '%1' failed.").arg(ontPreCompItem->getOntology()->getTerminologyName()),getLogObject());
totallyPreCompItem->getSaturationPrecomputationStep()->submitRequirementsUpdate(COntologyProcessingStatus::PSFAILED | COntologyProcessingStatus::PSFAILEDREQUIREMENT);
} else {
#ifdef KONCLUDE_PRECOMPUTATION_SATURATION_STATISTIC_OUTPUT
logSaturationInfos(totallyPreCompItem);
#endif
extractCommonDisjunctConceptsFromPrecomputedSaturation(totallyPreCompItem);
totallyPreCompItem->getSaturationPrecomputationStep()->submitRequirementsUpdate(COntologyProcessingStatus::PSSUCESSFULL);
}
}
}
} else {
totallyPreCompItem->getSaturationPrecomputationStep()->setStepFinished(true);
totallyPreCompItem->getSaturationPrecomputationStep()->submitRequirementsUpdate(COntologyProcessingStatus::PSFAILED | COntologyProcessingStatus::PSFAILEDREQUIREMENT);
}
} else {
totallyPreCompItem->getSaturationPrecomputationStep()->submitRequirementsUpdate();
}
}
if (!workTestCreated && totallyPreCompItem->isCycleStepRequired() && totallyPreCompItem->isConsistenceStepFinished() && !totallyPreCompItem->isSaturationStepRunning() && !totallyPreCompItem->isIndividualComputationRunning() && !totallyPreCompItem->isSaturationComputationRunning()) {
if (!totallyPreCompItem->isCycleStepFinished()) {
if (totallyPreCompItem->areCycleStepProcessingRequirementSatisfied()) {
if (!totallyPreCompItem->hasConceptCyclePrecomputationChecked()) {
if (!totallyPreCompItem->hasConceptCyclePrecomputationInitialized()) {
CConcreteOntology* ontology = totallyPreCompItem->getOntology();
COntologyCoreConceptCyclesData* coreConceptCycles = ontology->getCoreConceptCyclesData();
bool hasConceptCycles = false;
if (coreConceptCycles) {
CBOXLIST* coreConceptCycleList = coreConceptCycles->getConceptCycleList();
for (CBOXLIST::const_iterator it = coreConceptCycleList->constBegin(), itEnd = coreConceptCycleList->constEnd(); it != itEnd; ++it) {
CConceptCycleData* conCycleData = *it;
if (mReuseCompletionGraphCacheWriter) {
totallyPreCompItem->addRemainingConceptCyclePrecomputation(conCycleData);
hasConceptCycles = true;
}
}
}
cint64 conceptCount = totallyPreCompItem->getOntology()->getTBox()->getConceptCount();
cint64 minSizeCount = qMax(conceptCount*0.001,50);
totallyPreCompItem->setMinConceptCycleTestSize(minSizeCount);
totallyPreCompItem->setConceptCyclePrecomputationInitialized(true);
if (!hasConceptCycles) {
totallyPreCompItem->setConceptCyclePrecomputationChecked(true);
}
}
if (totallyPreCompItem->hasRemainingConceptCyclePrecomputation()) {
CConcreteOntology* ontology = totallyPreCompItem->getOntology();
COntologyCoreConceptCyclesData* coreConceptCycles = ontology->getCoreConceptCyclesData();
QList* remConCycPreTestList = totallyPreCompItem->getRemainingConceptCyclePrecomputationList();
while (!workTestCreated && !remConCycPreTestList->isEmpty()) {
CConceptCycleData* conceptCycleData = remConCycPreTestList->takeFirst();
if (conceptCycleData->hasNominalConnection()) {
workTestCreated = createConceptCyclePrecomputation(conceptCycleData,totallyPreCompItem);
if (workTestCreated) {
LOG(INFO,getLogDomain(),logTr("Precompute core concept cycle '%1' of '%2' with '%3' concepts.").arg(conceptCycleData->getCycleID()).arg(coreConceptCycles->getConceptCycleCount()).arg(conceptCycleData->getConceptCount()),this);
}
}
}
}
if (!workTestCreated) {
if (!totallyPreCompItem->hasRemainingConceptCyclePrecomputation() && !totallyPreCompItem->hasCurrentPrecomputationTesting()) {
totallyPreCompItem->setConceptCyclePrecomputationChecked(true);
}
}
}
if (!workTestCreated) {
if (totallyPreCompItem->hasConceptCyclePrecomputationChecked()) {
totallyPreCompItem->getCyclePrecomputationStep()->setStepFinished(true);
totallyPreCompItem->getCyclePrecomputationStep()->submitRequirementsUpdate(COntologyProcessingStatus::PSSUCESSFULL);
}
}
} else {
totallyPreCompItem->getCyclePrecomputationStep()->submitRequirementsUpdate(COntologyProcessingStatus::PSFAILED | COntologyProcessingStatus::PSFAILEDREQUIREMENT);
totallyPreCompItem->getCyclePrecomputationStep()->setStepFinished(true);
}
} else {
totallyPreCompItem->getCyclePrecomputationStep()->submitRequirementsUpdate();
}
}
if (!workTestCreated) {
if (!totallyPreCompItem->hasRemainingProcessingRequirements()) {
finishOntologyPrecomputation(totallyPreCompItem);
mProcessingOntItemList.removeFirst();
} else {
mProcessingOntItemList.removeFirst();
mProcessingOntItemList.append(totallyPreCompItem);
loopOntPreCompItem = ontPreCompItem;
}
}
}
return workTestCreated;
}
void CTotallyPrecomputationThread::extractCommonDisjunctConceptsFromPrecomputedSaturation(CTotallyOntologyPrecomputationItem* totallyPreCompItem) {
LOG(INFO,getLogDomain(),logTr("Extract common disjunct concepts."),this);
CSaturationCommonDisjunctConceptsExtractor satCommonDisjConExt;
satCommonDisjConExt.extractCommonDisjunctConcepts(totallyPreCompItem->getOntology(),totallyPreCompItem->getSaturationData(),totallyPreCompItem->getSaturatedDisjunctionSet());
}
bool CTotallyPrecomputationThread::isAllAssertionIndividualSaturationSufficient(CTotallyOntologyPrecomputationItem* totallyPreCompItem) {
if (!totallyPreCompItem->hasAllAssertionIndividualSufficientSaturationChecked()) {
CIndividual* allAssertionIndi = totallyPreCompItem->getAllAssertionIndividual();
bool saturationSufficient = false;
if (allAssertionIndi) {
CSaturationData* saturationData = totallyPreCompItem->getSaturationData();
CSaturationTaskData* satTaskData = (CSaturationTaskData*)saturationData;
if (satTaskData) {
CSatisfiableCalculationTask* satCalcTask = satTaskData->getSaturationTask();
if (satCalcTask) {
CProcessingDataBox* procDataBox = satCalcTask->getProcessingDataBox();
if (procDataBox) {
CIndividualSaturationProcessNodeVector* satIndiNodeVec = procDataBox->getIndividualSaturationProcessNodeVector(false);
if (satIndiNodeVec) {
CIndividualSaturationProcessNode* allAssertionIndiNode = satIndiNodeVec->getData(allAssertionIndi->getIndividualID());
if (allAssertionIndiNode) {
CIndividualSaturationProcessNodeStatusFlags* indFlags = allAssertionIndiNode->getIndirectStatusFlags();
CIndividualSaturationProcessNodeStatusFlags* dirFlags = allAssertionIndiNode->getDirectStatusFlags();
if (!indFlags->hasClashedFlag() && indFlags->hasCompletedFlag() && dirFlags->hasCompletedFlag() && !indFlags->hasInsufficientFlag()) {
saturationSufficient = true;
}
}
}
}
}
}
}
if (saturationSufficient) {
totallyPreCompItem->setAllAssertionIndividualSaturationSufficient(true);
}
totallyPreCompItem->setAllAssertionIndividualSufficientSaturationChecked(true);
}
return totallyPreCompItem->isAllAssertionIndividualSaturationSufficient();
}
void CTotallyPrecomputationThread::logSaturationInfos(CTotallyOntologyPrecomputationItem* totallyPreCompItem) {
CConcreteOntology* ontology = totallyPreCompItem->getOntology();
CSaturationData* saturationData = totallyPreCompItem->getSaturationData();
CSaturationTaskData* satTaskData = (CSaturationTaskData*)saturationData;
if (satTaskData) {
CSatisfiableCalculationTask* satCalcTask = satTaskData->getSaturationTask();
if (satCalcTask) {
CProcessingDataBox* procDataBox = satCalcTask->getProcessingDataBox();
if (procDataBox) {
CIndividualSaturationProcessNodeVector* satIndiNodeVec = procDataBox->getIndividualSaturationProcessNodeVector(false);
if (satIndiNodeVec) {
cint64 saturatedCount = 0;
cint64 insufficientCount = 0;
cint64 criticalCount = 0;
cint64 clashedCount = 0;
cint64 indiCount = satIndiNodeVec->getItemCount();
for (cint64 indiID = 0; indiID < indiCount; ++indiID) {
CIndividualSaturationProcessNode* indiNode = satIndiNodeVec->getData(indiID);
if (indiNode && indiNode->isCompleted()) {
++saturatedCount;
CIndividualSaturationProcessNodeStatusFlags* flags = indiNode->getIndirectStatusFlags();
if (flags->hasClashedFlag()) {
++clashedCount;
}
if (flags->hasInsufficientFlag()) {
++insufficientCount;
}
if (flags->hasCriticalFlag()) {
++criticalCount;
}
}
}
LOG(INFO,getLogDomain(),logTr("Saturated %1 individual nodes, thereof %2 with clashes, %3 critical and %4 insufficient.").arg(saturatedCount).arg(clashedCount).arg(criticalCount).arg(insufficientCount),this);
}
}
}
}
}
bool CTotallyPrecomputationThread::createConceptCyclePrecomputation(CConceptCycleData* conceptCycleData, CTotallyOntologyPrecomputationItem* totallyPreCompItem) {
CConcreteOntology* onto = totallyPreCompItem->getOntology();
CConcept* repCycleClassConcept = conceptCycleData->getRepresentativeCycleClass();
if (repCycleClassConcept) {
CSaturationConceptDataItem* conSatItem = totallyPreCompItem->getSaturationConceptDataItem(repCycleClassConcept,false,false);
if (conSatItem) {
CIndividualSaturationProcessNode* satNode = (CIndividualSaturationProcessNode*)conSatItem->getIndividualProcessNodeForConcept();
if (!satNode->getIndirectStatusFlags()->hasInsufficientFlag()) {
return false;
}
}
if (conceptCycleData->getConceptCount() >= totallyPreCompItem->getMinConceptCycleTestSize()) {
CConceptCyclePrecomputationTestingItem* conCycPrecTestItem = new CConceptCyclePrecomputationTestingItem(conceptCycleData,totallyPreCompItem);
CSatisfiableCalculationJobGenerator satCalcJobGen(onto);
CSatisfiableCalculationJob* satCalcJob = satCalcJobGen.getSatisfiableCalculationJob(repCycleClassConcept);
CCalculationConfigurationExtension* calcConfig = totallyPreCompItem->getCalculationConfiguration();
satCalcJob->setCalculationConfiguration(calcConfig);
satCalcJob->setSatisfiableTaskPreyingAdapter(conCycPrecTestItem->getTaskPreyingAdapter());
processCalculationJob(satCalcJob,totallyPreCompItem,conCycPrecTestItem);
return true;
}
}
return false;
}
QList CTotallyPrecomputationThread::getIndividualComputationList(CTotallyOntologyPrecomputationItem* totallyPreCompItem) {
QList indiList;
CConcreteOntology* onto = totallyPreCompItem->getOntology();
CIndividualVector* indiVec = onto->getABox()->getIndividualVector(false);
QSet* activeIndiSet = onto->getABox()->getActiveIndividualSet(false);
cint64 indiCount = 0;
if (indiVec) {
indiCount = indiVec->getItemCount();
}
bool allIndiProRequired = true;
QSet* procReqIndiSet = procReqIndiSet = totallyPreCompItem->getIncompletelyHandledIndividualSet();
cint64 maxTriplesIndexedIndiId = -1;
cint64 maxABoxIndiId = 0;
COntologyTriplesAssertionsAccessor* triplesAccessor = totallyPreCompItem->getOntology()->getOntologyTriplesData()->getTripleAssertionAccessor();
if (triplesAccessor) {
maxTriplesIndexedIndiId = totallyPreCompItem->getOntology()->getOntologyTriplesData()->getTripleAssertionAccessor()->getMaxIndexedIndividualId();
}
if (indiVec) {
cint64 indiCount = indiVec->getItemCount();
for (cint64 idx = 0; idx < indiCount; ++idx) {
CIndividual* indi = indiVec->getData(idx);
if (allIndiProRequired || procReqIndiSet && procReqIndiSet->contains(CIndividualReference(idx))) {
if (indi && activeIndiSet->contains(indi)) {
indiList.append(CIndividualReference(indi));
}
else if (idx <= maxTriplesIndexedIndiId) {
indiList.append(CIndividualReference(idx));
}
}
maxABoxIndiId = qMax(idx, maxABoxIndiId);
}
}
for (cint64 idx = maxABoxIndiId + 1; idx <= maxTriplesIndexedIndiId; ++idx) {
if (allIndiProRequired || procReqIndiSet && procReqIndiSet->contains(CIndividualReference(idx))) {
indiList.append(CIndividualReference(idx));
}
}
return indiList;
}
QList CTotallyPrecomputationThread::getNextIndividualComputationPartList(CTotallyOntologyPrecomputationItem* totallyPreCompItem) {
QList indiList;
CIndividualPrecomputationCoordinationHash* precomputationProcessingCoordinationHash = totallyPreCompItem->getPrecomputationProcessingCoordinationHash();
bool compuation = false;
bool forceCompuation = false;
cint64 computationLimit = totallyPreCompItem->getCurrentIncompletelyHandledIndividualComputationLimit();
cint64 reducedComputationLimit = computationLimit * totallyPreCompItem->getBasicPrecompuationSubsequentIndividualLimitReductionFactor();
if (totallyPreCompItem->hadBasicPrecompuationMode() && precomputationProcessingCoordinationHash->hasBasicPrecomputationFinished()) {
totallyPreCompItem->setBasicPrecompuationParallelizationReductionFactor(totallyPreCompItem->getBasicPrecompuationParallelizationInitializationFactor());
totallyPreCompItem->setBasicPrecompuationSubsequentIndividualLimitReductionFactor(totallyPreCompItem->getBasicPrecompuationSubsequentIndividualLimitReductionInitializationFactor());
reducedComputationLimit = computationLimit * totallyPreCompItem->getBasicPrecompuationSubsequentIndividualLimitReductionFactor();
precomputationProcessingCoordinationHash->setBasicPrecomputationFinished(false);
totallyPreCompItem->setBasicPrecompuationMode(true);
}
if (totallyPreCompItem->getCurrentRecomputationExpansionPropagationCuttedReductionFactor() > 0) {
cint64 propCutReductionCount = computationLimit * totallyPreCompItem->getCurrentRecomputationExpansionPropagationCuttedReductionFactor();
reducedComputationLimit -= propCutReductionCount;
reducedComputationLimit = qMax(reducedComputationLimit, (cint64)1);
}
if (totallyPreCompItem->getBasicPrecompuationParallelizationReductionFactor() > 0) {
if (totallyPreCompItem->getIndividualComputationRunningCount() >= 1 && totallyPreCompItem->getIndividualComputationRunningCount() >= totallyPreCompItem->getBasicPrecompuationParallelizationReductionFactor()) {
return indiList;
}
}
cint64 approxRemainingIncHandIndiCount = precomputationProcessingCoordinationHash->getApproximateRemainingIncompletelyHandledCount();
if (precomputationProcessingCoordinationHash->hasBasicPrecomputationMode()) {
compuation = true;
forceCompuation = true;
totallyPreCompItem->setBasicPrecompuationMode(true);
} else if (approxRemainingIncHandIndiCount > reducedComputationLimit) {
compuation = true;
} else {
if (!totallyPreCompItem->isPrecompuationRetrievingIncompletelyHandledIndividuals() && !totallyPreCompItem->isIndividualComputationRunning() && !totallyPreCompItem->isPrecompuationRetrievingIncompletelyHandledIndividuals()) {
retrieveIndividualsPrecomputation(totallyPreCompItem);
precomputationProcessingCoordinationHash = totallyPreCompItem->getPrecomputationProcessingCoordinationHash();
if (precomputationProcessingCoordinationHash->getHashRemainingCount() <= 0 && precomputationProcessingCoordinationHash->getApproximateTotalIncompletelyHandledCount() <= 0) {
QTime* indiPrecTimer = totallyPreCompItem->getIndividualPrecomputationTime();
LOG(INFO, getLogDomain(), logTr("Finished insufficiently handled individual reprocessing with representative cache in %1 ms.").arg(indiPrecTimer->elapsed()), getLogObject());
totallyPreCompItem->setAllIncompletelyHandledIndividualsRetrieved(true);
decrementUsageFromIndividualPrecomputationCoordinationHash(precomputationProcessingCoordinationHash, totallyPreCompItem);
precomputationProcessingCoordinationHash = nullptr;
totallyPreCompItem->setPrecomputationProcessingCoordinationHash(nullptr);
} else {
totallyPreCompItem->incNonPrecompuationDirectIncompletelyHandledIndividualsRetrievingStreakSize();
forceCompuation = true;
compuation = true;
}
}
}
if (forceCompuation || (compuation && precomputationProcessingCoordinationHash && precomputationProcessingCoordinationHash->getHashRemainingCount() >= reducedComputationLimit || compuation && approxRemainingIncHandIndiCount > reducedComputationLimit && !totallyPreCompItem->isIndividualComputationRunning())) {
CIndividualPrecomputationCoordinationHash* precomputationProcessingCoordinationHash = totallyPreCompItem->getPrecomputationProcessingCoordinationHash();
CIndividualPrecomputationCoordinationHash::const_iterator itCurr = totallyPreCompItem->getPrecomputationProcessingCoordinationHashIteratorCurrent();
CIndividualPrecomputationCoordinationHash::const_iterator itEnd = totallyPreCompItem->getPrecomputationProcessingCoordinationHashIteratorEnd();
cint64 currentCount = 0;
while (currentCount < reducedComputationLimit && itCurr != itEnd && precomputationProcessingCoordinationHash->getHashRemainingCount() > 0) {
cint64 indiId = itCurr.key();
CBackendIndividualRetrievalComputationUpdateCoordinationHashData* data = itCurr.value();
if (!data->isProcessed()) {
if (!data->isComputationIntegrated() && !data->isComputationOrdered()) {
data->setComputationOrdered(true);
++currentCount;
indiList.append(indiId);
precomputationProcessingCoordinationHash->decHashRemainingCount();
precomputationProcessingCoordinationHash->incHashComputationCount();
totallyPreCompItem->setNonPrecompuationDirectIncompletelyHandledIndividualsRetrievingStreakSize(0);
} else if (data->isComputationIntegrated() && !data->isComputationOrdered()) {
data->setProcessed(true);
precomputationProcessingCoordinationHash->incHashProcessedCount();
precomputationProcessingCoordinationHash->decHashRemainingCount();
}
}
++itCurr;
}
if (itCurr == itEnd) {
precomputationProcessingCoordinationHash->setHashRemainingCount(0);
}
}
if ((indiList.isEmpty()
|| !forceCompuation && precomputationProcessingCoordinationHash->getHashRemainingCount() <= reducedComputationLimit && precomputationProcessingCoordinationHash->getApproximateRemainingIncompletelyHandledCount() > reducedComputationLimit
|| !forceCompuation && precomputationProcessingCoordinationHash->getHashRemainingCount() <= totallyPreCompItem->getCurrentIncompletelyHandledIndividualRetrievalThreshold() && !totallyPreCompItem->isPrecompuationIncompletelyHandledIndividualsRetrievalThresholdRequested() && precomputationProcessingCoordinationHash->getApproximateRemainingIncompletelyHandledCount() > computationLimit)
&& !totallyPreCompItem->hasAllIncompletelyHandledIndividualsRetrieved()) {
if (precomputationProcessingCoordinationHash->getHashRemainingCount() <= totallyPreCompItem->getCurrentIncompletelyHandledIndividualRetrievalThreshold()) {
totallyPreCompItem->setPrecompuationIncompletelyHandledIndividualsRetrievalThresholdRequested(true);
}
requestIndividualsPrecomputationRetrieval(totallyPreCompItem, true);
}
return indiList;
}
bool CTotallyPrecomputationThread::createIndividualPrecomputationCheck(CTotallyOntologyPrecomputationItem* totallyPreCompItem) {
CConcreteOntology* onto = totallyPreCompItem->getOntology();
CSatisfiableCalculationJob* satCalcJob = nullptr;
bool fullCompletionGraphConstruction = totallyPreCompItem->isFullCompletionGraphConstruction();
if (!fullCompletionGraphConstruction && !totallyPreCompItem->hasIndividualsSaturated()) {
fullCompletionGraphConstruction = true;
totallyPreCompItem->setFullCompletionGraphConstruction(true);
}
if (totallyPreCompItem->hasIndividualsSaturationCacheSynchronisation() && totallyPreCompItem->hasIndividualsSaturated() && totallyPreCompItem->hasALLIndividualsSaturationOrderd() && totallyPreCompItem->hasInsufficientSaturationIndividuals()) {
if (!fullCompletionGraphConstruction) {
if (!totallyPreCompItem->hasIndividualPrecomputationChecked()) {
QList indiList;
indiList = getNextIndividualComputationPartList(totallyPreCompItem);
if (!indiList.isEmpty()) {
CIndividualPrecomputationCoordinationHash* precomputationProcessingCoordinationHash = totallyPreCompItem->getPrecomputationProcessingCoordinationHash();
cint64 minIndiId = CINT64_MAX;
QList filteredIndiList = indiList;
QSet* currentIndiComputationSet = totallyPreCompItem->getCurrentIndividualComputationSet();
for (CIndividualReference indiRef : indiList) {
minIndiId = qMin(minIndiId, indiRef.getIndividualID());
//filteredIndiList.append(indiRef);
//currentIndiComputationSet->insert(indiRef);
}
cint64 indiCompCount = filteredIndiList.size();
cint64 remainingLimitIncreasingIndiCompCount = totallyPreCompItem->getRemainingIncompletelyHandlingIndividualComputationLimitIncreasingCount();
remainingLimitIncreasingIndiCompCount -= indiCompCount;
cint64 currentIndiCompLimit = totallyPreCompItem->getCurrentIncompletelyHandledIndividualComputationLimit();
if (totallyPreCompItem->getBasicPrecompuationSubsequentIndividualLimitReductionFactor() < 1.) {
totallyPreCompItem->setBasicPrecompuationSubsequentIndividualLimitReductionFactor(totallyPreCompItem->getBasicPrecompuationSubsequentIndividualLimitReductionFactor() * totallyPreCompItem->getBasicPrecompuationSubsequentIndividualLimitReductionIncreasingFactor());
} else if (remainingLimitIncreasingIndiCompCount < 0) {
remainingLimitIncreasingIndiCompCount = 0;
double limitIncreasingFactor = totallyPreCompItem->getIncompletelyHandlingIndividualComputationLimitIncreasingFactor();
currentIndiCompLimit *= limitIncreasingFactor;
if (currentIndiCompLimit < 0) {
currentIndiCompLimit = CINT64_MAX;
}
totallyPreCompItem->setCurrentIncompletelyHandledIndividualComputationLimit(currentIndiCompLimit);
cint64 currentRetrievalLimit = totallyPreCompItem->getCurrentIncompletelyHandledIndividualRetrievalLimit();
currentRetrievalLimit *= limitIncreasingFactor;
if (currentRetrievalLimit < 0) {
currentRetrievalLimit = CINT64_MAX;
}
totallyPreCompItem->setCurrentIncompletelyHandledIndividualRetrievalLimit(currentRetrievalLimit);
}
totallyPreCompItem->setRemainingIncompletelyHandlingIndividualComputationLimitIncreasingCount(remainingLimitIncreasingIndiCompCount);
if (totallyPreCompItem->getBasicPrecompuationParallelizationReductionFactor() > 0) {
totallyPreCompItem->setBasicPrecompuationParallelizationReductionFactor(totallyPreCompItem->getBasicPrecompuationParallelizationReductionFactor() * totallyPreCompItem->getBasicPrecompuationParallelizationIncreasingFactor());
}
CSatisfiableCalculationJobGenerator satCalcJobGen(onto);
satCalcJob = satCalcJobGen.getSatisfiableCalculationJob(QList());
CIndividualPrecomputationTestingItem* indiTestItem = new CIndividualPrecomputationTestingItem(totallyPreCompItem, precomputationProcessingCoordinationHash, totallyPreCompItem, totallyPreCompItem->getIndividualPrecomputationClashedPointer(), totallyPreCompItem->getIndividualPrecomputationExpansionLimitedReachedPointer());
if (precomputationProcessingCoordinationHash) {
precomputationProcessingCoordinationHash->incUsageCount();
}
indiTestItem->addIndividualComputations(filteredIndiList);
CCalculationConfigurationExtension* calcConfig = totallyPreCompItem->getCalculationConfiguration();
satCalcJob->setCalculationConfiguration(calcConfig);
CSatisfiableTaskRepresentativeBackendUpdatingAdapter* repCacheUpdAdapter = indiTestItem->getRepresentativeCacheUpdatingAdapter();
if (filteredIndiList.size() < currentIndiCompLimit) {
repCacheUpdAdapter->setAdditionalIndividualComputationSize(currentIndiCompLimit - filteredIndiList.size());
}
repCacheUpdAdapter->setFirstIndividualRecomputationId(minIndiId);
repCacheUpdAdapter->setRepresentativeCacheRecomputationId(totallyPreCompItem->getNextRepresentativeCacheRecomputationId());
satCalcJob->setSatisfiableRepresentativeBackendCacheUpdatingAdapter(repCacheUpdAdapter);
CIndividual* allAssertionIndi = totallyPreCompItem->getAllAssertionIndividual();
if (allAssertionIndi) {
repCacheUpdAdapter->setAllAssertionIndividualId(allAssertionIndi->getIndividualID());
}
QMap* recomIdTestingItemMap = totallyPreCompItem->getRecomputationIdTestingItemMap();
recomIdTestingItemMap->insert(repCacheUpdAdapter->getRepresentativeCacheRecomputationId(), indiTestItem);
LOG(INFO, getLogDomain(), logTr("Scheduled precomputation of next %1 of approximately %2/%3 remaining insufficiently handled individuals (%4 batchs queued, %5 computing, %6 processed).").arg(filteredIndiList.size()).arg(filteredIndiList.size() + precomputationProcessingCoordinationHash->getHashRemainingCount()).arg(filteredIndiList.size() + precomputationProcessingCoordinationHash->getApproximateRemainingIncompletelyHandledCount()).arg(mCurrRunningTestParallelCount + 1).arg(precomputationProcessingCoordinationHash->getHashComputationCount()).arg(precomputationProcessingCoordinationHash->getHashProcessedCount()), getLogObject());
processCalculationJob(satCalcJob, totallyPreCompItem, indiTestItem);
return true;
}
}
}
}
return false;
}
bool CTotallyPrecomputationThread::createConsistencePrecomputationCheck(CTotallyOntologyPrecomputationItem* totallyPreCompItem) {
CConcreteOntology* onto = totallyPreCompItem->getOntology();
CSatisfiableCalculationJob* satCalcJob = nullptr;
CIndividualVector* indiVec = onto->getABox()->getIndividualVector(false);
QSet* activeIndiSet = onto->getABox()->getActiveIndividualSet(false);
cint64 indiCount = 0;
if (indiVec) {
indiCount = indiVec->getItemCount();
}
cint64 maxTriplesIndexedIndiId = -1;
COntologyTriplesAssertionsAccessor* triplesAccessor = totallyPreCompItem->getOntology()->getOntologyTriplesData()->getTripleAssertionAccessor();
if (triplesAccessor) {
maxTriplesIndexedIndiId = totallyPreCompItem->getOntology()->getOntologyTriplesData()->getTripleAssertionAccessor()->getMaxIndexedIndividualId();
}
bool forceCompletionGraphConstruction = totallyPreCompItem->isForceCompletionGraphConstruction();
bool fullCompletionGraphConstruction = totallyPreCompItem->isFullCompletionGraphConstruction();
bool consistencyDetected = false;
bool detectedConsistency = true;
QSet* procReqIndiSet = procReqIndiSet = totallyPreCompItem->getIncompletelyHandledIndividualSet();
if (!procReqIndiSet->isEmpty()) {
forceCompletionGraphConstruction = true;
}
if (!forceCompletionGraphConstruction && isAllAssertionIndividualSaturationSufficient(totallyPreCompItem)) {
consistencyDetected = true;
detectedConsistency = true;
LOG(INFO,getLogDomain(),logTr("Trivial consistency detected with merged individual."),getLogObject());
} else if (totallyPreCompItem->hasClashedSaturationIndividuals()) {
consistencyDetected = true;
detectedConsistency = false;
LOG(INFO,getLogDomain(),logTr("Trivial inconsistency detected with individual saturation."),getLogObject());
} else if (!forceCompletionGraphConstruction && totallyPreCompItem->hasIndividualsSaturated() && !totallyPreCompItem->hasInsufficientSaturationIndividuals() && (indiCount > 0 || maxTriplesIndexedIndiId > 0)) {
consistencyDetected = true;
detectedConsistency = true;
LOG(INFO,getLogDomain(),logTr("Trivial consistency detected with individual saturation."),getLogObject());
} else if (!fullCompletionGraphConstruction && totallyPreCompItem->hasIndividualsSaturated() && totallyPreCompItem->hasAllIncompletelyHandledIndividualsRetrieved() && (indiCount > 0 || maxTriplesIndexedIndiId > 0)) {
consistencyDetected = true;
detectedConsistency = !totallyPreCompItem->hasIndividualPrecomputationClashed();
if (detectedConsistency) {
LOG(INFO, getLogDomain(), logTr("Consistency detected with individual precomputation."), getLogObject());
} else {
LOG(INFO, getLogDomain(), logTr("Inconsistency detected with individual precomputation."), getLogObject());
}
} else if (totallyPreCompItem->hasIndividualPrecomputationClashed()) {
consistencyDetected = true;
detectedConsistency = false;
LOG(INFO, getLogDomain(), logTr("Inconsistency detected with individual precomputation."), getLogObject());
} else {
QList indiList;
if (!fullCompletionGraphConstruction && procReqIndiSet) {
indiList = procReqIndiSet->toList();
procReqIndiSet->clear();
} else {
indiList = getIndividualComputationList(totallyPreCompItem);
}
totallyPreCompItem->setAllIncompletelyHandledIndividualsRetrieved(true);
totallyPreCompItem->setFullCompletionGraphConstructed(true);
CSatisfiableCalculationJobGenerator satCalcJobGen(onto);
if (indiList.isEmpty()) {
CConcept* topConcept = onto->getTBox()->getTopConcept();
if (topConcept->getOperandCount() > 0) {
satCalcJob = satCalcJobGen.getSatisfiableCalculationJob(topConcept);
} else {
consistencyDetected = true;
detectedConsistency = true;
}
} else {
satCalcJob = satCalcJobGen.getSatisfiableCalculationJob(indiList);
}
}
CConsistence* consistence = onto->getConsistence();
if (!consistence) {
consistence = new CConsistence();
onto->setConsistence(consistence);
}
if (totallyPreCompItem->hasIndividualsSaturated() && totallyPreCompItem->hasALLIndividualsSaturationOrderd()) {
consistence->setIndividualsRepresentativelyStored(true);
if (!totallyPreCompItem->hasInsufficientSaturationIndividuals()) {
consistence->setAllIndividualsRepresentativelyStored(true);
} else if (totallyPreCompItem->hasAllIndividualPrecomputationCreated()) {
consistence->setAllIndividualsRepresentativelyStored(true);
}
}
totallyPreCompItem->setConsistence(consistence);
if (consistencyDetected) {
consistence->setOntologyConsistent(detectedConsistency);
totallyPreCompItem->setConsistenceCheched(true);
return false;
} else {
CConsistencePrecomputationTestingItem* consTestItem = new CConsistencePrecomputationTestingItem(totallyPreCompItem,totallyPreCompItem);
CCalculationConfigurationExtension* calcConfig = totallyPreCompItem->getCalculationConfiguration();
satCalcJob->setCalculationConfiguration(calcConfig);
satCalcJob->setSatisfiableTaskPreyingAdapter(consTestItem->getTaskPreyingAdapter());
processCalculationJob(satCalcJob,totallyPreCompItem,consTestItem);
return true;
}
}
void CTotallyPrecomputationThread::propagateExistInitializationFlag(CSaturationConceptDataItem* ontConSatDataItem) {
if (ontConSatDataItem->isPotentiallyExistInitializationConcept()) {
CSaturationConceptDataItem* parentOntConceptSatTestItem = ontConSatDataItem->getSpecialItemReference();
while (parentOntConceptSatTestItem && !parentOntConceptSatTestItem->isPotentiallyExistInitializationConcept()) {
parentOntConceptSatTestItem->setPotentiallyExistInitializationConcept(true);
parentOntConceptSatTestItem = parentOntConceptSatTestItem->getSpecialItemReference();
}
}
}
void CTotallyPrecomputationThread::propagateSubsumerItemFlag(CSaturationConceptDataItem* ontClassSatTestItem) {
if (ontClassSatTestItem->hasIndirectSuccessorsItems()) {
CSaturationConceptDataItem* subsumerItem = ontClassSatTestItem->getSpecialItemReference();
while (subsumerItem && !subsumerItem->hasIndirectSuccessorsItems()) {
subsumerItem->setIndirectSuccessorsItems(true);
subsumerItem = subsumerItem->getSpecialItemReference();
}
}
}
void CTotallyPrecomputationThread::analyseConceptSaturationSubsumerExistItems(CTotallyOntologyPrecomputationItem* totallyPreCompItem, QList& processList) {
QSet processSet;
for (QList::const_iterator it = processList.constBegin(), itEnd = processList.constEnd(); it != itEnd; ++it) {
CConceptNegationSaturationItem conNegSatItem(*it);
processSet.insert(conNegSatItem);
}
while (!processList.isEmpty()) {
CConceptNegationSaturationItem conNegSatItem(processList.takeLast());
CConcept* concept = conNegSatItem.getConcept();
bool negated = conNegSatItem.getNegation();
CSaturationConceptDataItem* ontConSatDataItem = conNegSatItem.getSaturationConceptDataItem();
cint64 nOpCode = concept->getOperatorCode();
cint64 opCount = concept->getOperandCount();
if (!negated && (nOpCode == CCAND || nOpCode == CCSUB || nOpCode == CCEQ || ((nOpCode == CCOR) && opCount == 1)) || negated && (nOpCode == CCOR || ((nOpCode == CCAND || nOpCode == CCEQ) && opCount == 1))) {
CSortedNegLinker* opConIt = concept->getOperandList();
while (opConIt) {
CConcept* opConcept = opConIt->getData();
bool opNegation = opConIt->isNegated()^negated;
cint64 opConOpCode = opConcept->getOperatorCode();
if (!opNegation && (opConOpCode == CCEQ || opConOpCode == CCSUB || opConOpCode == CCATOM && opConcept->hasClassName())) {
CSaturationConceptDataItem* subsumerClassTestItem = totallyPreCompItem->getSaturationConceptDataItem(opConcept,opNegation,false);
subsumerClassTestItem->setIndirectSuccessorsItems(true);
if (!ontConSatDataItem->getSpecialItemReference() && !ontConSatDataItem->isInvalidSpecialItemReference()) {
ontConSatDataItem->setSpecialItemReference(subsumerClassTestItem);
} else {
ontConSatDataItem->setMultipleDirectPredecessorsItems(true);
}
} else if (!opNegation && opConOpCode == CCAND || opNegation && opConOpCode == CCOR) {
if (opConcept->getOperandCount() > 1) {
ontConSatDataItem->setMultipleDirectPredecessorsItems(true);
}
if (!processSet.contains(CConceptNegationSaturationItem(opConcept,opNegation,ontConSatDataItem))) {
processSet.insert(CConceptNegationSaturationItem(opConcept,opNegation,ontConSatDataItem));
processList.append(CConceptNegationSaturationItem(opConcept,opNegation,ontConSatDataItem));
}
} else if ((!opNegation && (opConOpCode == CCSOME || opConOpCode == CCAQSOME)) || opNegation && opConOpCode == CCALL) {
CRole* role = opConcept->getRole();
CConcept* existConcept = opConcept->getOperandList()->getData();
bool existConNegation = opConcept->getOperandList()->isNegated()^opNegation;
CSaturationConceptDataItem* existConceptTestItem = nullptr;
if (hasRoleRanges(totallyPreCompItem,role)) {
existConceptTestItem = totallyPreCompItem->getSaturationRoleSuccessorConceptDataItem(role,existConcept,existConNegation,false);
} else {
existConceptTestItem = totallyPreCompItem->getSaturationConceptDataItem(existConcept,existConNegation,false);
}
ontConSatDataItem->addExistReferenceConceptItemList(existConceptTestItem);
ontConSatDataItem->setMultipleDirectPredecessorsItems(true);
} else if (negated && opConOpCode == CCATMOST || !negated && opConOpCode == CCATLEAST) {
CRole* role = opConcept->getRole();
CConcept* existConcept = totallyPreCompItem->getOntology()->getTBox()->getTopConcept();
bool existConceptNegation = false;
if (opConcept->getOperandList()) {
existConcept = opConcept->getOperandList()->getData();
existConceptNegation = opConcept->getOperandList()->isNegated();
}
CSaturationConceptDataItem* existConceptTestItem = nullptr;
if (hasRoleRanges(totallyPreCompItem,role)) {
existConceptTestItem = totallyPreCompItem->getSaturationRoleSuccessorConceptDataItem(role,existConcept,existConceptNegation,false);
} else {
existConceptTestItem = totallyPreCompItem->getSaturationConceptDataItem(existConcept,existConceptNegation,false);
}
ontConSatDataItem->addExistReferenceConceptItemList(existConceptTestItem);
ontConSatDataItem->setMultipleDirectPredecessorsItems(true);
} else if (opConOpCode == CCAQCHOOCE) {
CSortedNegLinker* opOpConIt = opConcept->getOperandList();
while (opOpConIt) {
if (opNegation == opOpConIt->isNegated()) {
CConcept* opOpConcept = opOpConIt->getData();
if (opOpConcept->getOperatorCode() == CCAQSOME) {
CConcept* existConcept = opOpConcept->getOperandList()->getData();
bool existConNegation = opOpConcept->getOperandList()->isNegated();
CRole* role = opOpConcept->getRole();
CSaturationConceptDataItem* existConceptTestItem = nullptr;
if (hasRoleRanges(totallyPreCompItem,role)) {
existConceptTestItem = totallyPreCompItem->getSaturationRoleSuccessorConceptDataItem(role,existConcept,existConNegation,false);
} else {
existConceptTestItem = totallyPreCompItem->getSaturationConceptDataItem(existConcept,existConNegation,false);
}
ontConSatDataItem->addExistReferenceConceptItemList(existConceptTestItem);
ontConSatDataItem->setMultipleDirectPredecessorsItems(true);
} else if (opOpConcept->getOperatorCode() == CCAQAND) {
if (opConcept->getOperandCount() > 1) {
ontConSatDataItem->setMultipleDirectPredecessorsItems(true);
}
if (!processSet.contains(CConceptNegationSaturationItem(opOpConcept,false,ontConSatDataItem))) {
processSet.insert(CConceptNegationSaturationItem(opOpConcept,false,ontConSatDataItem));
processList.append(CConceptNegationSaturationItem(opOpConcept,false,ontConSatDataItem));
}
}
}
opOpConIt = opOpConIt->getNext();
}
} else {
ontConSatDataItem->setMultipleDirectPredecessorsItems(true);
}
opConIt = opConIt->getNext();
}
}
}
}
CConcept* CTotallyPrecomputationThread::getDisjunctCheckingConcept(CConcept* opConcept, bool opConNegation, bool* checkingNegation) {
if (opConcept->getOperatorCode() == CCAQCHOOCE) {
cint64 replaceCount = 0;
CConcept* replaceCheckingConcept = nullptr;
CSortedNegLinker* opConceptOpLinkerIt = opConcept->getOperandList();
while (opConceptOpLinkerIt) {
CConcept* opOpConcept = opConceptOpLinkerIt->getData();
bool opOpNegation = opConceptOpLinkerIt->isNegated();
if (opOpNegation == opConNegation) {
replaceCheckingConcept = opOpConcept;
replaceCount++;
}
opConceptOpLinkerIt = opConceptOpLinkerIt->getNext();
}
if (replaceCount == 1 && replaceCheckingConcept) {
if (checkingNegation) {
*checkingNegation = false;
}
return replaceCheckingConcept;
}
}
return opConcept;
}
void CTotallyPrecomputationThread::extendDisjunctionsCandidateAlternativesItems(CTotallyOntologyPrecomputationItem* totallyPreCompItem, CSaturationConceptDataItem* ontConSatDataItem, QList* newDisjunctionCandidateAlternativeList) {
CConcept* concept = ontConSatDataItem->getSaturationConcept();
bool negation = ontConSatDataItem->getSaturationNegation();
QList conNegPairList;
bool directExamine = true;
bool candidateAlternativeExtraction = false;
if (!negation && concept->hasClassName()) {
if (concept->getOperatorCode() == CCEQ) {
//CBOXHASH* eqConHash = totallyPreCompItem->getOntology()->getTBox()->getEquivalentConceptCandidateHash(false);
//if (eqConHash->contains(concept)) {
candidateAlternativeExtraction = true;
//}
}
for (CSortedNegLinker* opConIt = concept->getOperandList(); opConIt; opConIt = opConIt->getNext()) {
CConcept* opConcept = opConIt->getData();
bool opNegation = opConIt->isNegated()^negation;
conNegPairList.append(TConceptNegPair(opConcept,opNegation));
directExamine = false;
}
}
if (directExamine) {
conNegPairList.append(TConceptNegPair(concept,negation));
}
while (!conNegPairList.isEmpty()) {
TConceptNegPair nextConNegPair(conNegPairList.takeFirst());
CConcept* nextConcept = nextConNegPair.first;
bool nextNegation = nextConNegPair.second;
cint64 nOpCode = nextConcept->getOperatorCode();
cint64 opCount = nextConcept->getOperandCount();
if (!nextNegation && (nOpCode == CCAND || (nOpCode == CCOR && opCount == 1)) || nextNegation && (nOpCode == CCOR || ((nOpCode == CCAND) && opCount == 1))) {
for (CSortedNegLinker* opConIt = nextConcept->getOperandList(); opConIt; opConIt = opConIt->getNext()) {
CConcept* opConcept = opConIt->getData();
bool opNegation = opConIt->isNegated()^nextNegation;
conNegPairList.append(TConceptNegPair(opConcept,opNegation));
}
} else if (nOpCode == CCAQCHOOCE) {
for (CSortedNegLinker* opConIt = nextConcept->getOperandList(); opConIt; opConIt = opConIt->getNext()) {
CConcept* opConcept = opConIt->getData();
bool opNegation = opConIt->isNegated();
if (nextNegation == opNegation) {
conNegPairList.append(TConceptNegPair(opConcept,opNegation));
}
if (candidateAlternativeExtraction && nextNegation != opNegation) {
conNegPairList.append(TConceptNegPair(opConcept,!opNegation));
}
}
} else if (nextNegation && ((nOpCode == CCAND || nOpCode == CCEQ) && opCount > 1) || !nextNegation && nOpCode == CCOR) {
// saturate all disjuncts to extract common concepts from all disjunctions
totallyPreCompItem->addSaturatedDisjunction(nextConcept);
for (CSortedNegLinker* opConIt = nextConcept->getOperandList(); opConIt; opConIt = opConIt->getNext()) {
CConcept* opConcept = opConIt->getData();
bool opNegation = opConIt->isNegated()^nextNegation;
bool opCheckingNegation = opNegation;
CConcept* opCheckConcept = getDisjunctCheckingConcept(opConcept, opNegation, &opCheckingNegation);
CConceptProcessData* conProcData = (CConceptProcessData*)opCheckConcept->getConceptData();
CConceptSaturationReferenceLinkingData* conRefSatLinking = (CConceptSaturationReferenceLinkingData*)conProcData->getConceptReferenceLinking();
if (!conRefSatLinking) {
conRefSatLinking = new CConceptSaturationReferenceLinkingData();
conProcData->setConceptReferenceLinking(conRefSatLinking);
}
if (conRefSatLinking->getConceptSaturationReferenceLinkingData(opCheckingNegation) == nullptr) {
CSaturationConceptDataItem* conItem = totallyPreCompItem->getSaturationConceptDataItem(opCheckConcept, opCheckingNegation,true);
conRefSatLinking->setSaturationReferenceLinkingData(conItem, opCheckingNegation);
newDisjunctionCandidateAlternativeList->append(conItem);
if (!opCheckConcept->hasClassName() || opCheckingNegation) {
conItem->setInvalidSpecialItemReference(true);
}
}
}
CConceptProcessData* conProcData = (CConceptProcessData*)nextConcept->getConceptData();
CConceptSaturationReferenceLinkingData* conRefSatLinking = (CConceptSaturationReferenceLinkingData*)conProcData->getConceptReferenceLinking();
if (!conRefSatLinking) {
conRefSatLinking = new CConceptSaturationReferenceLinkingData();
conProcData->setConceptReferenceLinking(conRefSatLinking);
}
if (conRefSatLinking->getConceptSaturationReferenceLinkingData(nextNegation) == nullptr) {
CSaturationConceptDataItem* conItem = totallyPreCompItem->getSaturationConceptDataItem(nextConcept,nextNegation,true);
conRefSatLinking->setSaturationReferenceLinkingData(conItem,nextNegation);
newDisjunctionCandidateAlternativeList->append(conItem);
conItem->setInvalidSpecialItemReference(true);
}
} else if (!nextNegation && (nOpCode == CCALL) || nextNegation && (nOpCode == CCSOME || nOpCode == CCAQSOME)) {
if (candidateAlternativeExtraction) {
// saturate pseudo models to invalidate candidates concepts
bool saturateNegation = !nextNegation;
CConceptProcessData* conProcData = (CConceptProcessData*)nextConcept->getConceptData();
CConceptSaturationReferenceLinkingData* conRefSatLinking = (CConceptSaturationReferenceLinkingData*)conProcData->getConceptReferenceLinking();
if (!conRefSatLinking) {
conRefSatLinking = new CConceptSaturationReferenceLinkingData();
conProcData->setConceptReferenceLinking(conRefSatLinking);
}
if (conRefSatLinking->getConceptSaturationReferenceLinkingData(saturateNegation) == nullptr) {
CSaturationConceptDataItem* conItem = totallyPreCompItem->getSaturationConceptDataItem(nextConcept,saturateNegation,true);
conRefSatLinking->setSaturationReferenceLinkingData(conItem,saturateNegation);
newDisjunctionCandidateAlternativeList->append(conItem);
conItem->setInvalidSpecialItemReference(true);
}
}
}
}
}
bool CTotallyPrecomputationThread::saturateRemainingRequiredItems(CTotallyOntologyPrecomputationItem* totallyPreCompItem) {
cint64 saturatingConceptCount = 0;
bool allConSaturReq = totallyPreCompItem->requiresAllConceptsSaturation();
QList* reqConSatList = totallyPreCompItem->getRemainingRequiredSaturationConceptList();
if (allConSaturReq) {
totallyPreCompItem->setAllConceptsSaturationOrdered(true);
QList* satOrderdItemList = totallyPreCompItem->getSaturationOrderedItemList();
for (QList::const_iterator it = satOrderdItemList->constBegin(), itEnd = satOrderdItemList->constEnd(); it != itEnd; ++it) {
CSaturationConceptDataItem* satConItem(*it);
if (!satConItem->isItemProcessingMarked()) {
++saturatingConceptCount;
satConItem->setItemProcessingMarked(true);
}
}
} else {
for (QList::const_iterator it = reqConSatList->constBegin(), itEnd = reqConSatList->constEnd(); it != itEnd; ++it) {
const TConceptNegPair& conNegPair(*it);
CConcept* concept(conNegPair.first);
bool negation = conNegPair.second;
saturatingConceptCount += markSaturationProcessingItems(totallyPreCompItem,nullptr,concept,negation);
}
}
reqConSatList->clear();
if (saturatingConceptCount > 0 || totallyPreCompItem->requiresNominalDelayedConceptsSaturationUpdate() && !totallyPreCompItem->hasNominalDelayedConceptsSaturationUpdated()) {
totallyPreCompItem->setNominalDelayedConceptsSaturationUpdateRequired(false);
totallyPreCompItem->setNominalDelayedConceptsSaturationUpdated(true);
createMarkedConceptSaturationProcessingJob(totallyPreCompItem,true);
return true;
}
return false;
}
bool CTotallyPrecomputationThread::addRequiredSaturationIndividuals(CTotallyOntologyPrecomputationItem* totallyPreCompItem) {
CIndividualVector* indiVec = totallyPreCompItem->getOntology()->getABox()->getIndividualVector(false);
QSet* activeIndiSet = totallyPreCompItem->getOntology()->getABox()->getActiveIndividualSet(false);
bool individualsAdded = false;
bool saturateConcepts = CConfigDataReader::readConfigBoolean(totallyPreCompItem->getCalculationConfiguration(),"Konclude.Calculation.Optimization.ConceptSaturation",true);
bool individualSaturation = CConfigDataReader::readConfigBoolean(totallyPreCompItem->getCalculationConfiguration(),"Konclude.Calculation.Optimization.IndividualSaturation",false);
cint64 maxId = totallyPreCompItem->getOntology()->getABox()->getNextIndividualId(false);
mBackendAssocCache->mDebugOntology = totallyPreCompItem->getOntology();
mBackendAssocCache->initializeIndividualsAssociationCaching(totallyPreCompItem->getOntology()->getOntologyID(), maxId);
if (saturateConcepts && individualSaturation) {
if (indiVec) {
cint64 indiCount = indiVec->getItemCount();
cint64 indiBatchCount = 5000;
if (indiCount > indiBatchCount * 2) {
class CIndividualLinkerPair {
public:
CIndividualLinkerPair() {
}
CIndividualLinkerPair(CXLinker* firstlinker, CXLinker* lastlinker, CXLinker* linkerArray, cint64 indiLinkerCount) {
mFirstLinker = firstlinker;
mLastLinker = lastlinker;
mLinkerArray = linkerArray;
mIndiLinkerCount = indiLinkerCount;
}
CXLinker* mFirstLinker = nullptr;
CXLinker* mLastLinker = nullptr;
CXLinker* mLinkerArray = nullptr;
cint64 mIndiLinkerCount = 0;
};
function& indiPosPair)> individualMapFunc =
[&](const QPair& indiPosPair) -> CIndividualLinkerPair {
cint64 indiStartPos = indiPosPair.first;
cint64 indiEndPos = indiPosPair.second;
CXLinker* firstlinker = nullptr;
CXLinker* lastlinker = nullptr;
CXLinker* linkerArray = new CXLinker[indiBatchCount];
cint64 nextLinkerPos = 0;
cint64 indiLinkerCount = 0;
for (cint64 indiId = indiStartPos; indiId < indiEndPos; ++indiId) {
CIndividual* indi = indiVec->getData(indiId);
if (indi && activeIndiSet->contains(indi)) {
CXLinker* linker = &linkerArray[nextLinkerPos++];
linker->initLinker(indi);
if (!firstlinker) {
firstlinker = linker;
lastlinker = linker;
} else {
linker->setNext(firstlinker);
firstlinker = linker;
}
indiLinkerCount++;
individualsAdded = true;
}
}
return CIndividualLinkerPair(firstlinker, lastlinker, linkerArray, indiLinkerCount);
};
function individualReduceFunc = [&](CIndividualLinkerPair& reducePair, const CIndividualLinkerPair& pair) -> void {
CXLinker* firstIndiLinker = pair.mFirstLinker;
CXLinker* lastIndiLinker = pair.mLastLinker;
if (firstIndiLinker && lastIndiLinker) {
if (reducePair.mFirstLinker) {
lastIndiLinker->setNext(reducePair.mFirstLinker);
reducePair.mFirstLinker = firstIndiLinker;
} else {
reducePair.mFirstLinker = firstIndiLinker;
reducePair.mLastLinker = lastIndiLinker;
}
reducePair.mIndiLinkerCount += pair.mIndiLinkerCount;
totallyPreCompItem->addRequiredABoxSaturationIndividualLinkerArray(reducePair.mLinkerArray);
}
};
QList> indiPosPairList;
for (cint64 nextIndiPos = 0; nextIndiPos < maxId; nextIndiPos += indiBatchCount) {
indiPosPairList.append(QPair(nextIndiPos, nextIndiPos + indiBatchCount));
}
CIndividualLinkerPair indiLinkerPair = QtConcurrent::blockingMappedReduced(indiPosPairList, individualMapFunc, individualReduceFunc);
if (indiLinkerPair.mFirstLinker) {
individualsAdded = true;
}
totallyPreCompItem->setRequiredABoxSaturationIndividualLinker(indiLinkerPair.mFirstLinker);
totallyPreCompItem->setRemainingRequiredABoxSaturationIndividualCount(indiLinkerPair.mIndiLinkerCount);
} else {
QSet assConNegPairSet;
cint64 collectedIndiCount = 0;
for (cint64 indiId = 0; indiId < indiCount; ++indiId) {
CIndividual* indi = indiVec->getData(indiId);
if (indi && activeIndiSet->contains(indi)) {
totallyPreCompItem->addRequiredABoxSaturationIndividual(indi);
individualsAdded = true;
++collectedIndiCount;
}
}
totallyPreCompItem->setRemainingRequiredABoxSaturationIndividualCount(collectedIndiCount);
}
}
totallyPreCompItem->setIndividualsSaturated(true);
} else {
totallyPreCompItem->setIndividualsSaturated(false);
}
return individualsAdded;
}
bool CTotallyPrecomputationThread::addIdentifiedRemainingConsistencyRequiredConcepts(CTotallyOntologyPrecomputationItem* totallyPreCompItem) {
bool constConReqSatAdded = false;
bool saturateConcepts = CConfigDataReader::readConfigBoolean(totallyPreCompItem->getCalculationConfiguration(),"Konclude.Calculation.Optimization.ConceptSaturation",true);
if (saturateConcepts) {
CConcept* topConcept = totallyPreCompItem->getOntology()->getTBox()->getTopConcept();
if (topConcept->getOperandList() != nullptr) {
totallyPreCompItem->addConsistencyRequiredSaturationConcept(topConcept,false);
constConReqSatAdded = true;
}
CIndividualVector* indiVec = totallyPreCompItem->getOntology()->getABox()->getIndividualVector(false);
QSet* activeIndiSet = totallyPreCompItem->getOntology()->getABox()->getActiveIndividualSet(false);
QSet assConNegPairSet;
QSet assRoleSet;
QSet dataRoleSet;
bool createAllAssertionIndi = false;
if (indiVec) {
cint64 indiCount = indiVec->getItemCount();
for (cint64 indiId = 0; indiId < indiCount; ++indiId) {
CIndividual* indi = indiVec->getData(indiId);
if (indi && activeIndiSet->contains(indi)) {
createAllAssertionIndi = true;
for (CConceptAssertionLinker* conAssLinkIt = indi->getAssertionConceptLinker(); conAssLinkIt; conAssLinkIt = conAssLinkIt->getNext()) {
CConcept* assCon = conAssLinkIt->getData();
bool assConNeg = conAssLinkIt->isNegated();
if (assCon->getOperatorCode() == CCNOMINAL) {
if (assCon->getNominalIndividual() != indi || assConNeg && assCon->getNominalIndividual() == indi) {
assConNegPairSet.insert(TConceptNegPair(assCon,assConNeg));
}
} else {
assConNegPairSet.insert(TConceptNegPair(assCon,assConNeg));
}
}
for (CRoleAssertionLinker* roleAssLinkIt = indi->getAssertionRoleLinker(); roleAssLinkIt; roleAssLinkIt = roleAssLinkIt->getNext()) {
CRole* role = roleAssLinkIt->getRole();
assRoleSet.insert(role);
}
for (CDataAssertionLinker* dataAssLinkIt = indi->getAssertionDataLinker(); dataAssLinkIt; dataAssLinkIt = dataAssLinkIt->getNext()) {
CRole* role = dataAssLinkIt->getRole();
dataRoleSet.insert(role);
}
}
}
}
COntologyTriplesData* ontTripleData = totallyPreCompItem->getOntology()->getOntologyTriplesData();
if (ontTripleData && ontTripleData->getTripleAssertionAccessor()) {
COntologyTriplesAssertionsAccessor* assertionAccessor = ontTripleData->getTripleAssertionAccessor();
if (assertionAccessor->getIndexedIndividualCount() > 0) {
createAllAssertionIndi = true;
CTriplesAssertedConceptsExtendingSetVisitor assertedConceptsVisitor(&assConNegPairSet);
assertionAccessor->visitAllAssertedConcepts(&assertedConceptsVisitor);
CTriplesAssertedRolesExtendingSetVisitor assertedRolesVisitor(&assRoleSet);
assertionAccessor->visitAllAssertedObjectRoles(&assertedRolesVisitor);
CTriplesAssertedRolesExtendingSetVisitor assertedDataRolesVisitor(&dataRoleSet);
assertionAccessor->visitAllAssertedDataRoles(&assertedDataRolesVisitor);
}
}
if (!dataRoleSet.isEmpty()) {
for (CRole* dataRole : dataRoleSet) {
for (CSortedNegLinker* superDataRoleIt = dataRole->getIndirectSuperRoleList(); superDataRoleIt; superDataRoleIt = superDataRoleIt->getNext()) {
CRole* superDataRole = superDataRoleIt->getData();
for (CSortedNegLinker* domainConLinkerIt = superDataRole->getDomainRangeConceptList(superDataRoleIt->isNegated()); domainConLinkerIt; domainConLinkerIt = domainConLinkerIt->getNext()) {
CConcept* domainCon = domainConLinkerIt->getData();
bool domainConNegation = domainConLinkerIt->isNegated();
assConNegPairSet.insert(TConceptNegPair(domainCon, domainConNegation));
}
}
}
totallyPreCompItem->setAllAssertionIndividualSufficientSaturationChecked(true);
totallyPreCompItem->setAllAssertionIndividualSaturationSufficient(false);
}
if (createAllAssertionIndi && (!assConNegPairSet.isEmpty() || !assRoleSet.isEmpty())) {
CIndividual* tmpAllAssertionIndi = new CIndividual();
tmpAllAssertionIndi->initIndividual(totallyPreCompItem->getOntology()->getABox()->getNextIndividualId(false));
tmpAllAssertionIndi->setTemporaryFakeIndividual(true);
for (QSet::const_iterator it = assConNegPairSet.constBegin(), itEnd = assConNegPairSet.constEnd(); it != itEnd; ++it) {
constConReqSatAdded = true;
TConceptNegPair assConNegPair(*it);
totallyPreCompItem->addConsistencyRequiredSaturationConcept(assConNegPair.first, assConNegPair.second);
CConceptAssertionLinker* tmpConAssLinker = new CConceptAssertionLinker();
tmpConAssLinker->initNegLinker(assConNegPair.first, assConNegPair.second);
tmpAllAssertionIndi->addAssertionConceptLinker(tmpConAssLinker);
}
for (QSet::const_iterator it = assRoleSet.constBegin(), itEnd = assRoleSet.constEnd(); it != itEnd; ++it) {
CRole* role(*it);
CRoleAssertionLinker* tmpRoleAssLinker = new CRoleAssertionLinker();
tmpRoleAssLinker->initRoleAssertionLinker(role, tmpAllAssertionIndi);
tmpAllAssertionIndi->addAssertionRoleLinker(tmpRoleAssLinker);
}
totallyPreCompItem->setAllAssertionIndividual(tmpAllAssertionIndi);
}
}
return constConReqSatAdded;
}
bool CTotallyPrecomputationThread::saturateRemainingConsistencyRequiredConcepts(CTotallyOntologyPrecomputationItem* totallyPreCompItem) {
cint64 saturatingConceptCount = 0;
QList* reqConSatList = totallyPreCompItem->getRemainingConsistencyRequiredSaturationConceptList();
for (QList::const_iterator it = reqConSatList->constBegin(), itEnd = reqConSatList->constEnd(); it != itEnd; ++it) {
const TConceptNegPair& conNegPair(*it);
CConcept* concept(conNegPair.first);
bool negation = conNegPair.second;
saturatingConceptCount += markSaturationProcessingItems(totallyPreCompItem,nullptr,concept,negation);
}
reqConSatList->clear();
if (saturatingConceptCount > 0) {
createMarkedConceptSaturationProcessingJob(totallyPreCompItem,false);
return true;
}
return false;
}
bool CTotallyPrecomputationThread::saturateRemainingRequiredSaturationIndividuals(CTotallyOntologyPrecomputationItem* totallyPreCompItem) {
cint64 saturatingIndividualCount = 0;
QTime*& satTimer = totallyPreCompItem->getIndividualSaturationTime();
if (!satTimer) {
satTimer = new QTime();
satTimer->start();
}
cint64 individualSaturationCount = totallyPreCompItem->getIndividualSaturationSizeLimit();
QList indiSatList;
QList* reqIndiSatList = totallyPreCompItem->getRemainingRequiredABoxSaturationIndividuals();
while (!reqIndiSatList->isEmpty() && saturatingIndividualCount < individualSaturationCount) {
CIndividual* individual = reqIndiSatList->takeFirst();
indiSatList.append(individual);
totallyPreCompItem->setHandledTriplesIndividualSaturatedId(qMax(individual->getIndividualID(), totallyPreCompItem->getHandledTriplesIndividualSaturatedId()));
++saturatingIndividualCount;
}
while (totallyPreCompItem->hasRequiredABoxSaturationIndividualLinker() && saturatingIndividualCount < individualSaturationCount) {
CIndividual* individual = totallyPreCompItem->getNextRequiredABoxSaturationIndividual();
indiSatList.append(individual);
totallyPreCompItem->setHandledTriplesIndividualSaturatedId(qMax(individual->getIndividualID(), totallyPreCompItem->getHandledTriplesIndividualSaturatedId()));
++saturatingIndividualCount;
}
if (!totallyPreCompItem->hasRequiredABoxSaturationIndividualLinker()) {
totallyPreCompItem->clearRequiredABoxSaturationIndividualLinkerArrays();
}
totallyPreCompItem->decRemainingRequiredABoxSaturationIndividualCount(saturatingIndividualCount);
cint64 maxTriplesIndexedIndiId = -1;
COntologyTriplesAssertionsAccessor* triplesAccessor = totallyPreCompItem->getOntology()->getOntologyTriplesData()->getTripleAssertionAccessor();
if (triplesAccessor) {
maxTriplesIndexedIndiId = totallyPreCompItem->getOntology()->getOntologyTriplesData()->getTripleAssertionAccessor()->getMaxIndexedIndividualId();
}
cint64 handledTriplesIndexedIndis = totallyPreCompItem->getHandledTriplesIndividualSaturatedId();
bool remainingTripleIndexedIndis = handledTriplesIndexedIndis < maxTriplesIndexedIndiId;
if (saturatingIndividualCount <= 0 && remainingTripleIndexedIndis) {
cint64 newHandledCount = handledTriplesIndexedIndis + individualSaturationCount;
newHandledCount = qMin(newHandledCount, maxTriplesIndexedIndiId);
createTripleIndexedIndividualsSaturationProcessingJob(totallyPreCompItem, handledTriplesIndexedIndis + 1, newHandledCount);
handledTriplesIndexedIndis = newHandledCount;
totallyPreCompItem->setHandledTriplesIndividualSaturatedId(handledTriplesIndexedIndis);
remainingTripleIndexedIndis = handledTriplesIndexedIndis < maxTriplesIndexedIndiId;
if (!totallyPreCompItem->hasRemainingRequiredABoxSaturationIndividuals() && !remainingTripleIndexedIndis) {
totallyPreCompItem->setALLIndividualsSaturationOrderd(true);
totallyPreCompItem->setTripleIndexedIndividualSaturated(true);
}
return true;
} else {
if (!totallyPreCompItem->hasRemainingRequiredABoxSaturationIndividuals() && !remainingTripleIndexedIndis) {
totallyPreCompItem->setALLIndividualsSaturationOrderd(true);
totallyPreCompItem->setTripleIndexedIndividualSaturated(true);
}
createIndividualSaturationProcessingJob(totallyPreCompItem, indiSatList);
return true;
}
return false;
}
cint64 CTotallyPrecomputationThread::markSaturationProcessingItems(CTotallyOntologyPrecomputationItem* totallyPreCompItem, CSaturationConceptDataItem* startMarkingItem, CConcept* startMarkingConcept, bool startMarkingConceptNegation) {
cint64 markedConcepts = 0;
QList itemList;
QList conNegPairList;
if (startMarkingItem && !startMarkingItem->isItemProcessingMarked()) {
++markedConcepts;
startMarkingItem->setItemProcessingMarked(true);
itemList.append(startMarkingItem);
}
if (startMarkingConcept) {
CSaturationConceptDataItem* opMarkingItem = totallyPreCompItem->getSaturationConceptDataItem(startMarkingConcept,startMarkingConceptNegation,false);
if (opMarkingItem) {
if (!opMarkingItem->isItemProcessingMarked()) {
++markedConcepts;
opMarkingItem->setItemProcessingMarked(true);
itemList.append(opMarkingItem);
}
} else {
conNegPairList.append(TConceptNegPair(startMarkingConcept,startMarkingConceptNegation));
}
}
while (!itemList.isEmpty() || !conNegPairList.isEmpty()) {
while (!itemList.isEmpty()) {
CSaturationConceptDataItem* item = itemList.takeFirst();
CConcept* nextConcept = item->getSaturationConcept();
bool nextNegation = item->getSaturationNegation();
conNegPairList.append(TConceptNegPair(nextConcept,nextNegation));
QList* existRefItemList = item->getExistReferenceConceptItemList();
for (QList::const_iterator it2 = existRefItemList->constBegin(), it2End = existRefItemList->constEnd(); it2 != it2End; ++it2) {
CSaturationConceptDataItem* existRefItem = *it2;
if (!existRefItem->isItemProcessingMarked()) {
++markedConcepts;
existRefItem->setItemProcessingMarked(true);
itemList.append(existRefItem);
}
}
}
while (!conNegPairList.isEmpty()) {
TConceptNegPair nextConNegPair(conNegPairList.takeFirst());
CConcept* nextConcept = nextConNegPair.first;
bool nextNegation = nextConNegPair.second;
cint64 nOpCode = nextConcept->getOperatorCode();
cint64 opCount = nextConcept->getOperandCount();
if (!nextNegation && (nOpCode == CCAND || nOpCode == CCSUB || nOpCode == CCIMPLTRIG || nOpCode == CCEQ || (nOpCode == CCOR && opCount == 1) || nOpCode == CCAQSOME || nOpCode == CCSOME || nOpCode == CCATLEAST) || nextNegation && (nOpCode == CCOR || ((nOpCode == CCAND || nOpCode == CCEQ) && opCount == 1) || nOpCode == CCALL || nOpCode == CCATMOST)) {
for (CSortedNegLinker* opConIt = nextConcept->getOperandList(); opConIt; opConIt = opConIt->getNext()) {
CConcept* opConcept = opConIt->getData();
bool influenceNegation = true;
if (nOpCode == CCAQSOME || nOpCode == CCATLEAST || nOpCode == CCATMOST) {
influenceNegation = false;
}
bool opNegation = opConIt->isNegated();
if (influenceNegation && nextNegation) {
opNegation = !opNegation;
}
bool opConceptMarked = false;
if (nOpCode != CCAQCHOOCE || !opNegation) {
CSaturationConceptDataItem* opMarkingItem = totallyPreCompItem->getSaturationConceptDataItem(opConcept,opNegation,false);
CSaturationConceptDataItem* roleOpMarkingItem = nullptr;
if (nextConcept->getRole()) {
CRole* role = nextConcept->getRole();
roleOpMarkingItem = totallyPreCompItem->getSaturationRoleSuccessorConceptDataItem(role,opConcept,opNegation,false);
}
if (opMarkingItem || roleOpMarkingItem) {
if (opMarkingItem && !opMarkingItem->isItemProcessingMarked()) {
++markedConcepts;
opMarkingItem->setItemProcessingMarked(true);
itemList.append(opMarkingItem);
}
if (roleOpMarkingItem && !roleOpMarkingItem->isItemProcessingMarked()) {
++markedConcepts;
roleOpMarkingItem->setItemProcessingMarked(true);
itemList.append(roleOpMarkingItem);
}
} else {
conNegPairList.append(TConceptNegPair(opConcept,opNegation));
}
}
}
} else if (nOpCode == CCAQCHOOCE) {
for (CSortedNegLinker* opConIt = nextConcept->getOperandList(); opConIt; opConIt = opConIt->getNext()) {
CConcept* opConcept = opConIt->getData();
if (opConIt->isNegated() == nextNegation) {
CSaturationConceptDataItem* opMarkingItem = totallyPreCompItem->getSaturationConceptDataItem(opConcept,false,false);
if (opMarkingItem) {
if (!opMarkingItem->isItemProcessingMarked()) {
++markedConcepts;
opMarkingItem->setItemProcessingMarked(true);
itemList.append(opMarkingItem);
}
} else {
conNegPairList.append(TConceptNegPair(opConcept,false));
}
}
}
}
}
}
return markedConcepts;
}
void CTotallyPrecomputationThread::orderItemsSaturationTesting(CTotallyOntologyPrecomputationItem* totallyPreCompItem, CSaturationConceptDataItem* ontConSatDataItem, QList& orderedItemList, QStack& saturationItemStack) {
if (!ontConSatDataItem->isOrderingQueued()) {
saturationItemStack.push(CSaturationSaturationListItem(ontConSatDataItem));
while (!saturationItemStack.isEmpty()) {
CSaturationSaturationListItem& topSatSatListItem = saturationItemStack.top();
CSaturationConceptDataItem* topOntConSatDataItem = topSatSatListItem.mSaturationItem;
QList& markOderingSubsumerList = topSatSatListItem.mSaturationItemList;
if (!topOntConSatDataItem->isOrderingQueued()) {
if (!topSatSatListItem.mSubsumerOrdered) {
topSatSatListItem.mSubsumerOrdered = true;
// first mark all subsumer items
CSaturationConceptDataItem* subsumerItem = topOntConSatDataItem;
while (subsumerItem && !subsumerItem->isOrderingQueued()) {
subsumerItem->setOrderingQueued(true);
markOderingSubsumerList.prepend(subsumerItem);
subsumerItem = subsumerItem->getSpecialItemReference();
}
}
}
if (topSatSatListItem.mLastSubsumerSaturationItem) {
orderedItemList.append(topSatSatListItem.mLastSubsumerSaturationItem);
topSatSatListItem.mLastSubsumerSaturationItem = nullptr;
}
if (!markOderingSubsumerList.isEmpty()) {
CSaturationConceptDataItem* subsumerItem = markOderingSubsumerList.takeFirst();
topSatSatListItem.mLastSubsumerSaturationItem = subsumerItem;
QList* existRefItemList = subsumerItem->getExistReferenceConceptItemList();
for (QList::const_iterator it2 = existRefItemList->constBegin(), it2End = existRefItemList->constEnd(); it2 != it2End; ++it2) {
CSaturationConceptDataItem* existRefItem = *it2;
saturationItemStack.push(CSaturationSaturationListItem(existRefItem));
}
//QListIterator it2(*existRefItemList);
//it2.toBack();
//while (it2.hasPrevious()) {
// CSaturationConceptDataItem* existRefItem = it2.previous();
// saturationItemStack.push(CSaturationSaturationListItem(existRefItem));
//}
} else {
saturationItemStack.pop();
}
}
}
//if (!ontConSatDataItem->isOrderingQueued()) {
// // first mark all subsumer items
// QList markOderingSubsumerList;
// CSaturationConceptDataItem* subsumerItem = ontConSatDataItem;
// while (subsumerItem && !subsumerItem->isOrderingQueued()) {
// subsumerItem->setOrderingQueued(true);
// markOderingSubsumerList.prepend(subsumerItem);
// subsumerItem = subsumerItem->getSpecialItemReference();
// }
// for (QList::const_iterator it1 = markOderingSubsumerList.constBegin(), it1End = markOderingSubsumerList.constEnd(); it1 != it1End; ++it1) {
// CSaturationConceptDataItem* subsumerItem = *it1;
// QList* existRefItemList = subsumerItem->getExistReferenceConceptItemList();
// for (QList::const_iterator it2 = existRefItemList->constBegin(), it2End = existRefItemList->constEnd(); it2 != it2End; ++it2) {
// CSaturationConceptDataItem* existRefItem = *it2;
// orderItemsSaturationTesting(totallyPreCompItem,existRefItem,orderedItemList);
// }
// orderedItemList.append(subsumerItem);
// }
//}
}
bool CTotallyPrecomputationThread::hasRoleRanges(CTotallyOntologyPrecomputationItem* totallyPreCompItem, CRole* role) {
for (CSortedNegLinker* superRoleIt = role->getIndirectSuperRoleList(); superRoleIt; superRoleIt = superRoleIt->getNext()) {
CRole* superRole = superRoleIt->getData();
bool superRoleNegation = superRoleIt->isNegated();
CSortedNegLinker* rangeConceptLinker = superRole->getDomainRangeConceptList(!superRoleNegation);
if (rangeConceptLinker) {
return true;
}
}
return false;
}
void CTotallyPrecomputationThread::createMarkedConceptSaturationProcessingJob(CTotallyOntologyPrecomputationItem* totallyPreCompItem, bool allowAllSaturation) {
CApproximatedSaturationCalculationJob* satCalculationJob = nullptr;
CConcreteOntology *onto = totallyPreCompItem->getOntology();
CApproximatedSaturationCalculationJobGenerator satCalculationJobGenerator(onto);
satCalculationJob = satCalculationJobGenerator.getApproximatedSaturationCalculationJob(0, totallyPreCompItem->getSaturationData());
bool imiadelyProcessAllConcepts = totallyPreCompItem->requiresAllConceptsSaturation();
if (!allowAllSaturation) {
imiadelyProcessAllConcepts = false;
}
cint64 saturationConceptCount = 0;
QList* orderedItemList = totallyPreCompItem->getSaturationOrderedItemList();
for (QList::const_iterator it = orderedItemList->constBegin(), itEnd = orderedItemList->constEnd(); it != itEnd; ++it) {
CSaturationConceptDataItem* conItem = *it;
CConcept* concept = conItem->getSaturationConcept();
bool negation = conItem->getSaturationNegation();
CRole* role = conItem->getSaturationRoleRanges();
if (conItem->isItemProcessingMarked() && !conItem->isItemProcessingQueued() || imiadelyProcessAllConcepts) {
conItem->setItemProcessingQueued(true);
satCalculationJob = satCalculationJobGenerator.extendApproximatedSaturationCalculationJobProcessing(conItem, satCalculationJob);
saturationConceptCount++;
}
}
if (!totallyPreCompItem->isAllAssertionIndividualSaturated()) {
CIndividual* allAssertionIndi = totallyPreCompItem->getAllAssertionIndividual();
if (allAssertionIndi) {
satCalculationJob = satCalculationJobGenerator.extendApproximatedSaturationCalculationJobProcessing(allAssertionIndi, nullptr, satCalculationJob);
}
totallyPreCompItem->setAllAssertionIndividualSaturated(true);
}
LOG(INFO, getLogDomain(), logTr("Scheduled %1 concepts for saturation for ontology '%2'.").arg(saturationConceptCount).arg(totallyPreCompItem->getOntology()->getTerminologyName()), this);
CSaturationPrecomputationTestingItem* satTestingItem = new CSaturationPrecomputationTestingItem(totallyPreCompItem, CPrecomputationTestingItem::CONCEPTSATURATIONPRECOMPUTATIONTYPE);
CCalculationConfigurationExtension* calcConfig = totallyPreCompItem->getCalculationConfiguration();
satCalculationJob->setCalculationConfiguration(calcConfig);
satCalculationJob->setSaturationTaskPreyingAdapter(new CTaskPreyingAdapter((CSaturationObserver*)totallyPreCompItem));
totallyPreCompItem->setSaturationCalculationJob(satCalculationJob);
processCalculationJob(satCalculationJob, totallyPreCompItem, satTestingItem);
}
void CTotallyPrecomputationThread::createOccurrenceStatisticsCollectingSaturationProcessingJob(CTotallyOntologyPrecomputationItem* totallyPreCompItem) {
CApproximatedSaturationCalculationJob* satCalculationJob = nullptr;
CConcreteOntology *onto = totallyPreCompItem->getOntology();
CApproximatedSaturationCalculationJobGenerator satCalculationJobGenerator(onto);
satCalculationJob = satCalculationJobGenerator.getApproximatedSaturationCalculationJob(0,totallyPreCompItem->getSaturationData());
CSaturationPrecomputationTestingItem* satTestingItem = new CSaturationPrecomputationTestingItem(totallyPreCompItem,CPrecomputationTestingItem::OCCURRENCESTATISTICSSATURATIONPRECOMPUTATIONTYPE);
CCalculationConfigurationExtension* calcConfig = totallyPreCompItem->getCalculationConfiguration();
satCalculationJob->setCalculationConfiguration(calcConfig);
satCalculationJob->setOccurrenceStatisticsCollectingAdapter(new CSaturationOccurrenceStatisticsCollectingAdapter());
totallyPreCompItem->setSaturationCalculationJob(satCalculationJob);
processCalculationJob(satCalculationJob,totallyPreCompItem,satTestingItem);
}
void CTotallyPrecomputationThread::createIndividualSaturationProcessingJob(CTotallyOntologyPrecomputationItem* totallyPreCompItem, const QList& individualList) {
CApproximatedSaturationCalculationJob* satCalculationJob = nullptr;
CConcreteOntology *onto = totallyPreCompItem->getOntology();
CApproximatedSaturationCalculationJobGenerator satCalculationJobGenerator(onto);
satCalculationJob = satCalculationJobGenerator.getApproximatedSaturationCalculationJob(0,totallyPreCompItem->getSaturationData());
satCalculationJob->setSeparateSaturation(true);
cint64 nextSaturationID = totallyPreCompItem->getNextSaturationID();
CSaturationIndividualDataItem* satIndiDataItemLinker = nullptr;
cint64 saturationIndividualCount = 0;
for (QList::const_iterator it = individualList.constBegin(), itEnd = individualList.constEnd(); it != itEnd; ++it) {
CIndividual* individual = *it;
CSaturationIndividualDataItem* satIndiDataItem = totallyPreCompItem->takeFreeSaturationIndividualDataItem();
satIndiDataItem->initConceptSaturationTestingItem(individual,nextSaturationID);
satIndiDataItemLinker = satIndiDataItem->append(satIndiDataItemLinker);
CIndividualProcessData* indiProcessData = (CIndividualProcessData*)individual->getIndividualData();
if (indiProcessData) {
indiProcessData->setSaturationReferenceLinkingData(satIndiDataItem);
}
satCalculationJob = satCalculationJobGenerator.extendApproximatedSaturationCalculationJobProcessing(individual,satIndiDataItem,satCalculationJob);
++saturationIndividualCount;
}
totallyPreCompItem->setSaturationIDIndividualDataItems(nextSaturationID,satIndiDataItemLinker);
cint64 remainingRequiredABoxSaturationIndividualCount = totallyPreCompItem->getRemainingRequiredABoxSaturationIndividualCount();
LOG(INFO,getLogDomain(),logTr("Scheduled next %1 of %2 remaining individuals for saturation of ontology '%3' (%4 batchs queued).").arg(saturationIndividualCount).arg(saturationIndividualCount + remainingRequiredABoxSaturationIndividualCount).arg(totallyPreCompItem->getOntology()->getTerminologyName()).arg(mCurrRunningTestParallelCount + 1),this);
CSaturationPrecomputationTestingItem* satTestingItem = new CSaturationPrecomputationTestingItem(totallyPreCompItem,CPrecomputationTestingItem::INDIVIDUALSATURATIONPRECOMPUTATIONTYPE,nextSaturationID);
CCalculationConfigurationExtension* calcConfig = totallyPreCompItem->getCalculationConfiguration();
satCalculationJob->setCalculationConfiguration(calcConfig);
satCalculationJob->setSaturationIndividualsAnalysationObserver(new CSaturationIndividualsAnalysingAdapter((CSaturationIndividualAnalysationObserver*)totallyPreCompItem));
totallyPreCompItem->setSaturationCalculationJob(satCalculationJob);
processCalculationJob(satCalculationJob,totallyPreCompItem,satTestingItem);
}
void CTotallyPrecomputationThread::createTripleIndexedIndividualsSaturationProcessingJob(CTotallyOntologyPrecomputationItem* totallyPreCompItem, cint64 startIndiId, cint64 endIndiId) {
CApproximatedSaturationCalculationJob* satCalculationJob = nullptr;
CConcreteOntology *onto = totallyPreCompItem->getOntology();
CApproximatedSaturationCalculationJobGenerator satCalculationJobGenerator(onto);
satCalculationJob = satCalculationJobGenerator.getApproximatedSaturationCalculationJob(0,totallyPreCompItem->getSaturationData());
satCalculationJob->setSeparateSaturation(true);
cint64 nextSaturationID = totallyPreCompItem->getNextSaturationID();
CSaturationIndividualDataItem* satIndiDataItemLinker = nullptr;
cint64 saturationIndividualCount = 0;
for (cint64 indiId = startIndiId; indiId <= endIndiId; ++indiId) {
CSaturationIndividualDataItem* satIndiDataItem = totallyPreCompItem->takeFreeSaturationIndividualDataItem();
satIndiDataItem->initConceptSaturationTestingItem(nullptr, nextSaturationID);
satIndiDataItemLinker = satIndiDataItem->append(satIndiDataItemLinker);
satCalculationJob = satCalculationJobGenerator.extendApproximatedSaturationCalculationJobProcessing(indiId, satIndiDataItem, satCalculationJob);
++saturationIndividualCount;
}
totallyPreCompItem->setSaturationIDIndividualDataItems(nextSaturationID,satIndiDataItemLinker);
LOG(INFO,getLogDomain(),logTr("Scheduled next %1 of %2 remaining triple-indexed individuals for saturation of ontology '%3' (%4 batchs queued).").arg(saturationIndividualCount).arg(saturationIndividualCount + totallyPreCompItem->getOntology()->getOntologyTriplesData()->getTripleAssertionAccessor()->getMaxIndexedIndividualId() - endIndiId).arg(totallyPreCompItem->getOntology()->getTerminologyName()).arg(mCurrRunningTestParallelCount + 1),this);
CSaturationPrecomputationTestingItem* satTestingItem = new CSaturationPrecomputationTestingItem(totallyPreCompItem,CPrecomputationTestingItem::INDIVIDUALSATURATIONPRECOMPUTATIONTYPE,nextSaturationID);
CCalculationConfigurationExtension* calcConfig = totallyPreCompItem->getCalculationConfiguration();
satCalculationJob->setCalculationConfiguration(calcConfig);
satCalculationJob->setSaturationIndividualsAnalysationObserver(new CSaturationIndividualsAnalysingAdapter((CSaturationIndividualAnalysationObserver*)totallyPreCompItem));
totallyPreCompItem->setSaturationCalculationJob(satCalculationJob);
processCalculationJob(satCalculationJob,totallyPreCompItem,satTestingItem);
}
void CTotallyPrecomputationThread::createSaturationConstructionJob(CTotallyOntologyPrecomputationItem* totallyPreCompItem) {
CTBox* tBox = totallyPreCompItem->getOntology()->getDataBoxes()->getTBox();
CConcreteOntology *onto = totallyPreCompItem->getOntology();
CConcept* topConcept = onto->getDataBoxes()->getTopConcept();
CConcept* bottomConcept = onto->getDataBoxes()->getBottomConcept();
CConcept* topDataRangeConcept = tBox->getTopDataRangeConcept();
CConceptVector* conVec = tBox->getConceptVector();
LOG(INFO,getLogDomain(),logTr("Determine concepts for saturation for ontology '%1'.").arg(totallyPreCompItem->getOntology()->getTerminologyName()),this);
CApproximatedSaturationCalculationJob* satCalculationJob = nullptr;
CApproximatedSaturationCalculationJobGenerator satCalculationJobGenerator(onto);
bool saturateConcepts = CConfigDataReader::readConfigBoolean(totallyPreCompItem->getCalculationConfiguration(),"Konclude.Calculation.Optimization.ConceptSaturation",true);
cint64 statExistsConceptCount = 0;
cint64 statClassCount = 0;
QList existConList;
QList analyseExistSubsumerList;
QList allConItemList;
QList disjunctionCandidateAlternativeList;
cint64 conCount = conVec->getItemCount();
if (saturateConcepts) {
for (cint64 conIdx = 1; conIdx < conCount; ++conIdx) {
CConcept* concept = conVec->getData(conIdx);
//if (CIRIName::getRecentIRIName(concept->getClassNameLinker()) == "file:F:/Projects/OIL/DAMLOilEd/ontologies/ka.daml#Publication") {
// bool bug = true;
//}
if (concept) {
cint64 opCode = concept->getOperatorCode();
if (opCode == CCSOME || opCode == CCALL || opCode == CCAQSOME || opCode == CCATLEAST || opCode == CCATMOST) {
++statExistsConceptCount;
bool negation = opCode == CCALL;
CRole* role = concept->getRole();
CSortedNegLinker* opLinker = concept->getOperandList();
CConcept* opConcept = nullptr;
bool opNegation = false;
if (opLinker) {
opConcept = opLinker->getData();
if (opCode == CCATLEAST || opCode == CCATMOST) {
opNegation = opLinker->isNegated();
} else {
opNegation = opLinker->isNegated()^negation;
}
} else {
opConcept = topConcept;
if (opCode == CCATLEAST || opCode == CCATMOST) {
opNegation = false;
} else {
opNegation = negation;
}
}
if (hasRoleRanges(totallyPreCompItem,role)) {
CSaturationConceptDataItem* conItem = nullptr;
conItem = totallyPreCompItem->getSaturationRoleSuccessorConceptDataItem(role,opConcept,opNegation,true);
conItem->setPotentiallyExistInitializationConcept(true);
existConList.append(conItem);
if (role->isDataRole()) {
conItem->setDataRangeConcept(true);
}
CConceptProcessData* conProcData = (CConceptProcessData*)concept->getConceptData();
CConceptSaturationReferenceLinkingData* conRefSatLinking = (CConceptSaturationReferenceLinkingData*)conProcData->getConceptReferenceLinking();
if (!conRefSatLinking) {
conRefSatLinking = new CConceptSaturationReferenceLinkingData();
conProcData->setConceptReferenceLinking(conRefSatLinking);
}
if (conRefSatLinking->getExistentialSuccessorConceptSaturationReferenceLinkingData() == nullptr) {
conRefSatLinking->setExistentialSuccessorConceptSaturationReferenceLinkingData(conItem);
allConItemList.append(conItem);
}
} else {
CSaturationConceptDataItem* conItem = nullptr;
conItem = totallyPreCompItem->getSaturationConceptDataItem(opConcept,opNegation,true);
conItem->setPotentiallyExistInitializationConcept(true);
existConList.append(conItem);
if (role->isDataRole()) {
conItem->setDataRangeConcept(true);
}
CConceptProcessData* opConProcData = (CConceptProcessData*)opConcept->getConceptData();
CConceptSaturationReferenceLinkingData* opConRefSatLinking = (CConceptSaturationReferenceLinkingData*)opConProcData->getConceptReferenceLinking();
if (!opConRefSatLinking) {
opConRefSatLinking = new CConceptSaturationReferenceLinkingData();
opConProcData->setConceptReferenceLinking(opConRefSatLinking);
}
if (opConRefSatLinking->getConceptSaturationReferenceLinkingData(opNegation) == nullptr) {
opConRefSatLinking->setSaturationReferenceLinkingData(conItem,opNegation);
allConItemList.append(conItem);
}
}
} else if (concept->hasClassName() || concept == topDataRangeConcept) {
++statClassCount;
CConceptProcessData* conProcData = (CConceptProcessData*)concept->getConceptData();
CConceptSaturationReferenceLinkingData* conRefSatLinking = (CConceptSaturationReferenceLinkingData*)conProcData->getConceptReferenceLinking();
if (!conRefSatLinking) {
conRefSatLinking = new CConceptSaturationReferenceLinkingData();
conProcData->setConceptReferenceLinking(conRefSatLinking);
}
if (conRefSatLinking->getConceptSaturationReferenceLinkingData(false) == nullptr) {
CSaturationConceptDataItem* conItem = totallyPreCompItem->getSaturationConceptDataItem(concept,false,true);
conRefSatLinking->setSaturationReferenceLinkingData(conItem,false);
allConItemList.append(conItem);
}
}
}
}
}
disjunctionCandidateAlternativeList = allConItemList;
while (!disjunctionCandidateAlternativeList.isEmpty()) {
CSaturationConceptDataItem* conTestItem = disjunctionCandidateAlternativeList.takeFirst();
QList tempNewItems;
extendDisjunctionsCandidateAlternativesItems(totallyPreCompItem,conTestItem,&tempNewItems);
if (!tempNewItems.isEmpty()) {
disjunctionCandidateAlternativeList.append(tempNewItems);
allConItemList.append(tempNewItems);
}
}
analyseExistSubsumerList = allConItemList;
QList processList;
// identify all items, which aren't subsumers of other items
for (QList::const_iterator it = analyseExistSubsumerList.constBegin(), itEnd = analyseExistSubsumerList.constEnd(); it != itEnd; ++it) {
CSaturationConceptDataItem* conTestItem = *it;
CConcept* concept = conTestItem->getSaturationConcept();
bool conceptNegation = conTestItem->getSaturationNegation();
processList.append(CConceptNegationSaturationItem(concept,conceptNegation,conTestItem));
}
analyseConceptSaturationSubsumerExistItems(totallyPreCompItem,processList);
// find all potentially subsumer items
for (QList::const_iterator it = analyseExistSubsumerList.constBegin(), itEnd = analyseExistSubsumerList.constEnd(); it != itEnd; ++it) {
CSaturationConceptDataItem* conTestItem = *it;
propagateSubsumerItemFlag(conTestItem);
}
// find all potentially role successors, for which back propagation is necessary
for (QList::const_iterator it = existConList.constBegin(), itEnd = existConList.constEnd(); it != itEnd; ++it) {
CSaturationConceptDataItem* conTestItem = *it;
propagateExistInitializationFlag(conTestItem);
}
QList leafConList;
QList leafExistConList;
// collect all leaf nodes
for (QList::const_iterator it = allConItemList.constBegin(), itEnd = allConItemList.constEnd(); it != itEnd; ++it) {
CSaturationConceptDataItem* conTestItem = *it;
if (!conTestItem->hasIndirectSuccessorsItems()) {
if (!conTestItem->isPotentiallyExistInitializationConcept()) {
leafConList.append(conTestItem);
} else {
leafExistConList.append(conTestItem);
}
}
}
QList* orderedItemList = totallyPreCompItem->getSaturationOrderedItemList();
QStack saturationItemStack;
// order all concepts associated to the items for efficient Saturation testing, start with the real leaf items
for (QList::const_iterator it = leafConList.constBegin(), itEnd = leafConList.constEnd(); it != itEnd; ++it) {
CSaturationConceptDataItem* conTestItem = *it;
if (!conTestItem->isOrderingQueued()) {
orderItemsSaturationTesting(totallyPreCompItem,conTestItem,*orderedItemList,saturationItemStack);
}
}
for (QList::const_iterator it = leafExistConList.constBegin(), itEnd = leafExistConList.constEnd(); it != itEnd; ++it) {
CSaturationConceptDataItem* conTestItem = *it;
if (!conTestItem->isOrderingQueued()) {
orderItemsSaturationTesting(totallyPreCompItem,conTestItem,*orderedItemList,saturationItemStack);
}
}
for (QList::const_iterator it = allConItemList.constBegin(), itEnd = allConItemList.constEnd(); it != itEnd; ++it) {
CSaturationConceptDataItem* conTestItem = *it;
if (!conTestItem->isOrderingQueued()) {
orderItemsSaturationTesting(totallyPreCompItem,conTestItem,*orderedItemList,saturationItemStack);
}
}
// set reference mode for saturation: substitute or copy
QListIterator reverseOrderedIt(*orderedItemList);
reverseOrderedIt.toBack();
CBOXHASH* triggerImpHash = totallyPreCompItem->getOntology()->getTBox()->getTriggerImplicationHash(false);
while (reverseOrderedIt.hasPrevious()) {
CSaturationConceptDataItem* conTestItem = reverseOrderedIt.previous();
CSaturationConceptDataItem* referencedTestItem = conTestItem->getSpecialItemReference();
if (referencedTestItem) {
if (!conTestItem->isPotentiallyExistInitializationConcept() && !conTestItem->hasMultipleDirectPredecessorsItems() && (!triggerImpHash || !triggerImpHash->contains(conTestItem->getSaturationConcept()))) {
conTestItem->setSpecialItemReferenceMode(CSaturationConceptDataItem::SATURATIONSUBSTITUTEMODE);
} else {
conTestItem->setSpecialItemReferenceMode(CSaturationConceptDataItem::SATURATIONCOPYMODE);
}
}
}
satCalculationJob = satCalculationJobGenerator.getApproximatedSaturationCalculationJob(0);
cint64 mSaturationConceptCount = 0;
for (QList::const_iterator it = orderedItemList->constBegin(), itEnd = orderedItemList->constEnd(); it != itEnd; ++it) {
CSaturationConceptDataItem* conItem = *it;
CConcept* concept = conItem->getSaturationConcept();
bool negation = conItem->getSaturationNegation();
CRole* role = conItem->getSaturationRoleRanges();
conItem->setItemProcessingQueued(false);
satCalculationJob = satCalculationJobGenerator.extendApproximatedSaturationCalculationJobConstruction(role,concept,negation,false,conItem,satCalculationJob);
mSaturationConceptCount++;
}
LOG(INFO,getLogDomain(),logTr("Preparing saturation of %1 concepts for ontology '%2'.").arg(mSaturationConceptCount).arg(totallyPreCompItem->getOntology()->getTerminologyName()),this);
CSaturationPrecomputationTestingItem* satTestingItem = new CSaturationPrecomputationTestingItem(totallyPreCompItem,CPrecomputationTestingItem::CONCEPTSATURATIONPRECOMPUTATIONTYPE);
CCalculationConfigurationExtension* calcConfig = totallyPreCompItem->getCalculationConfiguration();
satCalculationJob->setCalculationConfiguration(calcConfig);
satCalculationJob->setSaturationTaskPreyingAdapter(new CTaskPreyingAdapter((CSaturationObserver*)totallyPreCompItem));
totallyPreCompItem->setSaturationCalculationJob(satCalculationJob);
processCalculationJob(satCalculationJob,totallyPreCompItem,satTestingItem);
}
bool CTotallyPrecomputationThread::precomputationTested(COntologyPrecomputationItem* ontPreCompItem, CPrecomputationTestingItem* preTestItem, CSaturationPrecomputationCalculatedCallbackEvent* pcce) {
CTotallyOntologyPrecomputationItem* totallyPreCompItem = (CTotallyOntologyPrecomputationItem*)ontPreCompItem;
if (preTestItem->getPrecomputationTestingType() == CPrecomputationTestingItem::OCCURRENCESTATISTICSSATURATIONPRECOMPUTATIONTYPE) {
totallyPreCompItem->setSaturationOccurrenceStatisticsCollected(true);
return true;
} else if (preTestItem->getPrecomputationTestingType() == CPrecomputationTestingItem::CONCEPTSATURATIONPRECOMPUTATIONTYPE) {
CSaturationPrecomputationTestingItem* satPreTestItem = (CSaturationPrecomputationTestingItem*)preTestItem;
totallyPreCompItem->setConceptSaturationPrecomputationChecked(true);
CPrecomputation* precomputation = totallyPreCompItem->getPrecomputation();
CSaturationData* saturationData = totallyPreCompItem->getSaturationData();
precomputation->setSaturationModelData(saturationData);
totallyPreCompItem->setSaturationComputationRunning(false);
CSaturationTaskData* satTaskData = (CSaturationTaskData*)saturationData;
if (satTaskData) {
CSatisfiableCalculationTask* satCalcTask = satTaskData->getSaturationTask();
if (satCalcTask) {
CProcessingDataBox* procDataBox = satCalcTask->getProcessingDataBox();
if (procDataBox) {
totallyPreCompItem->setNominalDelayedConceptsSaturationUpdateRequired(procDataBox->isDelayedNominalProcessingOccured());
}
}
}
return true;
} else if (preTestItem->getPrecomputationTestingType() == CPrecomputationTestingItem::INDIVIDUALSATURATIONPRECOMPUTATIONTYPE) {
CSaturationPrecomputationTestingItem* satPreTestItem = (CSaturationPrecomputationTestingItem*)preTestItem;
cint64 saturationID = satPreTestItem->getSaturationID();
totallyPreCompItem->releaseSaturationIDIndividualDataItems(saturationID);
totallyPreCompItem->decIndividualSaturationRunningCount();
if (!totallyPreCompItem->hasIndividualSaturationRunning()) {
totallyPreCompItem->setSaturationComputationRunning(false);
if (totallyPreCompItem->hasIndividualsSaturated() && totallyPreCompItem->hasALLIndividualsSaturationOrderd()) {
if (!totallyPreCompItem->hasIndividualsSaturationCacheSynchronisation()) {
CIndividualPrecomputationCoordinationHash* procCoordHash = totallyPreCompItem->getPrecomputationProcessingCoordinationHash();
if (!totallyPreCompItem->isPrecompuationRetrievingIncompletelyHandledIndividuals()) {
if (!procCoordHash || procCoordHash->isEmpty()) {
synchronouslyRetrieveIndividualsPrecomputation(totallyPreCompItem, ontPreCompItem);
} else {
QTime*& indiPrecTimer = totallyPreCompItem->getIndividualPrecomputationTime();
if (!indiPrecTimer) {
indiPrecTimer = new QTime();
indiPrecTimer->start();
}
totallyPreCompItem->setIndividualsSaturationCacheSynchronisation(true);
QTime* indiSatTimer = totallyPreCompItem->getIndividualSaturationTime();
LOG(INFO, getLogDomain(), logTr("Individual saturation finished in %2, preparing processing of %1 previusly received insufficiently saturated individuals.").arg(procCoordHash->size()).arg(indiSatTimer->elapsed()), getLogObject());
}
} else {
QTime* indiSatTimer = totallyPreCompItem->getIndividualSaturationTime();
LOG(INFO, getLogDomain(), logTr("Individual saturation finished in %1 ms, waiting first insufficient individuals retrieval.").arg(indiSatTimer->elapsed()), getLogObject());
}
}
CConsistence* consistence = totallyPreCompItem->getConsistence();
if (consistence) {
consistence->setIndividualsRepresentativelyStored(true);
if (!totallyPreCompItem->hasInsufficientSaturationIndividuals()) {
consistence->setAllIndividualsRepresentativelyStored(true);
}
}
}
}
if (mConfPrecomputationIndividualsRetrievalWhileSaturation && !totallyPreCompItem->hasIndividualsSaturationCacheSynchronisation() && totallyPreCompItem->hasALLIndividualsSaturationOrderd() && !totallyPreCompItem->hasIndividualsSaturationAllOrderedCacheRetrieved() && totallyPreCompItem->hasInsufficientSaturationIndividuals()) {
bool fullCompletionGraphConstruction = totallyPreCompItem->isFullCompletionGraphConstruction();
if (!fullCompletionGraphConstruction) {
totallyPreCompItem->setIndividualsSaturationAllOrderedCacheRetrieved(true);
requestIndividualsPrecomputationRetrieval(totallyPreCompItem, false);
}
}
return true;
}
return false;
}
void CTotallyPrecomputationThread::synchronouslyRetrieveIndividualsPrecomputation(CTotallyOntologyPrecomputationItem* totallyPreCompItem, COntologyPrecomputationItem* ontPreCompItem) {
totallyPreCompItem->setIndividualsSaturationCacheSynchronisation(true);
cint64 limit = totallyPreCompItem->getCurrentIncompletelyHandledIndividualRetrievalLimit();
bool fullCompletionGraphConstruction = totallyPreCompItem->isFullCompletionGraphConstruction();
if (fullCompletionGraphConstruction) {
limit = -1;
}
totallyPreCompItem->setPrecompuationRetrievingIncompletelyHandledIndividuals(true);
totallyPreCompItem->setFirstIncompletelyHandledIndividualsRetrieved(true);
CIndividualPrecomputationCoordinationHash* newRetrievalCoordHash = new CIndividualPrecomputationCoordinationHash();
newRetrievalCoordHash->incUsageCount();
mBackendAssocCache->getIncompletlyAssociationCachedIndividuals(ontPreCompItem->getOntology()->getOntologyID(), nullptr, newRetrievalCoordHash, true, false, limit);
totallyPreCompItem->setPrecompuationRetrievingIncompletelyHandledIndividuals(false);
if (newRetrievalCoordHash->size() > 0) {
totallyPreCompItem->setRemainingIncompletelyHandlingIndividualComputationLimitIncreasingCount(newRetrievalCoordHash->getApproximateTotalIncompletelyHandledCount());
QTime*& indiPrecTimer = totallyPreCompItem->getIndividualPrecomputationTime();
if (!indiPrecTimer) {
indiPrecTimer = new QTime();
indiPrecTimer->start();
}
QTime* indiSatTimer = totallyPreCompItem->getIndividualSaturationTime();
LOG(INFO, getLogDomain(), logTr("Individual saturation finished in %2, preparing processing of first %1 insufficiently saturated individuals.").arg(newRetrievalCoordHash->size()).arg(indiSatTimer->elapsed()), getLogObject());
totallyPreCompItem->setPrecomputationProcessingCoordinationHash(newRetrievalCoordHash);
totallyPreCompItem->setPrecomputationProcessingCoordinationHashIteratorCurrent(newRetrievalCoordHash->constBegin());
totallyPreCompItem->setPrecomputationProcessingCoordinationHashIteratorEnd(newRetrievalCoordHash->constEnd());
} else {
QTime* indiSatTimer = totallyPreCompItem->getIndividualSaturationTime();
LOG(INFO, getLogDomain(), logTr("Individual saturation finished in %1 ms without insufficient individuals.").arg(indiSatTimer->elapsed()), getLogObject());
decrementUsageFromIndividualPrecomputationCoordinationHash(newRetrievalCoordHash, totallyPreCompItem);
totallyPreCompItem->setAllIncompletelyHandledIndividualsRetrieved(true);
}
}
bool CTotallyPrecomputationThread::retrieveIndividualsPrecomputation(CTotallyOntologyPrecomputationItem* totallyPreCompItem) {
if (!totallyPreCompItem->isPrecompuationRetrievingIncompletelyHandledIndividuals()) {
totallyPreCompItem->setPrecompuationRetrievingIncompletelyHandledIndividuals(true);
cint64 limit = totallyPreCompItem->getCurrentIncompletelyHandledIndividualRetrievalLimit();
CIndividualPrecomputationCoordinationHash* newRetrievalCoordHash = new CIndividualPrecomputationCoordinationHash();
newRetrievalCoordHash->incUsageCount();
totallyPreCompItem->setPrecomputationRetrievalCoordinationHash(newRetrievalCoordHash);
bool refillRetrievalCoordHash = totallyPreCompItem->getNonPrecompuationDirectIncompletelyHandledIndividualsRetrievingStreakSize() > 1;
mBackendAssocCache->getIncompletlyAssociationCachedIndividuals(totallyPreCompItem->getOntology()->getOntologyID(), totallyPreCompItem->getPrecomputationProcessingCoordinationHash(), newRetrievalCoordHash, true, refillRetrievalCoordHash, limit);
precomputationIndividualsRetrieved(totallyPreCompItem, true);
return true;
} else {
return false;
}
}
bool CTotallyPrecomputationThread::requestIndividualsPrecomputationRetrieval(CTotallyOntologyPrecomputationItem* totallyPreCompItem, bool allIndividualsSaturated) {
if (!totallyPreCompItem->isPrecompuationRetrievingIncompletelyHandledIndividuals()) {
totallyPreCompItem->setPrecompuationRetrievingIncompletelyHandledIndividuals(true);
cint64 limit = totallyPreCompItem->getCurrentIncompletelyHandledIndividualRetrievalLimit();
CIndividualPrecomputationCoordinationHash* newRetrievalCoordHash = new CIndividualPrecomputationCoordinationHash();
newRetrievalCoordHash->incUsageCount();
totallyPreCompItem->setPrecomputationRetrievalCoordinationHash(newRetrievalCoordHash);
CIndividualPrecomputationCoordinationHash* procCoordHash = totallyPreCompItem->getPrecomputationProcessingCoordinationHash();
mBackendAssocCache->getIncompletlyAssociationCachedIndividuals(totallyPreCompItem->getOntology()->getOntologyID(), procCoordHash, newRetrievalCoordHash, allIndividualsSaturated, false, limit, new CRetrievedPrecomputationIndividualsCallbackEvent(this, totallyPreCompItem));
if (procCoordHash) {
LOG(INFO, getLogDomain(), logTr("Requesting more insufficiently handled individuals from representative cache, only %1 of appxoximately %2 remaining, %3 computing, %4 processed.").arg(totallyPreCompItem->getPrecomputationProcessingCoordinationHash()->getHashRemainingCount())
.arg(totallyPreCompItem->getPrecomputationProcessingCoordinationHash()->getApproximateRemainingIncompletelyHandledCount()).arg(totallyPreCompItem->getPrecomputationProcessingCoordinationHash()->getHashComputationCount()).arg(totallyPreCompItem->getPrecomputationProcessingCoordinationHash()->getHashProcessedCount()), getLogObject());
} else {
LOG(INFO, getLogDomain(), logTr("Requesting first insufficiently handled individuals from representative cache."), getLogObject());
}
return true;
} else {
return false;
}
}
bool CTotallyPrecomputationThread::precomputationIndividualsRetrieved(COntologyPrecomputationItem* ontPreCompItem, CRetrievedPrecomputationIndividualsCallbackEvent* pcce) {
CTotallyOntologyPrecomputationItem* totallyPreCompItem = (CTotallyOntologyPrecomputationItem*)ontPreCompItem;
precomputationIndividualsRetrieved(totallyPreCompItem, false);
CIndividualPrecomputationCoordinationHash* coordHash = totallyPreCompItem->getPrecomputationProcessingCoordinationHash();
cint64 computationLimit = totallyPreCompItem->getCurrentIncompletelyHandledIndividualComputationLimit();
if (coordHash && coordHash->getHashRemainingCount() >= computationLimit || !totallyPreCompItem->isIndividualComputationRunning()) {
doNextPendingTests();
}
return true;
}
bool CTotallyPrecomputationThread::precomputationIndividualsRetrieved(COntologyPrecomputationItem* ontPreCompItem, bool directly) {
CTotallyOntologyPrecomputationItem* totallyPreCompItem = (CTotallyOntologyPrecomputationItem*)ontPreCompItem;
CIndividualPrecomputationCoordinationHash* prevHash = totallyPreCompItem->getPrecomputationProcessingCoordinationHash();
CIndividualPrecomputationCoordinationHash* newHash = totallyPreCompItem->getPrecomputationRetrievalCoordinationHash();
totallyPreCompItem->setPrecomputationProcessingCoordinationHashIteratorCurrent(newHash->constBegin());
totallyPreCompItem->setPrecomputationProcessingCoordinationHashIteratorEnd(newHash->constEnd());
totallyPreCompItem->setPrecompuationIncompletelyHandledIndividualsRetrievalThresholdRequested(false);
totallyPreCompItem->setCurrentIncompletelyHandledIndividualRetrievalThreshold(newHash->getHashRemainingCount() * totallyPreCompItem->getCurrentIncompletelyHandledIndividualRetrievalThresholdFactor());
if (prevHash) {
#ifdef KONCLUDE_DEBUG_PRECOMPUTATION_SCHEDULING_LOGS
LOG(INFO, getLogDomain(), logTr("prev hash: %1 insufficiently handled individuals, now at %2 computation, %3 processed, %4 correction, %5 total insufficient, %6 approx remain insufficient.").arg(prevHash->getHashRemainingCount()).arg(prevHash->getHashComputationCount()).arg(prevHash->getHashProcessedCount()).arg(prevHash->getApproximateCorrectionIncompletelyHandledCount()).arg(prevHash->getApproximateTotalIncompletelyHandledCount()).arg(prevHash->getApproximateRemainingIncompletelyHandledCount()), getLogObject());
#endif // KONCLUDE_DEBUG_PRECOMPUTATION_SCHEDULING_LOGS
newHash->incHashRemainingCount(prevHash->getHashRemainingCount());
newHash->incHashProcessedCount(prevHash->getHashProcessedCount());
newHash->incHashComputationCount(prevHash->getHashComputationCount());
}
#ifdef KONCLUDE_DEBUG_PRECOMPUTATION_SCHEDULING_LOGS
LOG(INFO, getLogDomain(), logTr("new hash: %1 insufficiently handled individuals, now at %2 computation, %3 processed, %4 correction, %5 total insufficient, %6 approx remain insufficient.").arg(newHash->getHashRemainingCount()).arg(newHash->getHashComputationCount()).arg(newHash->getHashProcessedCount()).arg(newHash->getApproximateCorrectionIncompletelyHandledCount()).arg(newHash->getApproximateTotalIncompletelyHandledCount()).arg(newHash->getApproximateRemainingIncompletelyHandledCount()), getLogObject());
#endif // KONCLUDE_DEBUG_PRECOMPUTATION_SCHEDULING_LOGS
totallyPreCompItem->setPrecomputationRetrievalCoordinationHash(nullptr);
totallyPreCompItem->setPrecomputationProcessingCoordinationHash(newHash);
totallyPreCompItem->setPrecompuationRetrievingIncompletelyHandledIndividuals(false);
if (!totallyPreCompItem->hasFirstIncompletelyHandledIndividualsRetrieved()) {
totallyPreCompItem->setFirstIncompletelyHandledIndividualsRetrieved(true);
totallyPreCompItem->setRemainingIncompletelyHandlingIndividualComputationLimitIncreasingCount(newHash->getApproximateTotalIncompletelyHandledCount());
if (!totallyPreCompItem->hasIndividualSaturationRunning()) {
if (newHash->getHashRemainingCount() <= 0) {
synchronouslyRetrieveIndividualsPrecomputation(totallyPreCompItem, ontPreCompItem);
} else {
QTime*& indiPrecTimer = totallyPreCompItem->getIndividualPrecomputationTime();
if (!indiPrecTimer) {
indiPrecTimer = new QTime();
indiPrecTimer->start();
}
totallyPreCompItem->setIndividualsSaturationCacheSynchronisation(true);
LOG(INFO, getLogDomain(), logTr("Retrieved first %1 insufficiently handled individuals from representative cache, scheduling processing.").arg(newHash->getHashRemainingCount()), getLogObject());
}
}
} else {
if (directly) {
LOG(INFO, getLogDomain(), logTr("Retrieved %1 insufficiently handled individuals directly from representative cache, scheduling processing.").arg(newHash->getHashRemainingCount()), getLogObject());
} else {
LOG(INFO, getLogDomain(), logTr("Retrieved %1 insufficiently handled individuals asynchronly from representative cache, scheduling processing.").arg(newHash->getHashRemainingCount()), getLogObject());
}
}
decrementUsageFromIndividualPrecomputationCoordinationHash(prevHash, totallyPreCompItem);
return true;
}
bool CTotallyPrecomputationThread::decrementUsageFromIndividualPrecomputationCoordinationHash(CIndividualPrecomputationCoordinationHash* indiPrecompCoordHash, CTotallyOntologyPrecomputationItem* totallyPreCompItem) {
if (indiPrecompCoordHash) {
indiPrecompCoordHash->decUsageCount();
if (indiPrecompCoordHash->getUsageCount() == 0) {
for (CIndividualPrecomputationCoordinationHash::iterator it = indiPrecompCoordHash->begin(), itEnd = indiPrecompCoordHash->end(); it != itEnd; ++it) {
CBackendIndividualRetrievalComputationUpdateCoordinationHashData*& data = it.value();
data->decUsageCount();
if (data->getUsageCount() == 0) {
delete data;
data = nullptr;
}
}
delete indiPrecompCoordHash;
return true;
}
}
return false;
}
bool CTotallyPrecomputationThread::determineMinimumNextConceptID(CConsistence* consistence, COntologyPrecomputationItem* ontPreCompItem) {
CConsistenceTaskData* consTaskData = (CConsistenceTaskData*)consistence->getConsistenceModelData();
if (consTaskData) {
CSatisfiableCalculationTask* detTask = consTaskData->getDeterministicSatisfiableTask();
CSatisfiableCalculationTask* nonDetTask = consTaskData->getCompletionGraphCachedSatisfiableTask();
CConceptVector* extendedConVec = nonDetTask->getProcessingDataBox()->getExtendedConceptVector(false);
if (extendedConVec) {
cint64 minNextId = extendedConVec->getItemCount();
ontPreCompItem->getOntology()->getTBox()->setMinimalNextConceptID(minNextId);
return true;
}
}
return false;
}
bool CTotallyPrecomputationThread::precomputationTested(COntologyPrecomputationItem* ontPreCompItem, CPrecomputationTestingItem* preTestItem, CPrecomputationCalculatedCallbackEvent* pcce) {
CTotallyOntologyPrecomputationItem* totallyPreCompItem = (CTotallyOntologyPrecomputationItem*)ontPreCompItem;
if (preTestItem->getPrecomputationTestingType() == CPrecomputationTestingItem::CONSISTENCEPRECOMPUTATIONTYPE) {
CConsistencePrecomputationTestingItem* consPreTestItem = (CConsistencePrecomputationTestingItem*)preTestItem;
totallyPreCompItem->setIndividualStepRunning(false);
totallyPreCompItem->setIndividualPrecomputationChecked(true);
totallyPreCompItem->setConsistenceCheched(true);
if (pcce->hasCalculationError()) {
LOG(ERROR,getLogDomain(),logTr("Error in computation, consistency testing for ontology '%1' failed.").arg(ontPreCompItem->getOntology()->getTerminologyName()),getLogObject());
totallyPreCompItem->getConsistencePrecomputationStep()->submitRequirementsUpdate(COntologyProcessingStatus::PSFAILED);
totallyPreCompItem->getIndividualPrecomputationStep()->submitRequirementsUpdate(COntologyProcessingStatus::PSFAILED);
} else {
CConsistence* consistence = totallyPreCompItem->getConsistence();
consistence->setConsistenceModelData(totallyPreCompItem->getConsistenceData());
consistence->setOntologyConsistent(pcce->getTestResultSatisfiable());
determineMinimumNextConceptID(consistence, ontPreCompItem);
if (consistence->isOntologyConsistent()) {
CIndividualPrecomputationCoordinationHash* newRetrievalCoordHash = new CIndividualPrecomputationCoordinationHash();
mBackendAssocCache->getIncompletlyAssociationCachedIndividuals(totallyPreCompItem->getOntology()->getOntologyID(), nullptr, newRetrievalCoordHash, true, false, -1);
if (!newRetrievalCoordHash->isEmpty()) {
LOG(WARN, getLogDomain(), logTr("Representative cache has incompletely processed individuals although individual computation finished and ontology is consistent."), getLogObject());
}
delete newRetrievalCoordHash;
}
}
return true;
} else if (preTestItem->getPrecomputationTestingType() == CPrecomputationTestingItem::INDIVIDUALPRECOMPUTATIONTYPE) {
CIndividualPrecomputationTestingItem* indiPreTestItem = (CIndividualPrecomputationTestingItem*)preTestItem;
totallyPreCompItem->decIndividualComputationRunningCount();
bool expansionPropagationCutted = indiPreTestItem->getRepresentativeCacheUpdatingAdapter()->isExpansionPropagationCutted();
if (expansionPropagationCutted) {
totallyPreCompItem->incRecomputationExpansionPropagationCuttedCount(1);
totallyPreCompItem->incCurrentRecomputationExpansionPropagationCuttedReductionFactor(totallyPreCompItem->getRecomputationExpansionPropagationCuttedReductionIncreaseFactor());
} else {
totallyPreCompItem->decCurrentRecomputationExpansionPropagationCuttedReductionFactor(totallyPreCompItem->getRecomputationExpansionPropagationCuttedReductionRecoveryFactor());
}
QMap* recomIdTestingItemMap = totallyPreCompItem->getRecomputationIdTestingItemMap();
recomIdTestingItemMap->remove(indiPreTestItem->getRepresentativeCacheUpdatingAdapter()->getRepresentativeCacheRecomputationId());
totallyPreCompItem->decMaxHandledRecomputationIdRemainingReporting();
if (totallyPreCompItem->getMaxHandledRecomputationIdRemainingReporting() <= 0 && !recomIdTestingItemMap->isEmpty()) {
cint64 firstRecomputationId = recomIdTestingItemMap->constBegin().key();
totallyPreCompItem->setMaxHandledRecomputationIdRemainingReporting(50);
mBackendAssocCache->reportMaximumHandledRecomputationId(totallyPreCompItem->getOntology()->getOntologyID(), firstRecomputationId-1);
}
CIndividualPrecomputationCoordinationHash* precomputationProcessingCoordinationHash = totallyPreCompItem->getPrecomputationProcessingCoordinationHash();
if (!precomputationProcessingCoordinationHash) {
precomputationProcessingCoordinationHash = indiPreTestItem->getIndividualComputationCoordinationHash();
}
QSet* currentIndiComputationSet = totallyPreCompItem->getCurrentIndividualComputationSet();
QList* computedIndiList = indiPreTestItem->getIndividualComputationList();
for (CIndividualReference indiRef : *indiPreTestItem->getIndividualComputationList()) {
//currentIndiComputationSet->remove(indiRef);
if (precomputationProcessingCoordinationHash) {
precomputationProcessingCoordinationHash->decHashComputationCount();
precomputationProcessingCoordinationHash->incHashProcessedCount();
CBackendIndividualRetrievalComputationUpdateCoordinationHashData* data = (*precomputationProcessingCoordinationHash)[indiRef.getIndividualID()];
if (data) {
data->setProcessed(true);
}
}
}
CIndividualPrecomputationCoordinationHash* testPrecomputationProcessingCoordinationHash = indiPreTestItem->getIndividualComputationCoordinationHash();
decrementUsageFromIndividualPrecomputationCoordinationHash(testPrecomputationProcessingCoordinationHash, totallyPreCompItem);
#ifdef KONCLUDE_DEBUG_PRECOMPUTATION_SCHEDULING_LOGS
LOG(INFO, getLogDomain(), logTr("%7 individuals precomputed, remaining coordination hash: %1 insufficiently handled individuals, now at %2 computation, %3 processed, %4 correction, %5 total insufficient, %6 approx remain insufficient.").arg(precomputationProcessingCoordinationHash->getHashRemainingCount()).arg(precomputationProcessingCoordinationHash->getHashComputationCount()).arg(precomputationProcessingCoordinationHash->getHashProcessedCount()).arg(precomputationProcessingCoordinationHash->getApproximateCorrectionIncompletelyHandledCount()).arg(precomputationProcessingCoordinationHash->getApproximateTotalIncompletelyHandledCount()).arg(precomputationProcessingCoordinationHash->getApproximateRemainingIncompletelyHandledCount()).arg(computedIndiList->count()), getLogObject());
#endif // KONCLUDE_DEBUG_PRECOMPUTATION_SCHEDULING_LOGS
if (!pcce->getTestResultSatisfiable()) {
totallyPreCompItem->setIndividualPrecomputationClashed(true);
}
if (!totallyPreCompItem->isIndividualComputationRunning() && totallyPreCompItem->hasAllIndividualPrecomputationCreated()) {
totallyPreCompItem->setIndividualStepRunning(false);
totallyPreCompItem->setIndividualComputationRunning(false);
totallyPreCompItem->setIndividualPrecomputationChecked(true);
totallyPreCompItem->setConsistenceCheched(true);
CConsistence* consistence = totallyPreCompItem->getConsistence();
if (consistence) {
consistence->setAllIndividualsRepresentativelyStored(true);
}
}
if (pcce->hasCalculationError()) {
if (!totallyPreCompItem->hasIndividualPrecomputationClashed()) {
LOG(ERROR, getLogDomain(), logTr("Error in computation, individual precomputation for ontology '%1' failed.").arg(ontPreCompItem->getOntology()->getTerminologyName()), getLogObject());
totallyPreCompItem->getConsistencePrecomputationStep()->submitRequirementsUpdate(COntologyProcessingStatus::PSFAILED);
totallyPreCompItem->getIndividualPrecomputationStep()->submitRequirementsUpdate(COntologyProcessingStatus::PSFAILED);
}
} else if (indiPreTestItem->getTaskPreyingAdapter()) {
CConsistence* consistence = totallyPreCompItem->getConsistence();
if (consistence) {
consistence->setConsistenceModelData(totallyPreCompItem->getConsistenceData());
consistence->setOntologyConsistent(!totallyPreCompItem->hasIndividualPrecomputationClashed());
determineMinimumNextConceptID(consistence, ontPreCompItem);
}
}
return true;
} else if (preTestItem->getPrecomputationTestingType() == CPrecomputationTestingItem::CONCEPTCYCLEPRECOMPUTATIONTYPE) {
CConceptCyclePrecomputationTestingItem* conCyclePreTestItem = (CConceptCyclePrecomputationTestingItem*)preTestItem;
CConceptCycleData* conCycleData = conCyclePreTestItem->getConceptCycleData();
CBOXSET* classCycleSet = conCycleData->getCycleClassSet();
if (pcce->hasCalculationError()) {
LOG(ERROR,getLogDomain(),logTr("Error in computation, cyclic concept graph caching for ontology '%1' failed.").arg(ontPreCompItem->getOntology()->getTerminologyName()),getLogObject());
totallyPreCompItem->setConsistenceCheched(true);
totallyPreCompItem->getCyclePrecomputationStep()->submitRequirementsUpdate(COntologyProcessingStatus::PSFAILED);
} else {
if (pcce->getTestResultSatisfiable()) {
// insert into cache
CTempMemoryPoolContainerAllocationManager tmpMemMan(mContext.getMemoryPoolProvider());
CContextBase* tmpContext = CObjectParameterizingAllocator< CContextBase,CMemoryAllocationManager* >::allocateAndConstructAndParameterize(&tmpMemMan,&tmpMemMan);
CCACHINGLIST* cacheValueList = CObjectParameterizingAllocator< CCACHINGLIST,CContext* >::allocateAndConstructAndParameterize(&tmpMemMan,tmpContext);
for (CBOXSET::const_iterator it = classCycleSet->constBegin(), itEnd = classCycleSet->constEnd(); it != itEnd; ++it) {
CConcept* classConcept(*it);
CCacheValue cacheValue(classConcept->getConceptTag(),(cint64)classConcept,CCacheValue::CACHEVALTAGANDCONCEPT);
cacheValueList->append(cacheValue);
}
CReuseCompletionGraphCacheEntryExpandWriteData* writeData = CObjectAllocator::allocateAndConstruct(&tmpMemMan);
writeData->initExpandWriteData(cacheValueList,cacheValueList,conCyclePreTestItem->getJobInstantiation());
mReuseCompletionGraphCacheWriter->writeExpandCache(writeData,tmpMemMan.takeMemoryPools());
}
}
if (!totallyPreCompItem->hasRemainingConceptCyclePrecomputation() && !totallyPreCompItem->hasCurrentPrecomputationTesting()) {
totallyPreCompItem->setConceptCyclePrecomputationChecked(true);
}
}
return false;
}
bool CTotallyPrecomputationThread::finishOntologyPrecomputation(CTotallyOntologyPrecomputationItem* totallyPreCompItem) {
CConcreteOntology* ontology = totallyPreCompItem->getOntology();
if (totallyPreCompItem->areAllStepFinished()) {
totallyPreCompItem->setPrecomputationFinished(true);
CPrecomputation* precomputation = ontology->getPrecomputation();
precomputation->setPrecomputed(true);
}
CConsistenceStatisticsCollectionStrings* classifStatCollStrings = totallyPreCompItem->getConsistenceStatisticsCollectionStrings();
CPrecomputation* precomputation = ontology->getPrecomputation();
if (precomputation) {
precomputation->setPrecomputationStatistics(classifStatCollStrings);
}
CConsistence* consistence = ontology->getConsistence();
if (consistence) {
consistence->setConsistenceStatistics(classifStatCollStrings);
}
totallyPreCompItem->doPrecomputationFinishedCallback(true);
mActiveOntItemSet.remove(totallyPreCompItem);
mInactiveOntItemSet.insert(totallyPreCompItem);
return true;
}
}; // end namespace Consistiser
}; // end namespace Reasoner
}; // end namespace Konclude