Debian Codegraph @

Source/Reasoner/Answerer/Conclusion/CQtConcurrentStreamingComplexQueryFinishingHandler.cpp

raw
1 /*
2 * Copyright (C) 2013-2015, 2019 by the Konclude Developer Team.
3 *
4 * This file is part of the reasoning system Konclude.
5 * For details and support, see <http://konclude.com/>.
6 *
7 * Konclude is free software: you can redistribute it and/or modify
8 * it under the terms of version 3 of the GNU Lesser General Public
9 * License (LGPLv3) as published by the Free Software Foundation.
10 *
11 * Konclude is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU (Lesser) General Public License for more details.
15 *
16 * You should have received a copy of the GNU (Lesser) General Public
17 * License along with Konclude. If not, see <http://www.gnu.org/licenses/>.
18 *
19 */
20
21 #include "CQtConcurrentStreamingComplexQueryFinishingHandler.h"
22
23
24 namespace Konclude {
25
26 namespace Reasoner {
27
28 namespace Answerer {
29
30 namespace Conclusion {
31
32 CQtConcurrentStreamingComplexQueryFinishingHandler::CQtConcurrentStreamingComplexQueryFinishingHandler(CComplexQueryExpressionProcessingData* queryProcessingData, COptimizedComplexExpressionOntologyAnsweringItem* ontoAnsweringItem, bool confExtendedLogging, const QString& logDomain, cint64 confMaximumBatchMappingsComputationSize, cint64 confFirstBatchMappingsComputationSize, double confBatchMappingsComputationSizeIncreasingFactor, bool confContinueMappingsCompuationWhenResultsNotWriteable, cint64 confMappingsComputationUnlimitedInterpretationSize)
33 : CSequentialStreamingComplexQueryFinishingHandler(queryProcessingData, ontoAnsweringItem, confExtendedLogging, logDomain, confMaximumBatchMappingsComputationSize, confFirstBatchMappingsComputationSize, confBatchMappingsComputationSizeIncreasingFactor, confContinueMappingsCompuationWhenResultsNotWriteable, confMappingsComputationUnlimitedInterpretationSize) {
34
35 mAllowConcurrentAnswerGeneration = true;
36 }
37
38
39 CQtConcurrentStreamingComplexQueryFinishingHandler::~CQtConcurrentStreamingComplexQueryFinishingHandler() {
40 }
41
42
43
44
45
46 void CQtConcurrentStreamingComplexQueryFinishingHandler::processUpdatedBindingsCardinalityLinkers() {
47
48 if (!mAllowConcurrentAnswerGeneration || mCompVarQuery->getResultLimitIncludingOffset() > 0 || mFilteringAnsweringMapping || !mExistBindsAnswersStreamingResult) {
49 mAllowConcurrentAnswerGeneration = false;
50 CSequentialStreamingComplexQueryFinishingHandler::processUpdatedBindingsCardinalityLinkers();
51 }
52
53 COptimizedComplexVariableCompositionItemDependence* varItemProcessingDep = mLastVarItemProcessingDep;
54 if (varItemProcessingDep->getDependentItem()->hasSplitComputations()) {
55 varItemProcessingDep = new COptimizedComplexVariableCompositionItemDependence(varItemProcessingDep->getDependentItem()->getCurrentSplitComputationItem());
56 mSplitComputedMappingCount += varItemProcessingDep->getDependentItem()->getVariableMappingsCurrentCount();
57 }
58
59 if (!mAnswersWriteable && !mSameRealization->hasPotentiallySameIndividuals() && (varItemProcessingDep->getDependentItem()->getVariableMapping()->getMaximumCardinalitySameIndividualsSeparatelyConsidered() <= 1 || mDistinct)) {
60 //TODO: max cardinality must be 1
61 cint64 currentBindingCount = varItemProcessingDep->getDependentItem()->getVariableMapping()->getBindingCount();
62 cint64 writtenResultCount = mLastOnlyCountingWritten;
63 mResultWriter.addReusedVariableBindingAnswerToResultConsideringOffsetLimit(mExistBindsAnswersStreamingResult, nullptr, mFilteringAnsweringMapping, mCompVarQuery, mQueryProcessingData, currentBindingCount - writtenResultCount);
64 if (!varItemProcessingDep->getDependentItem()->isSplitComputationMode()) {
65 mLastOnlyCountingWritten = currentBindingCount;
66 }
67 } else {
68
69 if (mLastOnlyCountingWritten > 0) {
70 // get back to 0 answers
71 mExistBindsAnswersStreamingResult->addResultVariableBindings(nullptr, -mLastOnlyCountingWritten);
72 mLastOnlyCountingWritten = 0;
73 }
74
75 while (!varItemProcessingDep->isBatchProcessed() || varItemProcessingDep->loadNextBatchProvidedBatch()) {
76
77
78
79 cint64 bindingCardLinkerCount = 0;
80 QVector<COptimizedComplexVariableIndividualBindingsCardinalityBatchLinker*> bindingsCardLinkerBatchVec;
81 while (!varItemProcessingDep->isBatchProcessed(false) || varItemProcessingDep->loadNextBatchProvidedBatch(false)) {
82
83 while (varItemProcessingDep->getBatchProvidedBindingCardinalityBatchLinker(false)) {
84 COptimizedComplexVariableIndividualBindingsCardinalityBatchLinker* bindingBatchLinker = varItemProcessingDep->getBatchProvidedBindingCardinalityBatchLinker(true);
85 varItemProcessingDep->setBatchProvidedBindingCardinalityLinkersProcessed(bindingBatchLinker);
86 bindingCardLinkerCount += bindingBatchLinker->getLinkerCount();
87 bindingsCardLinkerBatchVec.append(bindingBatchLinker);
88 }
89
90 }
91 if (!bindingsCardLinkerBatchVec.isEmpty()) {
92 handleBindingCardinalityLinkerBatches(bindingsCardLinkerBatchVec, bindingCardLinkerCount);
93 }
94
95
96 while (varItemProcessingDep->getBatchCurrentUpdatedCardinalityLinker(false)) {
97 COptimizedComplexVariableIndividualUpdateCardinalityLinker* updatedCardinalityLinker = varItemProcessingDep->getBatchCurrentUpdatedCardinalityLinker(true);
98 if (!mDistinct) {
99 COptimizedComplexVariableIndividualBindingsCardinality* prevCardinalites = updatedCardinalityLinker->getPreviousCardinality();
100 COptimizedComplexVariableIndividualBindingsCardinalityLinker* bindingLinker = updatedCardinalityLinker->getUpdatedBindingsCardinalityLinker();
101 COptimizedComplexVariableIndividualBindingsCardinality* newCardinalites = updatedCardinalityLinker->getNewCardinality();
102 COptimizedComplexVariableIndividualBindings* bindings = bindingLinker->getBindings();
103
104 COptimizedComplexVariableIndividualBindingsCardinality tmpDiffCardinalites = *newCardinalites;
105 tmpDiffCardinalites.substractCardinalities(prevCardinalites);
106
107 handleBindingCardinalityLinker(bindings, &tmpDiffCardinalites);
108 }
109 }
110 }
111 }
112
113 if (varItemProcessingDep != mLastVarItemProcessingDep) {
114 delete varItemProcessingDep;
115 }
116 }
117
118
119
120 void CQtConcurrentStreamingComplexQueryFinishingHandler::handleBindingCardinalityLinkerBatches(const QVector<COptimizedComplexVariableIndividualBindingsCardinalityBatchLinker*>& bindingsCardLinkerBatchVec, cint64 bindingCardLinkerCount) {
121
122 class CAnswerGenerationData {
123 public:
124 cint64 useAnswerNr = 0;
125 cint64 useImplicitMaximalCardinality = 0;
126 cint64 useImplicitTotalCardinality = 0;
127 cint64 useExplicitMaximalCardinality = 0;
128 cint64 useExplicitTotalCardinality = 0;
129 CVariableBindingsAnswersStreamingResult* concurrentStreamBindsAnswersResult = nullptr;
130 CVariableBindingsAnswersStreamingResult* existBindsAnswersStreamingResult = nullptr;
131
132 CAnswerGenerationData* addData(const CAnswerGenerationData& genData) {
133 useAnswerNr += genData.useAnswerNr;
134 useImplicitMaximalCardinality = qMax(genData.useImplicitMaximalCardinality, useImplicitMaximalCardinality);
135 useImplicitTotalCardinality += genData.useImplicitTotalCardinality;
136 useExplicitMaximalCardinality = qMax(genData.useExplicitMaximalCardinality, useExplicitMaximalCardinality);
137 useExplicitTotalCardinality += genData.useExplicitTotalCardinality;
138 if (genData.existBindsAnswersStreamingResult) {
139 genData.existBindsAnswersStreamingResult->releaseConcurrentlyWriteableStreamingResults(genData.concurrentStreamBindsAnswersResult);
140 }
141 return this;
142 }
143 };
144
145 function<CAnswerGenerationData(COptimizedComplexVariableIndividualBindingsCardinalityBatchLinker* const & batchData)> bindingBatchesAnswerGenerationFunc = [&](COptimizedComplexVariableIndividualBindingsCardinalityBatchLinker* const & batchData) -> CAnswerGenerationData {
146
147 CVariableBindingResultVector** useBindingResultVectorArray = new CVariableBindingResultVector*[mAnswerVarCount];
148 for (cint64 i = 0; i < mAnswerVarCount; ++i) {
149 useBindingResultVectorArray[i] = new CVariableBindingResultVector();
150 useBindingResultVectorArray[i]->mVectorPos = i;
151 }
152
153 COptimizedComplexVariableIndividualBindingsCardinalityLinker* statBindCardLinker = batchData->getStartBindingsCardinalityLinker();
154 COptimizedComplexVariableIndividualBindingsCardinalityLinker* endBindCardLinker = batchData->getEndBindingsCardinalityLinker();
155
156 CVariableBindingsAnswersStreamingResult* concurrentStreamBindsAnswersResult = mExistBindsAnswersStreamingResult->getConcurrentlyWriteableStreamingResults();
157 CVariableBindingsAnswersResult* useBindsAnswersResult = concurrentStreamBindsAnswersResult;
158 CVariableBindingsListAnswerResult* useBindResListAnswer = new CVariableBindingsListAnswerResult(mAnswerVarCount);
159 cint64 useAnswerNr = 0;
160 cint64 useImplicitMaximalCardinality = 0;
161 cint64 useImplicitTotalCardinality = 0;
162 cint64 useExplicitMaximalCardinality = 0;
163 cint64 useExplicitTotalCardinality = 0;
164
165
166 for (COptimizedComplexVariableIndividualBindingsCardinalityLinker* bindCardLinkerIt = statBindCardLinker; bindCardLinkerIt; ) {
167 COptimizedComplexVariableIndividualBindings* bindings = bindCardLinkerIt->getBindings();
168 COptimizedComplexVariableIndividualBindingsCardinality* cardinalities = bindCardLinkerIt->getInitialCardinalities();
169
170 handleBindingCardinalityLinker(bindings, cardinalities, useBindsAnswersResult, useBindResListAnswer, useBindingResultVectorArray, useAnswerNr, useImplicitMaximalCardinality, useImplicitTotalCardinality, useExplicitMaximalCardinality, useExplicitTotalCardinality);
171
172 if (bindCardLinkerIt == endBindCardLinker) {
173 bindCardLinkerIt = nullptr;
174 } else {
175 bindCardLinkerIt = bindCardLinkerIt->getNext();
176 }
177 }
178
179
180 delete[] useBindingResultVectorArray;
181 delete useBindResListAnswer;
182
183 CAnswerGenerationData anserGenData;
184 anserGenData.useAnswerNr = useAnswerNr;
185 anserGenData.concurrentStreamBindsAnswersResult = concurrentStreamBindsAnswersResult;
186 anserGenData.existBindsAnswersStreamingResult = mExistBindsAnswersStreamingResult;
187 anserGenData.useImplicitMaximalCardinality = useImplicitMaximalCardinality;
188 anserGenData.useImplicitTotalCardinality = useImplicitTotalCardinality;
189 anserGenData.useExplicitMaximalCardinality = useExplicitMaximalCardinality;
190 anserGenData.useExplicitTotalCardinality = useExplicitTotalCardinality;
191 return anserGenData;
192
193 };
194
195
196 function<void(CAnswerGenerationData& genData, const CAnswerGenerationData& redGenData)> bindingBatchesAnswerReductionFunc = [&](CAnswerGenerationData& genData, const CAnswerGenerationData& redGenData) -> void {
197 genData.addData(redGenData);
198 };
199
200 mExistBindsAnswersStreamingResult->flushResults();
201
202 //QtConcurrent::blockingMapped(bindingsCardLinkerBatchVec, bindingBatchesAnswerGenerationFunc);
203 //CAnswerGenerationData genData;
204 CAnswerGenerationData genData = QtConcurrent::blockingMappedReduced<CAnswerGenerationData>(bindingsCardLinkerBatchVec, bindingBatchesAnswerGenerationFunc, bindingBatchesAnswerReductionFunc);
205
206 mExplicitMaximalCardinality += genData.useExplicitMaximalCardinality;
207 mExplicitTotalCardinality += genData.useExplicitTotalCardinality;
208 mImplicitMaximalCardinality += genData.useImplicitMaximalCardinality;
209 mImplicitTotalCardinality += genData.useImplicitTotalCardinality;
210 mAnswerNr += genData.useAnswerNr;
211 if (mConfExtendedLogging) {
212 LOG(INFO, mLogDomain, logTr("Finished concurrent answer generation and serialization from %1 batches with %2 mappings in average.").arg(bindingsCardLinkerBatchVec.size()).arg(bindingCardLinkerCount / bindingsCardLinkerBatchVec.size()), this);
213 }
214 }
215
216
217 }; // end namespace Conclusion
218
219 }; // end namespace Answerer
220
221 }; // end namespace Reasoner
222
223 }; // end namespace Konclude