Debian Codegraph @

Source/Test/Evaluation/CReasonerEvaluationGroupRequestReasonerResultSubsumptionAnalyser.cpp

raw
1 /*
2 * Copyright (C) 2013-2015, 2019 by the Konclude Developer Team.
3 *
4 * This file is part of the reasoning system Konclude.
5 * For details and support, see <http://konclude.com/>.
6 *
7 * Konclude is free software: you can redistribute it and/or modify
8 * it under the terms of version 3 of the GNU Lesser General Public
9 * License (LGPLv3) as published by the Free Software Foundation.
10 *
11 * Konclude is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU (Lesser) General Public License for more details.
15 *
16 * You should have received a copy of the GNU (Lesser) General Public
17 * License along with Konclude. If not, see <http://www.gnu.org/licenses/>.
18 *
19 */
20
21 #include "CReasonerEvaluationGroupRequestReasonerResultSubsumptionAnalyser.h"
22
23
24 namespace Konclude {
25
26 namespace Test {
27
28 namespace Evaluation {
29
30
31
32 CReasonerEvaluationGroupRequestReasonerResultSubsumptionAnalyser::CReasonerEvaluationGroupRequestReasonerResultSubsumptionAnalyser() {
33 }
34
35
36 CReasonerEvaluationGroupRequestReasonerResultSubsumptionAnalyser::~CReasonerEvaluationGroupRequestReasonerResultSubsumptionAnalyser() {
37 }
38
39
40
41 cint64 CReasonerEvaluationGroupRequestReasonerResultSubsumptionAnalyser::compareResultsSubsumptionWithOtherReasoners(const QString& groupString, const QString& requestString, const QString& reasonerString, CReasonerEvaluationStringListValue* stringListDataValue, const QStringList& reasonerList, const QString& detailedReportingOutputDirectory) {
42 cint64 totalResultSubsumptionCount = 0;
43 const QStringList& responseFileStringList = stringListDataValue->getValue();
44 if (!responseFileStringList.isEmpty()) {
45 QString responseFileString = responseFileStringList.first();
46 QFile responseFile(responseFileString);
47
48 bool fileOpened = false;
49 QDomDocument document;
50 CClassHierarchyResult* lastClassHierResult = nullptr;
51 cint64 lastClassHierResultNumber = 0;
52
53
54 foreach (const QString& otherReasoner, reasonerList) {
55 if (otherReasoner != reasonerString) {
56 cint64 reasonerResultSubsumptionCount = 0;
57
58
59 CReasonerEvaluationDataValue* otherReasonerEvalData = getGroupRequestReasonerEvaluationData(groupString,requestString,otherReasoner);
60 CReasonerEvaluationStringListValue* otherReasonerStringListDataValue = dynamic_cast<CReasonerEvaluationStringListValue*>(otherReasonerEvalData);
61 if (otherReasonerStringListDataValue) {
62 const QStringList& otherResponseFileStringList = otherReasonerStringListDataValue->getValue();
63 if (!otherResponseFileStringList.isEmpty()) {
64
65 if (mReasonerCompareResultSubsumptionCountHash.contains(QPair<CReasonerEvaluationStringListValue*,CReasonerEvaluationStringListValue*>(stringListDataValue, otherReasonerStringListDataValue))) {
66 reasonerResultSubsumptionCount += mReasonerCompareResultSubsumptionCountHash.value(QPair<CReasonerEvaluationStringListValue*,CReasonerEvaluationStringListValue*>(stringListDataValue, otherReasonerStringListDataValue));
67 } else {
68
69 if (!fileOpened) {
70 if (responseFile.open(QIODevice::ReadOnly)) {
71 document.setContent(&responseFile,false);
72 responseFile.close();
73 }
74 fileOpened = true;
75 }
76 QDomElement rootEl = document.documentElement();
77 QDomElement resultElement = rootEl.firstChildElement();
78 bool foundResult = false;
79 bool foundError = false;
80 cint64 nodeNumber = 0;
81 if (!resultElement.isNull()) {
82 while (!resultElement.isNull()) {
83
84 if (resultElement.nodeName() == "Error") {
85 foundError = true;
86 break;
87
88 } else if (resultElement.nodeName() == "results") {
89 foundResult = true;
90 CClassHierarchyResult* classHierResult = nullptr;
91 if (lastClassHierResult && lastClassHierResultNumber == nodeNumber) {
92 classHierResult = lastClassHierResult;
93 }
94 CBooleanQueryResult* booleanResult = nullptr;
95 CClassesResult* classesResult = nullptr;
96 CVariableBindingsAnswersSetResult* variableBindingsResult = nullptr;
97
98 cint64 subsumingCount = 0;
99
100 QHash<QPair<CReasonerEvaluationStringListValue*,CReasonerEvaluationStringListValue*>,cint64>*& compareDiffCountHash = mNumberCompareDiffCountHashHash[nodeNumber];
101 if (!compareDiffCountHash) {
102 compareDiffCountHash = new QHash<QPair<CReasonerEvaluationStringListValue*,CReasonerEvaluationStringListValue*>,cint64>();
103 }
104
105 if (compareDiffCountHash->contains(QPair<CReasonerEvaluationStringListValue*,CReasonerEvaluationStringListValue*>(stringListDataValue, otherReasonerStringListDataValue))) {
106 subsumingCount += compareDiffCountHash->value(QPair<CReasonerEvaluationStringListValue*,CReasonerEvaluationStringListValue*>(stringListDataValue, otherReasonerStringListDataValue));
107 } else {
108 if (resultElement.nodeName() == "results") {
109 // handle SPARQL binding results
110 QHash<QString, cint64> variableNameIndexHash;
111 if (!variableBindingsResult) {
112 variableBindingsResult = mSPARQLResultParser.parseVariableBindingsSet(&resultElement, &variableNameIndexHash, true);
113 }
114 QString otherResponseFileString = otherResponseFileStringList.first();
115 LOG(INFO, "::Konclude::Test::Evaluation::ResultDifferenceAnalyser", logTr("Comparing variable binding results between '%1' and '%2'.").arg(responseFileString).arg(otherResponseFileString), this);
116 if (isVariableBindingResultSubsuming(variableBindingsResult, nodeNumber, otherResponseFileString, &variableNameIndexHash)) {
117 subsumingCount = 1;
118 }
119 }
120 compareDiffCountHash->insert(QPair<CReasonerEvaluationStringListValue*,CReasonerEvaluationStringListValue*>(stringListDataValue, otherReasonerStringListDataValue),subsumingCount);
121 }
122 reasonerResultSubsumptionCount += subsumingCount;
123 if (classHierResult) {
124 if (!lastClassHierResult) {
125 lastClassHierResult = classHierResult;
126 lastClassHierResultNumber = nodeNumber;
127 } else {
128 if (lastClassHierResult != classHierResult) {
129 delete classHierResult;
130 }
131 }
132 }
133 if (booleanResult) {
134 delete booleanResult;
135 }
136 if (classesResult) {
137 delete classesResult;
138 }
139 if (variableBindingsResult) {
140 delete variableBindingsResult;
141 }
142
143 }
144 if (resultElement.nodeName() != "answering-statistics") {
145 ++nodeNumber;
146 }
147 resultElement = resultElement.nextSiblingElement();
148 }
149 }
150
151 mReasonerCompareResultSubsumptionCountHash.insert(QPair<CReasonerEvaluationStringListValue*,CReasonerEvaluationStringListValue*>(stringListDataValue, otherReasonerStringListDataValue),reasonerResultSubsumptionCount);
152
153 }
154 }
155 }
156
157 totalResultSubsumptionCount += reasonerResultSubsumptionCount;
158
159 }
160 }
161 if (lastClassHierResult) {
162 delete lastClassHierResult;
163 }
164
165 }
166 return totalResultSubsumptionCount;
167 }
168
169
170
171
172
173 bool CReasonerEvaluationGroupRequestReasonerResultSubsumptionAnalyser::isVariableBindingResultSubsuming(CVariableBindingsAnswersSetResult* varBindingResult, cint64 nodeNumber, const QString& otherResponseFileString, QHash<QString, cint64>* variableNameIndexHash) {
174 bool resultSubsumption = false;
175 QFile responseFile(otherResponseFileString);
176 if (responseFile.open(QIODevice::ReadOnly)) {
177 QDomDocument document;
178 document.setContent(&responseFile,false);
179 QDomElement rootEl = document.documentElement();
180 QDomElement resultElement = rootEl.firstChildElement();
181 cint64 currNodeNumber = 0;
182 if (!resultElement.isNull()) {
183 while (!resultElement.isNull()) {
184 if (resultElement.nodeName() == "Error") {
185 break;
186
187 } else if (currNodeNumber == nodeNumber && resultElement.nodeName() == "results") {
188 CVariableBindingsAnswersSetResult* otherVarBindingResult = mSPARQLResultParser.parseVariableBindingsSet(&resultElement, variableNameIndexHash, true);
189 if (otherVarBindingResult) {
190
191 if (varBindingResult && varBindingResult->isResultEquivalentTo(otherVarBindingResult)) {
192 resultSubsumption = true;
193
194 } else if (varBindingResult) {
195 resultSubsumption = true;
196 if (varBindingResult && otherVarBindingResult && resultSubsumption) {
197 CVariableBindingsAnswersResultIterator* it = otherVarBindingResult->getVariableBindingsAnswersIterator();
198 while (it->hasNext()) {
199 CVariableBindingsAnswerResult* answerResult = it->getNext();
200 if (!varBindingResult->hasResultVariableBindings(answerResult)) {
201 resultSubsumption = false;
202 }
203 }
204 }
205 }
206
207 delete otherVarBindingResult;
208
209 } else if (varBindingResult) {
210 resultSubsumption = false;
211 }
212 break;
213
214 } else {
215 if (resultElement.nodeName() != "answering-statistics") {
216 ++currNodeNumber;
217 }
218 if (currNodeNumber > nodeNumber) {
219 break;
220 }
221 resultElement = resultElement.nextSiblingElement();
222 }
223 }
224 }
225 }
226 return resultSubsumption;
227 }
228
229
230
231 bool CReasonerEvaluationGroupRequestReasonerResultSubsumptionAnalyser::analyseGroupedEvaluationData(const QStringList& groubList, const QStringList& requestList, const QStringList& reasonerList, const QString& outputDirectory, CReasonerEvaluationAnalyseContext* context, CReasonerEvaluationGroupRequestSelector* selector) {
232
233 mOutputFileName = QString("%1").arg(selector->getOutputString());
234
235 bool hasSelectedRequestFile = false;
236
237 QStringList titleList;
238 titleList.append("Requests");
239 titleList.append(reasonerList);
240 titleList.append("Average");
241 titleList.append("Min");
242 titleList.append("Max");
243
244 QDir outDir(outputDirectory);
245 outDir.mkpath(mOutputFileName);
246 QString outputFileString(outputDirectory+mOutputFileName+"result-subsumption-over-other-reasoners");
247
248 QString detailedOutputReportingDirectory = QString(outputDirectory + mOutputFileName + "/result-subsumption-reports/");
249 outDir.mkpath(mOutputFileName + "/result-subsumption-reports");
250
251 CReasonerEvaluationTableMultiFormatOutputWriter<double> tableMultiFormatOutputWriter;
252 tableMultiFormatOutputWriter.addColumnTitles(titleList);
253
254 foreach (QString groupString, groubList) {
255
256 if (selector->isGroupSelected(groupString)) {
257
258 QSet<QString>* groupRequestList = getGroupRequests(groupString);
259 foreach (QString requestString, *groupRequestList) {
260
261 if (selector->isRequestSelected(groupString,requestString)) {
262
263 bool allDataAvailable = true;
264 foreach (QString reasonerString, reasonerList) {
265 CReasonerEvaluationDataValue* evalData = getGroupRequestReasonerEvaluationData(groupString,requestString,reasonerString);
266 if (!evalData || evalData->hasDataNotAvailableFlag()) {
267 allDataAvailable = false;
268 }
269 }
270
271
272 if (allDataAvailable) {
273
274 hasSelectedRequestFile = true;
275 double min = DOUBLE_MAX;
276 double max = DOUBLE_MIN;
277 double mean = 0.;
278 cint64 count = 0;
279
280 tableMultiFormatOutputWriter.addNewTableRow(requestString);
281
282 foreach (QString reasonerString, reasonerList) {
283 CReasonerEvaluationDataValue* evalData = getGroupRequestReasonerEvaluationData(groupString,requestString,reasonerString);
284 CReasonerEvaluationStringListValue* stringListDataValue = dynamic_cast<CReasonerEvaluationStringListValue*>(evalData);
285 if (stringListDataValue) {
286
287 QStringList responseFiles = stringListDataValue->getValue();
288
289 double subsumptionCount = compareResultsSubsumptionWithOtherReasoners(groupString,requestString,reasonerString,stringListDataValue,reasonerList, detailedOutputReportingDirectory);
290
291 tableMultiFormatOutputWriter.addValuesToLastTableRow(QList<double>()<<subsumptionCount);
292
293 mean += subsumptionCount;
294 min = qMin(min,subsumptionCount);
295 max = qMax(max,subsumptionCount);
296 count++;
297 }
298 }
299 mean /= (double)count;
300 tableMultiFormatOutputWriter.addValuesToLastTableRow(QList<double>()<<mean<<min<<max);
301 }
302 }
303 }
304 }
305 }
306
307 if (hasSelectedRequestFile) {
308
309 QStringList reasonerCompGNUPlotFileList;
310 QStringList reasonerCompHighchartPlotFileList;
311
312 tableMultiFormatOutputWriter.buildSortedIndex();
313 reasonerCompHighchartPlotFileList += tableMultiFormatOutputWriter.writeCVSTable(outputFileString);
314 reasonerCompHighchartPlotFileList += tableMultiFormatOutputWriter.writeCVSTablesSorted(outputFileString);
315 tableMultiFormatOutputWriter.writeCVSTablesSeparateSorted(outputFileString);
316
317 if (context->isGNUPlottingActivated()) {
318 reasonerCompGNUPlotFileList += tableMultiFormatOutputWriter.writeGNUPlotDataTable(outputFileString);
319 reasonerCompGNUPlotFileList += tableMultiFormatOutputWriter.writeGNUPlotDataTablesSorted(outputFileString);
320 foreach (QString plotFileString, reasonerCompGNUPlotFileList) {
321 if (true /*plotFileString.contains("-by-Average-")*/) {
322 QString outputFile = plotFileString;
323 outputFile.remove(".dat");
324 QString title = getPrettyTitleFromOutputFile(outputFile);
325 context->getGNUPlotPlotter()->createReasonerComparisonPlot(plotFileString,context->getAdditionalTitleString()+title+"[# subsumptions]",reasonerList,outputFile);
326 }
327 }
328 }
329 if (context->isHighchartPlottingActivated()) {
330 foreach (QString plotFileString, reasonerCompHighchartPlotFileList) {
331 if (true /*plotFileString.contains("-by-Average-")*/) {
332 QString outputFile = plotFileString;
333 outputFile.remove(".csv");
334 QString title = getPrettyTitleFromOutputFile(outputFile);
335 context->getHighchartPlotter()->createReasonerSubsumptionPlot(plotFileString,context->getAdditionalTitleString()+title+"[# subsumptions]",reasonerList,outputFile,selector->getNameString()+" :: "+context->getAdditionalTitleString());
336 }
337 }
338 }
339
340 }
341
342 return true;
343 }
344
345
346
347 QString CReasonerEvaluationGroupRequestReasonerResultSubsumptionAnalyser::getPrettyTitleFromOutputFile(const QString& outputFile) {
348 QString title = outputFile;
349 title.remove("table");
350 cint64 slashPos = title.lastIndexOf("/");
351 if (slashPos >= 0) {
352 title = title.mid(slashPos+1);
353 }
354 title.replace("-"," ");
355 title.replace("result subsumptions over other reasoners","Result Subsumptions over other Reasoners, ");
356 title.replace("sorted","Sorted");
357 if (title.contains(" ascending")) {
358 title.replace(" ascending","");
359 title.append(" in Ascending Order ");
360 }
361 if (title.contains(" descending")) {
362 title.replace(" descending","");
363 title.append(" in Descending Order ");
364 }
365 return title;
366 }
367 }; // end namespace Evaluation
368
369 }; // end namespace Test
370
371 }; // end namespace Konclude