Debian Codegraph @

Source/Test/Evaluation/CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser.cpp

raw
1 /*
2 * Copyright (C) 2013-2015, 2019 by the Konclude Developer Team.
3 *
4 * This file is part of the reasoning system Konclude.
5 * For details and support, see <http://konclude.com/>.
6 *
7 * Konclude is free software: you can redistribute it and/or modify
8 * it under the terms of version 3 of the GNU Lesser General Public
9 * License (LGPLv3) as published by the Free Software Foundation.
10 *
11 * Konclude is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU (Lesser) General Public License for more details.
15 *
16 * You should have received a copy of the GNU (Lesser) General Public
17 * License along with Konclude. If not, see <http://www.gnu.org/licenses/>.
18 *
19 */
20
21 #include "CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser.h"
22
23
24 namespace Konclude {
25
26 namespace Test {
27
28 namespace Evaluation {
29
30
31
32 CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser::CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser() {
33 }
34
35
36 CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser::~CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser() {
37 }
38
39
40
41 cint64 CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser::compareResultsSimilarityWithOtherReasoners(const QString& groupString, const QString& requestString, const QString& reasonerString, CReasonerEvaluationStringListValue* stringListDataValue, const QStringList& reasonerList, const QString& detailedReportingOutputDirectory) {
42 cint64 totalDifferentResultCount = 0;
43 const QStringList& responseFileStringList = stringListDataValue->getValue();
44 if (!responseFileStringList.isEmpty()) {
45 QString responseFileString = responseFileStringList.first();
46 QFile responseFile(responseFileString);
47
48 bool fileOpened = false;
49 QDomDocument document;
50 CClassHierarchyResult* lastClassHierResult = nullptr;
51 cint64 lastClassHierResultNumber = 0;
52
53
54 foreach (const QString& otherReasoner, reasonerList) {
55 if (otherReasoner != reasonerString) {
56 cint64 reasonerDifferentResultCount = 0;
57
58
59 CReasonerEvaluationDataValue* otherReasonerEvalData = getGroupRequestReasonerEvaluationData(groupString,requestString,otherReasoner);
60 CReasonerEvaluationStringListValue* otherReasonerStringListDataValue = dynamic_cast<CReasonerEvaluationStringListValue*>(otherReasonerEvalData);
61 if (otherReasonerStringListDataValue) {
62 const QStringList& otherResponseFileStringList = otherReasonerStringListDataValue->getValue();
63 if (!otherResponseFileStringList.isEmpty()) {
64 CReasonerEvaluationStringListValue* minVal = qMin(stringListDataValue,otherReasonerStringListDataValue);
65 CReasonerEvaluationStringListValue* maxVal = qMax(stringListDataValue,otherReasonerStringListDataValue);
66
67 if (mReasonerCompareDiffCountHash.contains(QPair<CReasonerEvaluationStringListValue*,CReasonerEvaluationStringListValue*>(minVal,maxVal))) {
68 reasonerDifferentResultCount += mReasonerCompareDiffCountHash.value(QPair<CReasonerEvaluationStringListValue*,CReasonerEvaluationStringListValue*>(minVal,maxVal));
69 } else {
70
71 if (!fileOpened) {
72 if (responseFile.open(QIODevice::ReadOnly)) {
73 document.setContent(&responseFile,false);
74 responseFile.close();
75 }
76 fileOpened = true;
77 }
78 QDomElement rootEl = document.documentElement();
79 QDomElement resultElement = rootEl.firstChildElement();
80 bool foundResult = false;
81 bool foundError = false;
82 cint64 nodeNumber = 0;
83 if (!resultElement.isNull()) {
84 while (!resultElement.isNull()) {
85
86 if (resultElement.nodeName() == "Error") {
87 foundError = true;
88 break;
89
90 } else if (resultElement.nodeName() == "ClassHierarchy" || resultElement.nodeName() == "BooleanResponse" || resultElement.nodeName() == "Classes" || resultElement.nodeName() == "results") {
91 foundResult = true;
92 CClassHierarchyResult* classHierResult = nullptr;
93 if (lastClassHierResult && lastClassHierResultNumber == nodeNumber) {
94 classHierResult = lastClassHierResult;
95 }
96 CBooleanQueryResult* booleanResult = nullptr;
97 CClassesResult* classesResult = nullptr;
98 CVariableBindingsAnswersSetResult* variableBindingsResult = nullptr;
99
100 cint64 diffCount = 0;
101
102 QHash<QPair<CReasonerEvaluationStringListValue*,CReasonerEvaluationStringListValue*>,cint64>*& compareDiffCountHash = mNumberCompareDiffCountHashHash[nodeNumber];
103 if (!compareDiffCountHash) {
104 compareDiffCountHash = new QHash<QPair<CReasonerEvaluationStringListValue*,CReasonerEvaluationStringListValue*>,cint64>();
105 }
106
107 if (compareDiffCountHash->contains(QPair<CReasonerEvaluationStringListValue*,CReasonerEvaluationStringListValue*>(minVal,maxVal))) {
108 diffCount += compareDiffCountHash->value(QPair<CReasonerEvaluationStringListValue*,CReasonerEvaluationStringListValue*>(minVal,maxVal));
109 } else {
110
111 if (resultElement.nodeName() == "ClassHierarchy") {
112 if (!classHierResult) {
113 classHierResult = mOWLlinkResultParser.parseClassHierarchyResult(&resultElement);
114 }
115
116 QString otherResponseFileString = otherResponseFileStringList.first();
117 LOG(INFO,"::Konclude::Test::Evaluation::ResultDifferenceAnalyser",logTr("Comparing classification results between '%1' and '%2'.").arg(responseFileString).arg(otherResponseFileString),this);
118 if (!isClassHierarchyResultSimilarTo(classHierResult,nodeNumber,otherResponseFileString)) {
119 diffCount = 1;
120 }
121 } else if (resultElement.nodeName() == "BooleanResponse") {
122 if (!booleanResult) {
123 booleanResult = mOWLlinkResultParser.parseBooleanQueryResult(&resultElement);
124 }
125
126 QString otherResponseFileString = otherResponseFileStringList.first();
127 LOG(INFO,"::Konclude::Test::Evaluation::ResultDifferenceAnalyser",logTr("Comparing boolean results between '%1' and '%2'.").arg(responseFileString).arg(otherResponseFileString),this);
128 if (!isBooleanResultSimilarTo(booleanResult,nodeNumber,otherResponseFileString)) {
129 diffCount = 1;
130 }
131 } else if (resultElement.nodeName() == "Classes") {
132 if (!classesResult) {
133 classesResult = mOWLlinkResultParser.parseClassesResult(&resultElement);
134 }
135
136 QString otherResponseFileString = otherResponseFileStringList.first();
137 LOG(INFO,"::Konclude::Test::Evaluation::ResultDifferenceAnalyser",logTr("Comparing classes results between '%1' and '%2'.").arg(responseFileString).arg(otherResponseFileString),this);
138 if (!isClassesResultSimilarTo(classesResult,nodeNumber,otherResponseFileString)) {
139 diffCount = 1;
140 }
141 } else if (resultElement.nodeName() == "results") {
142 // handle SPARQL binding results
143 QHash<QString, cint64> variableNameIndexHash;
144 if (!variableBindingsResult) {
145 variableBindingsResult = mSPARQLResultParser.parseVariableBindingsSet(&resultElement, &variableNameIndexHash, true);
146 }
147
148 QString otherResponseFileString = otherResponseFileStringList.first();
149 QString reportingFileName = responseFileString + "--vs--" + otherResponseFileString + ".txt";
150 reportingFileName = reportingFileName.replace("/", "_").replace(":", "_");
151 reportingFileName = detailedReportingOutputDirectory + reportingFileName;
152 LOG(INFO, "::Konclude::Test::Evaluation::ResultDifferenceAnalyser", logTr("Comparing variable binding results between '%1' and '%2', reporting details to '%3'.").arg(responseFileString).arg(otherResponseFileString).arg(reportingFileName), this);
153 if (!isVariableBindingResultSimilarTo(variableBindingsResult, nodeNumber, otherResponseFileString, &variableNameIndexHash, reportingFileName)) {
154 diffCount = 1;
155 }
156 }
157 compareDiffCountHash->insert(QPair<CReasonerEvaluationStringListValue*,CReasonerEvaluationStringListValue*>(minVal,maxVal),diffCount);
158 }
159 reasonerDifferentResultCount += diffCount;
160 if (classHierResult) {
161 if (!lastClassHierResult) {
162 lastClassHierResult = classHierResult;
163 lastClassHierResultNumber = nodeNumber;
164 } else {
165 if (lastClassHierResult != classHierResult) {
166 delete classHierResult;
167 }
168 }
169 }
170 if (booleanResult) {
171 delete booleanResult;
172 }
173 if (classesResult) {
174 delete classesResult;
175 }
176 if (variableBindingsResult) {
177 delete variableBindingsResult;
178 }
179
180 }
181 if (resultElement.nodeName() != "answering-statistics") {
182 ++nodeNumber;
183 }
184 resultElement = resultElement.nextSiblingElement();
185 }
186 }
187
188 mReasonerCompareDiffCountHash.insert(QPair<CReasonerEvaluationStringListValue*,CReasonerEvaluationStringListValue*>(minVal,maxVal),reasonerDifferentResultCount);
189
190 }
191 }
192 }
193
194 totalDifferentResultCount += reasonerDifferentResultCount;
195
196 }
197 }
198 if (lastClassHierResult) {
199 delete lastClassHierResult;
200 }
201
202 }
203 return totalDifferentResultCount;
204 }
205
206
207
208 bool CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser::isClassHierarchyResultSimilarTo(CClassHierarchyResult* classHierResult, cint64 nodeNumber, const QString& otherResponseFileString) {
209 bool resultSimilar = true;
210 QFile responseFile(otherResponseFileString);
211 if (responseFile.open(QIODevice::ReadOnly)) {
212 QDomDocument document;
213 document.setContent(&responseFile,false);
214 QDomElement rootEl = document.documentElement();
215 QDomElement classResEl = rootEl.firstChildElement();
216 cint64 currNodeNumber = 0;
217 if (!classResEl.isNull()) {
218 while (!classResEl.isNull()) {
219 if (classResEl.nodeName() == "Error") {
220 break;
221
222 } else if (currNodeNumber == nodeNumber && classResEl.nodeName() == "ClassHierarchy") {
223 CClassHierarchyResult* otherClassHierResult = mOWLlinkResultParser.parseClassHierarchyResult(&classResEl);
224 if (otherClassHierResult) {
225
226 if (!classHierResult->isResultEquivalentTo(otherClassHierResult)) {
227 resultSimilar = false;
228 }
229
230 delete otherClassHierResult;
231 }
232 break;
233
234 } else {
235 ++currNodeNumber;
236 if (currNodeNumber > nodeNumber) {
237 break;
238 }
239 classResEl = classResEl.nextSiblingElement();
240 }
241 }
242 }
243 }
244 return resultSimilar;
245 }
246
247
248
249 bool CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser::isBooleanResultSimilarTo(CBooleanQueryResult* booleanResult, cint64 nodeNumber, const QString& otherResponseFileString) {
250 bool resultSimilar = true;
251 QFile responseFile(otherResponseFileString);
252 if (responseFile.open(QIODevice::ReadOnly)) {
253 QDomDocument document;
254 document.setContent(&responseFile,false);
255 QDomElement rootEl = document.documentElement();
256 QDomElement classResEl = rootEl.firstChildElement();
257 cint64 currNodeNumber = 0;
258 if (!classResEl.isNull()) {
259 while (!classResEl.isNull()) {
260 if (classResEl.nodeName() == "Error") {
261 break;
262
263 } else if (currNodeNumber == nodeNumber && classResEl.nodeName() == "BooleanResponse") {
264 CBooleanQueryResult* otherBooleanResult = mOWLlinkResultParser.parseBooleanQueryResult(&classResEl);
265 if (otherBooleanResult) {
266
267 if (!booleanResult->isResultEquivalentTo(otherBooleanResult)) {
268 resultSimilar = false;
269 }
270
271 delete otherBooleanResult;
272 }
273 break;
274
275 } else {
276 ++currNodeNumber;
277 if (currNodeNumber > nodeNumber) {
278 break;
279 }
280 classResEl = classResEl.nextSiblingElement();
281 }
282 }
283 }
284 }
285 return resultSimilar;
286 }
287
288
289
290 bool CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser::isVariableBindingResultSimilarTo(CVariableBindingsAnswersSetResult* varBindingResult, cint64 nodeNumber, const QString& otherResponseFileString, QHash<QString, cint64>* variableNameIndexHash, const QString& reportingFileName) {
291 bool resultSimilar = true;
292 QFile responseFile(otherResponseFileString);
293 if (responseFile.open(QIODevice::ReadOnly)) {
294 QDomDocument document;
295 document.setContent(&responseFile,false);
296 QDomElement rootEl = document.documentElement();
297 QDomElement resultElement = rootEl.firstChildElement();
298 cint64 currNodeNumber = 0;
299 if (!resultElement.isNull()) {
300 while (!resultElement.isNull()) {
301 if (resultElement.nodeName() == "Error") {
302 break;
303
304 } else if (currNodeNumber == nodeNumber && resultElement.nodeName() == "results") {
305 CVariableBindingsAnswersSetResult* otherVarBindingResult = mSPARQLResultParser.parseVariableBindingsSet(&resultElement, variableNameIndexHash, true);
306 if (otherVarBindingResult) {
307
308 if (!varBindingResult || !varBindingResult->isResultEquivalentTo(otherVarBindingResult)) {
309 resultSimilar = false;
310
311 if (varBindingResult && otherVarBindingResult) {
312 CVariableBindingsAnswersResultIterator* it = varBindingResult->getVariableBindingsAnswersIterator();
313 while (it->hasNext()) {
314 CVariableBindingsAnswerResult* answerResult = it->getNext();
315 if (!otherVarBindingResult->hasResultVariableBindings(answerResult)) {
316 QFile reportingFile(reportingFileName);
317 if (reportingFile.open(QFile::Append)) {
318 reportingFile.write(QString("Difference (additional answer) for " + answerResult->getQueryResultString() + "\r\n").toLocal8Bit());
319 reportingFile.close();
320 }
321 }
322 }
323 CVariableBindingsAnswersResultIterator* otherIt = otherVarBindingResult->getVariableBindingsAnswersIterator();
324 while (otherIt->hasNext()) {
325 CVariableBindingsAnswerResult* answerResult = otherIt->getNext();
326 if (!varBindingResult->hasResultVariableBindings(answerResult)) {
327 QFile reportingFile(reportingFileName);
328 if (reportingFile.open(QFile::Append)) {
329 reportingFile.write(QString("Difference (missing answer) for " + answerResult->getQueryResultString() + "\r\n").toLocal8Bit());
330 reportingFile.close();
331 }
332 }
333 }
334 }
335 }
336
337 delete otherVarBindingResult;
338
339 } else if (varBindingResult) {
340 resultSimilar = false;
341 }
342 break;
343
344 } else {
345 if (resultElement.nodeName() != "answering-statistics") {
346 ++currNodeNumber;
347 }
348 if (currNodeNumber > nodeNumber) {
349 break;
350 }
351 resultElement = resultElement.nextSiblingElement();
352 }
353 }
354 }
355 }
356 return resultSimilar;
357 }
358
359
360
361 bool CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser::isClassesResultSimilarTo(CClassesResult* classesResult, cint64 nodeNumber, const QString& otherResponseFileString) {
362 bool resultSimilar = true;
363 QFile responseFile(otherResponseFileString);
364 if (responseFile.open(QIODevice::ReadOnly)) {
365 QDomDocument document;
366 document.setContent(&responseFile,false);
367 QDomElement rootEl = document.documentElement();
368 QDomElement classResEl = rootEl.firstChildElement();
369 cint64 currNodeNumber = 0;
370 if (!classResEl.isNull()) {
371 while (!classResEl.isNull()) {
372 if (classResEl.nodeName() == "Error") {
373 break;
374
375 } else if (currNodeNumber == nodeNumber && classResEl.nodeName() == "Classes") {
376 CClassesResult* otherClassesResult = mOWLlinkResultParser.parseClassesResult(&classResEl);
377 if (otherClassesResult) {
378
379 if (!classesResult->isResultEquivalentTo(otherClassesResult)) {
380 resultSimilar = false;
381 }
382
383 delete otherClassesResult;
384 }
385 break;
386
387 } else {
388 ++currNodeNumber;
389 if (currNodeNumber > nodeNumber) {
390 break;
391 }
392 classResEl = classResEl.nextSiblingElement();
393 }
394 }
395 }
396 }
397 return resultSimilar;
398 }
399
400 bool CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser::analyseGroupedEvaluationData(const QStringList& groubList, const QStringList& requestList, const QStringList& reasonerList, const QString& outputDirectory, CReasonerEvaluationAnalyseContext* context, CReasonerEvaluationGroupRequestSelector* selector) {
401
402 mOutputFileName = QString("%1").arg(selector->getOutputString());
403
404 bool hasSelectedRequestFile = false;
405
406 QStringList titleList;
407 titleList.append("Requests");
408 titleList.append(reasonerList);
409 titleList.append("Average");
410 titleList.append("Min");
411 titleList.append("Max");
412
413 QDir outDir(outputDirectory);
414 outDir.mkpath(mOutputFileName);
415 QString outputFileString(outputDirectory+mOutputFileName+"result-difference-to-other-reasoners");
416
417 QString detailedOutputReportingDirectory = QString(outputDirectory + mOutputFileName + "/result-differences-reports/");
418 outDir.mkpath(mOutputFileName + "/result-differences-reports");
419
420 CReasonerEvaluationTableMultiFormatOutputWriter<double> tableMultiFormatOutputWriter;
421 tableMultiFormatOutputWriter.addColumnTitles(titleList);
422
423 foreach (QString groupString, groubList) {
424
425 if (selector->isGroupSelected(groupString)) {
426
427 QSet<QString>* groupRequestList = getGroupRequests(groupString);
428 foreach (QString requestString, *groupRequestList) {
429
430 if (selector->isRequestSelected(groupString,requestString)) {
431
432 bool allDataAvailable = true;
433 foreach (QString reasonerString, reasonerList) {
434 CReasonerEvaluationDataValue* evalData = getGroupRequestReasonerEvaluationData(groupString,requestString,reasonerString);
435 if (!evalData || evalData->hasDataNotAvailableFlag()) {
436 allDataAvailable = false;
437 }
438 }
439
440
441 if (allDataAvailable) {
442
443 hasSelectedRequestFile = true;
444 double min = DOUBLE_MAX;
445 double max = DOUBLE_MIN;
446 double mean = 0.;
447 cint64 count = 0;
448
449 tableMultiFormatOutputWriter.addNewTableRow(requestString);
450
451 foreach (QString reasonerString, reasonerList) {
452 CReasonerEvaluationDataValue* evalData = getGroupRequestReasonerEvaluationData(groupString,requestString,reasonerString);
453 CReasonerEvaluationStringListValue* stringListDataValue = dynamic_cast<CReasonerEvaluationStringListValue*>(evalData);
454 if (stringListDataValue) {
455
456 QStringList responseFiles = stringListDataValue->getValue();
457
458 double diffCount = compareResultsSimilarityWithOtherReasoners(groupString,requestString,reasonerString,stringListDataValue,reasonerList, detailedOutputReportingDirectory);
459
460 tableMultiFormatOutputWriter.addValuesToLastTableRow(QList<double>()<<diffCount);
461
462 mean += diffCount;
463 min = qMin(min,diffCount);
464 max = qMax(max,diffCount);
465 count++;
466 }
467 }
468 mean /= (double)count;
469 tableMultiFormatOutputWriter.addValuesToLastTableRow(QList<double>()<<mean<<min<<max);
470 }
471 }
472 }
473 }
474 }
475
476 if (hasSelectedRequestFile) {
477
478 QStringList reasonerCompGNUPlotFileList;
479 QStringList reasonerCompHighchartPlotFileList;
480
481 tableMultiFormatOutputWriter.buildSortedIndex();
482 reasonerCompHighchartPlotFileList += tableMultiFormatOutputWriter.writeCVSTable(outputFileString);
483 reasonerCompHighchartPlotFileList += tableMultiFormatOutputWriter.writeCVSTablesSorted(outputFileString);
484 tableMultiFormatOutputWriter.writeCVSTablesSeparateSorted(outputFileString);
485
486 if (context->isGNUPlottingActivated()) {
487 reasonerCompGNUPlotFileList += tableMultiFormatOutputWriter.writeGNUPlotDataTable(outputFileString);
488 reasonerCompGNUPlotFileList += tableMultiFormatOutputWriter.writeGNUPlotDataTablesSorted(outputFileString);
489 foreach (QString plotFileString, reasonerCompGNUPlotFileList) {
490 if (true /*plotFileString.contains("-by-Average-")*/) {
491 QString outputFile = plotFileString;
492 outputFile.remove(".dat");
493 QString title = getPrettyTitleFromOutputFile(outputFile);
494 context->getGNUPlotPlotter()->createReasonerComparisonPlot(plotFileString,context->getAdditionalTitleString()+title+"[# differences]",reasonerList,outputFile);
495 }
496 }
497 }
498 if (context->isHighchartPlottingActivated()) {
499 foreach (QString plotFileString, reasonerCompHighchartPlotFileList) {
500 if (true /*plotFileString.contains("-by-Average-")*/) {
501 QString outputFile = plotFileString;
502 outputFile.remove(".csv");
503 QString title = getPrettyTitleFromOutputFile(outputFile);
504 context->getHighchartPlotter()->createReasonerDifferencePlot(plotFileString,context->getAdditionalTitleString()+title+"[# differences]",reasonerList,outputFile,selector->getNameString()+" :: "+context->getAdditionalTitleString());
505 }
506 }
507 }
508
509 }
510
511 return true;
512 }
513
514
515
516 QString CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser::getPrettyTitleFromOutputFile(const QString& outputFile) {
517 QString title = outputFile;
518 title.remove("table");
519 cint64 slashPos = title.lastIndexOf("/");
520 if (slashPos >= 0) {
521 title = title.mid(slashPos+1);
522 }
523 title.replace("-"," ");
524 title.replace("result difference to other reasoners","Result Differences to other Reasoners, ");
525 title.replace("sorted","Sorted");
526 if (title.contains(" ascending")) {
527 title.replace(" ascending","");
528 title.append(" in Ascending Order ");
529 }
530 if (title.contains(" descending")) {
531 title.replace(" descending","");
532 title.append(" in Descending Order ");
533 }
534 return title;
535 }
536 }; // end namespace Evaluation
537
538 }; // end namespace Test
539
540 }; // end namespace Konclude