/* * Copyright (C) 2013-2015, 2019 by the Konclude Developer Team. * * This file is part of the reasoning system Konclude. * For details and support, see . * * Konclude is free software: you can redistribute it and/or modify * it under the terms of version 3 of the GNU Lesser General Public * License (LGPLv3) as published by the Free Software Foundation. * * Konclude is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU (Lesser) General Public License for more details. * * You should have received a copy of the GNU (Lesser) General Public * License along with Konclude. If not, see . * */ #include "CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser.h" namespace Konclude { namespace Test { namespace Evaluation { CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser::CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser() { } CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser::~CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser() { } cint64 CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser::compareResultsSimilarityWithOtherReasoners(const QString& groupString, const QString& requestString, const QString& reasonerString, CReasonerEvaluationStringListValue* stringListDataValue, const QStringList& reasonerList, const QString& detailedReportingOutputDirectory) { cint64 totalDifferentResultCount = 0; const QStringList& responseFileStringList = stringListDataValue->getValue(); if (!responseFileStringList.isEmpty()) { QString responseFileString = responseFileStringList.first(); QFile responseFile(responseFileString); bool fileOpened = false; QDomDocument document; CClassHierarchyResult* lastClassHierResult = nullptr; cint64 lastClassHierResultNumber = 0; foreach (const QString& otherReasoner, reasonerList) { if (otherReasoner != reasonerString) { cint64 reasonerDifferentResultCount = 0; CReasonerEvaluationDataValue* otherReasonerEvalData = getGroupRequestReasonerEvaluationData(groupString,requestString,otherReasoner); CReasonerEvaluationStringListValue* otherReasonerStringListDataValue = dynamic_cast(otherReasonerEvalData); if (otherReasonerStringListDataValue) { const QStringList& otherResponseFileStringList = otherReasonerStringListDataValue->getValue(); if (!otherResponseFileStringList.isEmpty()) { CReasonerEvaluationStringListValue* minVal = qMin(stringListDataValue,otherReasonerStringListDataValue); CReasonerEvaluationStringListValue* maxVal = qMax(stringListDataValue,otherReasonerStringListDataValue); if (mReasonerCompareDiffCountHash.contains(QPair(minVal,maxVal))) { reasonerDifferentResultCount += mReasonerCompareDiffCountHash.value(QPair(minVal,maxVal)); } else { if (!fileOpened) { if (responseFile.open(QIODevice::ReadOnly)) { document.setContent(&responseFile,false); responseFile.close(); } fileOpened = true; } QDomElement rootEl = document.documentElement(); QDomElement resultElement = rootEl.firstChildElement(); bool foundResult = false; bool foundError = false; cint64 nodeNumber = 0; if (!resultElement.isNull()) { while (!resultElement.isNull()) { if (resultElement.nodeName() == "Error") { foundError = true; break; } else if (resultElement.nodeName() == "ClassHierarchy" || resultElement.nodeName() == "BooleanResponse" || resultElement.nodeName() == "Classes" || resultElement.nodeName() == "results") { foundResult = true; CClassHierarchyResult* classHierResult = nullptr; if (lastClassHierResult && lastClassHierResultNumber == nodeNumber) { classHierResult = lastClassHierResult; } CBooleanQueryResult* booleanResult = nullptr; CClassesResult* classesResult = nullptr; CVariableBindingsAnswersSetResult* variableBindingsResult = nullptr; cint64 diffCount = 0; QHash,cint64>*& compareDiffCountHash = mNumberCompareDiffCountHashHash[nodeNumber]; if (!compareDiffCountHash) { compareDiffCountHash = new QHash,cint64>(); } if (compareDiffCountHash->contains(QPair(minVal,maxVal))) { diffCount += compareDiffCountHash->value(QPair(minVal,maxVal)); } else { if (resultElement.nodeName() == "ClassHierarchy") { if (!classHierResult) { classHierResult = mOWLlinkResultParser.parseClassHierarchyResult(&resultElement); } QString otherResponseFileString = otherResponseFileStringList.first(); LOG(INFO,"::Konclude::Test::Evaluation::ResultDifferenceAnalyser",logTr("Comparing classification results between '%1' and '%2'.").arg(responseFileString).arg(otherResponseFileString),this); if (!isClassHierarchyResultSimilarTo(classHierResult,nodeNumber,otherResponseFileString)) { diffCount = 1; } } else if (resultElement.nodeName() == "BooleanResponse") { if (!booleanResult) { booleanResult = mOWLlinkResultParser.parseBooleanQueryResult(&resultElement); } QString otherResponseFileString = otherResponseFileStringList.first(); LOG(INFO,"::Konclude::Test::Evaluation::ResultDifferenceAnalyser",logTr("Comparing boolean results between '%1' and '%2'.").arg(responseFileString).arg(otherResponseFileString),this); if (!isBooleanResultSimilarTo(booleanResult,nodeNumber,otherResponseFileString)) { diffCount = 1; } } else if (resultElement.nodeName() == "Classes") { if (!classesResult) { classesResult = mOWLlinkResultParser.parseClassesResult(&resultElement); } QString otherResponseFileString = otherResponseFileStringList.first(); LOG(INFO,"::Konclude::Test::Evaluation::ResultDifferenceAnalyser",logTr("Comparing classes results between '%1' and '%2'.").arg(responseFileString).arg(otherResponseFileString),this); if (!isClassesResultSimilarTo(classesResult,nodeNumber,otherResponseFileString)) { diffCount = 1; } } else if (resultElement.nodeName() == "results") { // handle SPARQL binding results QHash variableNameIndexHash; if (!variableBindingsResult) { variableBindingsResult = mSPARQLResultParser.parseVariableBindingsSet(&resultElement, &variableNameIndexHash, true); } QString otherResponseFileString = otherResponseFileStringList.first(); QString reportingFileName = responseFileString + "--vs--" + otherResponseFileString + ".txt"; reportingFileName = reportingFileName.replace("/", "_").replace(":", "_"); reportingFileName = detailedReportingOutputDirectory + reportingFileName; LOG(INFO, "::Konclude::Test::Evaluation::ResultDifferenceAnalyser", logTr("Comparing variable binding results between '%1' and '%2', reporting details to '%3'.").arg(responseFileString).arg(otherResponseFileString).arg(reportingFileName), this); if (!isVariableBindingResultSimilarTo(variableBindingsResult, nodeNumber, otherResponseFileString, &variableNameIndexHash, reportingFileName)) { diffCount = 1; } } compareDiffCountHash->insert(QPair(minVal,maxVal),diffCount); } reasonerDifferentResultCount += diffCount; if (classHierResult) { if (!lastClassHierResult) { lastClassHierResult = classHierResult; lastClassHierResultNumber = nodeNumber; } else { if (lastClassHierResult != classHierResult) { delete classHierResult; } } } if (booleanResult) { delete booleanResult; } if (classesResult) { delete classesResult; } if (variableBindingsResult) { delete variableBindingsResult; } } if (resultElement.nodeName() != "answering-statistics") { ++nodeNumber; } resultElement = resultElement.nextSiblingElement(); } } mReasonerCompareDiffCountHash.insert(QPair(minVal,maxVal),reasonerDifferentResultCount); } } } totalDifferentResultCount += reasonerDifferentResultCount; } } if (lastClassHierResult) { delete lastClassHierResult; } } return totalDifferentResultCount; } bool CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser::isClassHierarchyResultSimilarTo(CClassHierarchyResult* classHierResult, cint64 nodeNumber, const QString& otherResponseFileString) { bool resultSimilar = true; QFile responseFile(otherResponseFileString); if (responseFile.open(QIODevice::ReadOnly)) { QDomDocument document; document.setContent(&responseFile,false); QDomElement rootEl = document.documentElement(); QDomElement classResEl = rootEl.firstChildElement(); cint64 currNodeNumber = 0; if (!classResEl.isNull()) { while (!classResEl.isNull()) { if (classResEl.nodeName() == "Error") { break; } else if (currNodeNumber == nodeNumber && classResEl.nodeName() == "ClassHierarchy") { CClassHierarchyResult* otherClassHierResult = mOWLlinkResultParser.parseClassHierarchyResult(&classResEl); if (otherClassHierResult) { if (!classHierResult->isResultEquivalentTo(otherClassHierResult)) { resultSimilar = false; } delete otherClassHierResult; } break; } else { ++currNodeNumber; if (currNodeNumber > nodeNumber) { break; } classResEl = classResEl.nextSiblingElement(); } } } } return resultSimilar; } bool CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser::isBooleanResultSimilarTo(CBooleanQueryResult* booleanResult, cint64 nodeNumber, const QString& otherResponseFileString) { bool resultSimilar = true; QFile responseFile(otherResponseFileString); if (responseFile.open(QIODevice::ReadOnly)) { QDomDocument document; document.setContent(&responseFile,false); QDomElement rootEl = document.documentElement(); QDomElement classResEl = rootEl.firstChildElement(); cint64 currNodeNumber = 0; if (!classResEl.isNull()) { while (!classResEl.isNull()) { if (classResEl.nodeName() == "Error") { break; } else if (currNodeNumber == nodeNumber && classResEl.nodeName() == "BooleanResponse") { CBooleanQueryResult* otherBooleanResult = mOWLlinkResultParser.parseBooleanQueryResult(&classResEl); if (otherBooleanResult) { if (!booleanResult->isResultEquivalentTo(otherBooleanResult)) { resultSimilar = false; } delete otherBooleanResult; } break; } else { ++currNodeNumber; if (currNodeNumber > nodeNumber) { break; } classResEl = classResEl.nextSiblingElement(); } } } } return resultSimilar; } bool CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser::isVariableBindingResultSimilarTo(CVariableBindingsAnswersSetResult* varBindingResult, cint64 nodeNumber, const QString& otherResponseFileString, QHash* variableNameIndexHash, const QString& reportingFileName) { bool resultSimilar = true; QFile responseFile(otherResponseFileString); if (responseFile.open(QIODevice::ReadOnly)) { QDomDocument document; document.setContent(&responseFile,false); QDomElement rootEl = document.documentElement(); QDomElement resultElement = rootEl.firstChildElement(); cint64 currNodeNumber = 0; if (!resultElement.isNull()) { while (!resultElement.isNull()) { if (resultElement.nodeName() == "Error") { break; } else if (currNodeNumber == nodeNumber && resultElement.nodeName() == "results") { CVariableBindingsAnswersSetResult* otherVarBindingResult = mSPARQLResultParser.parseVariableBindingsSet(&resultElement, variableNameIndexHash, true); if (otherVarBindingResult) { if (!varBindingResult || !varBindingResult->isResultEquivalentTo(otherVarBindingResult)) { resultSimilar = false; if (varBindingResult && otherVarBindingResult) { CVariableBindingsAnswersResultIterator* it = varBindingResult->getVariableBindingsAnswersIterator(); while (it->hasNext()) { CVariableBindingsAnswerResult* answerResult = it->getNext(); if (!otherVarBindingResult->hasResultVariableBindings(answerResult)) { QFile reportingFile(reportingFileName); if (reportingFile.open(QFile::Append)) { reportingFile.write(QString("Difference (additional answer) for " + answerResult->getQueryResultString() + "\r\n").toLocal8Bit()); reportingFile.close(); } } } CVariableBindingsAnswersResultIterator* otherIt = otherVarBindingResult->getVariableBindingsAnswersIterator(); while (otherIt->hasNext()) { CVariableBindingsAnswerResult* answerResult = otherIt->getNext(); if (!varBindingResult->hasResultVariableBindings(answerResult)) { QFile reportingFile(reportingFileName); if (reportingFile.open(QFile::Append)) { reportingFile.write(QString("Difference (missing answer) for " + answerResult->getQueryResultString() + "\r\n").toLocal8Bit()); reportingFile.close(); } } } } } delete otherVarBindingResult; } else if (varBindingResult) { resultSimilar = false; } break; } else { if (resultElement.nodeName() != "answering-statistics") { ++currNodeNumber; } if (currNodeNumber > nodeNumber) { break; } resultElement = resultElement.nextSiblingElement(); } } } } return resultSimilar; } bool CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser::isClassesResultSimilarTo(CClassesResult* classesResult, cint64 nodeNumber, const QString& otherResponseFileString) { bool resultSimilar = true; QFile responseFile(otherResponseFileString); if (responseFile.open(QIODevice::ReadOnly)) { QDomDocument document; document.setContent(&responseFile,false); QDomElement rootEl = document.documentElement(); QDomElement classResEl = rootEl.firstChildElement(); cint64 currNodeNumber = 0; if (!classResEl.isNull()) { while (!classResEl.isNull()) { if (classResEl.nodeName() == "Error") { break; } else if (currNodeNumber == nodeNumber && classResEl.nodeName() == "Classes") { CClassesResult* otherClassesResult = mOWLlinkResultParser.parseClassesResult(&classResEl); if (otherClassesResult) { if (!classesResult->isResultEquivalentTo(otherClassesResult)) { resultSimilar = false; } delete otherClassesResult; } break; } else { ++currNodeNumber; if (currNodeNumber > nodeNumber) { break; } classResEl = classResEl.nextSiblingElement(); } } } } return resultSimilar; } bool CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser::analyseGroupedEvaluationData(const QStringList& groubList, const QStringList& requestList, const QStringList& reasonerList, const QString& outputDirectory, CReasonerEvaluationAnalyseContext* context, CReasonerEvaluationGroupRequestSelector* selector) { mOutputFileName = QString("%1").arg(selector->getOutputString()); bool hasSelectedRequestFile = false; QStringList titleList; titleList.append("Requests"); titleList.append(reasonerList); titleList.append("Average"); titleList.append("Min"); titleList.append("Max"); QDir outDir(outputDirectory); outDir.mkpath(mOutputFileName); QString outputFileString(outputDirectory+mOutputFileName+"result-difference-to-other-reasoners"); QString detailedOutputReportingDirectory = QString(outputDirectory + mOutputFileName + "/result-differences-reports/"); outDir.mkpath(mOutputFileName + "/result-differences-reports"); CReasonerEvaluationTableMultiFormatOutputWriter tableMultiFormatOutputWriter; tableMultiFormatOutputWriter.addColumnTitles(titleList); foreach (QString groupString, groubList) { if (selector->isGroupSelected(groupString)) { QSet* groupRequestList = getGroupRequests(groupString); foreach (QString requestString, *groupRequestList) { if (selector->isRequestSelected(groupString,requestString)) { bool allDataAvailable = true; foreach (QString reasonerString, reasonerList) { CReasonerEvaluationDataValue* evalData = getGroupRequestReasonerEvaluationData(groupString,requestString,reasonerString); if (!evalData || evalData->hasDataNotAvailableFlag()) { allDataAvailable = false; } } if (allDataAvailable) { hasSelectedRequestFile = true; double min = DOUBLE_MAX; double max = DOUBLE_MIN; double mean = 0.; cint64 count = 0; tableMultiFormatOutputWriter.addNewTableRow(requestString); foreach (QString reasonerString, reasonerList) { CReasonerEvaluationDataValue* evalData = getGroupRequestReasonerEvaluationData(groupString,requestString,reasonerString); CReasonerEvaluationStringListValue* stringListDataValue = dynamic_cast(evalData); if (stringListDataValue) { QStringList responseFiles = stringListDataValue->getValue(); double diffCount = compareResultsSimilarityWithOtherReasoners(groupString,requestString,reasonerString,stringListDataValue,reasonerList, detailedOutputReportingDirectory); tableMultiFormatOutputWriter.addValuesToLastTableRow(QList()<()<isGNUPlottingActivated()) { reasonerCompGNUPlotFileList += tableMultiFormatOutputWriter.writeGNUPlotDataTable(outputFileString); reasonerCompGNUPlotFileList += tableMultiFormatOutputWriter.writeGNUPlotDataTablesSorted(outputFileString); foreach (QString plotFileString, reasonerCompGNUPlotFileList) { if (true /*plotFileString.contains("-by-Average-")*/) { QString outputFile = plotFileString; outputFile.remove(".dat"); QString title = getPrettyTitleFromOutputFile(outputFile); context->getGNUPlotPlotter()->createReasonerComparisonPlot(plotFileString,context->getAdditionalTitleString()+title+"[# differences]",reasonerList,outputFile); } } } if (context->isHighchartPlottingActivated()) { foreach (QString plotFileString, reasonerCompHighchartPlotFileList) { if (true /*plotFileString.contains("-by-Average-")*/) { QString outputFile = plotFileString; outputFile.remove(".csv"); QString title = getPrettyTitleFromOutputFile(outputFile); context->getHighchartPlotter()->createReasonerDifferencePlot(plotFileString,context->getAdditionalTitleString()+title+"[# differences]",reasonerList,outputFile,selector->getNameString()+" :: "+context->getAdditionalTitleString()); } } } } return true; } QString CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser::getPrettyTitleFromOutputFile(const QString& outputFile) { QString title = outputFile; title.remove("table"); cint64 slashPos = title.lastIndexOf("/"); if (slashPos >= 0) { title = title.mid(slashPos+1); } title.replace("-"," "); title.replace("result difference to other reasoners","Result Differences to other Reasoners, "); title.replace("sorted","Sorted"); if (title.contains(" ascending")) { title.replace(" ascending",""); title.append(" in Ascending Order "); } if (title.contains(" descending")) { title.replace(" descending",""); title.append(" in Descending Order "); } return title; } }; // end namespace Evaluation }; // end namespace Test }; // end namespace Konclude