/* * Copyright (C) 2013-2015, 2019 by the Konclude Developer Team. * * This file is part of the reasoning system Konclude. * For details and support, see . * * Konclude is free software: you can redistribute it and/or modify * it under the terms of version 3 of the GNU Lesser General Public * License (LGPLv3) as published by the Free Software Foundation. * * Konclude is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU (Lesser) General Public License for more details. * * You should have received a copy of the GNU (Lesser) General Public * License along with Konclude. If not, see . * */ #include "CReasonerEvaluationGroupRequestReasonerResultSubsumptionAnalyser.h" namespace Konclude { namespace Test { namespace Evaluation { CReasonerEvaluationGroupRequestReasonerResultSubsumptionAnalyser::CReasonerEvaluationGroupRequestReasonerResultSubsumptionAnalyser() { } CReasonerEvaluationGroupRequestReasonerResultSubsumptionAnalyser::~CReasonerEvaluationGroupRequestReasonerResultSubsumptionAnalyser() { } cint64 CReasonerEvaluationGroupRequestReasonerResultSubsumptionAnalyser::compareResultsSubsumptionWithOtherReasoners(const QString& groupString, const QString& requestString, const QString& reasonerString, CReasonerEvaluationStringListValue* stringListDataValue, const QStringList& reasonerList, const QString& detailedReportingOutputDirectory) { cint64 totalResultSubsumptionCount = 0; const QStringList& responseFileStringList = stringListDataValue->getValue(); if (!responseFileStringList.isEmpty()) { QString responseFileString = responseFileStringList.first(); QFile responseFile(responseFileString); bool fileOpened = false; QDomDocument document; CClassHierarchyResult* lastClassHierResult = nullptr; cint64 lastClassHierResultNumber = 0; foreach (const QString& otherReasoner, reasonerList) { if (otherReasoner != reasonerString) { cint64 reasonerResultSubsumptionCount = 0; CReasonerEvaluationDataValue* otherReasonerEvalData = getGroupRequestReasonerEvaluationData(groupString,requestString,otherReasoner); CReasonerEvaluationStringListValue* otherReasonerStringListDataValue = dynamic_cast(otherReasonerEvalData); if (otherReasonerStringListDataValue) { const QStringList& otherResponseFileStringList = otherReasonerStringListDataValue->getValue(); if (!otherResponseFileStringList.isEmpty()) { if (mReasonerCompareResultSubsumptionCountHash.contains(QPair(stringListDataValue, otherReasonerStringListDataValue))) { reasonerResultSubsumptionCount += mReasonerCompareResultSubsumptionCountHash.value(QPair(stringListDataValue, otherReasonerStringListDataValue)); } else { if (!fileOpened) { if (responseFile.open(QIODevice::ReadOnly)) { document.setContent(&responseFile,false); responseFile.close(); } fileOpened = true; } QDomElement rootEl = document.documentElement(); QDomElement resultElement = rootEl.firstChildElement(); bool foundResult = false; bool foundError = false; cint64 nodeNumber = 0; if (!resultElement.isNull()) { while (!resultElement.isNull()) { if (resultElement.nodeName() == "Error") { foundError = true; break; } else if (resultElement.nodeName() == "results") { foundResult = true; CClassHierarchyResult* classHierResult = nullptr; if (lastClassHierResult && lastClassHierResultNumber == nodeNumber) { classHierResult = lastClassHierResult; } CBooleanQueryResult* booleanResult = nullptr; CClassesResult* classesResult = nullptr; CVariableBindingsAnswersSetResult* variableBindingsResult = nullptr; cint64 subsumingCount = 0; QHash,cint64>*& compareDiffCountHash = mNumberCompareDiffCountHashHash[nodeNumber]; if (!compareDiffCountHash) { compareDiffCountHash = new QHash,cint64>(); } if (compareDiffCountHash->contains(QPair(stringListDataValue, otherReasonerStringListDataValue))) { subsumingCount += compareDiffCountHash->value(QPair(stringListDataValue, otherReasonerStringListDataValue)); } else { if (resultElement.nodeName() == "results") { // handle SPARQL binding results QHash variableNameIndexHash; if (!variableBindingsResult) { variableBindingsResult = mSPARQLResultParser.parseVariableBindingsSet(&resultElement, &variableNameIndexHash, true); } QString otherResponseFileString = otherResponseFileStringList.first(); LOG(INFO, "::Konclude::Test::Evaluation::ResultDifferenceAnalyser", logTr("Comparing variable binding results between '%1' and '%2'.").arg(responseFileString).arg(otherResponseFileString), this); if (isVariableBindingResultSubsuming(variableBindingsResult, nodeNumber, otherResponseFileString, &variableNameIndexHash)) { subsumingCount = 1; } } compareDiffCountHash->insert(QPair(stringListDataValue, otherReasonerStringListDataValue),subsumingCount); } reasonerResultSubsumptionCount += subsumingCount; if (classHierResult) { if (!lastClassHierResult) { lastClassHierResult = classHierResult; lastClassHierResultNumber = nodeNumber; } else { if (lastClassHierResult != classHierResult) { delete classHierResult; } } } if (booleanResult) { delete booleanResult; } if (classesResult) { delete classesResult; } if (variableBindingsResult) { delete variableBindingsResult; } } if (resultElement.nodeName() != "answering-statistics") { ++nodeNumber; } resultElement = resultElement.nextSiblingElement(); } } mReasonerCompareResultSubsumptionCountHash.insert(QPair(stringListDataValue, otherReasonerStringListDataValue),reasonerResultSubsumptionCount); } } } totalResultSubsumptionCount += reasonerResultSubsumptionCount; } } if (lastClassHierResult) { delete lastClassHierResult; } } return totalResultSubsumptionCount; } bool CReasonerEvaluationGroupRequestReasonerResultSubsumptionAnalyser::isVariableBindingResultSubsuming(CVariableBindingsAnswersSetResult* varBindingResult, cint64 nodeNumber, const QString& otherResponseFileString, QHash* variableNameIndexHash) { bool resultSubsumption = false; QFile responseFile(otherResponseFileString); if (responseFile.open(QIODevice::ReadOnly)) { QDomDocument document; document.setContent(&responseFile,false); QDomElement rootEl = document.documentElement(); QDomElement resultElement = rootEl.firstChildElement(); cint64 currNodeNumber = 0; if (!resultElement.isNull()) { while (!resultElement.isNull()) { if (resultElement.nodeName() == "Error") { break; } else if (currNodeNumber == nodeNumber && resultElement.nodeName() == "results") { CVariableBindingsAnswersSetResult* otherVarBindingResult = mSPARQLResultParser.parseVariableBindingsSet(&resultElement, variableNameIndexHash, true); if (otherVarBindingResult) { if (varBindingResult && varBindingResult->isResultEquivalentTo(otherVarBindingResult)) { resultSubsumption = true; } else if (varBindingResult) { resultSubsumption = true; if (varBindingResult && otherVarBindingResult && resultSubsumption) { CVariableBindingsAnswersResultIterator* it = otherVarBindingResult->getVariableBindingsAnswersIterator(); while (it->hasNext()) { CVariableBindingsAnswerResult* answerResult = it->getNext(); if (!varBindingResult->hasResultVariableBindings(answerResult)) { resultSubsumption = false; } } } } delete otherVarBindingResult; } else if (varBindingResult) { resultSubsumption = false; } break; } else { if (resultElement.nodeName() != "answering-statistics") { ++currNodeNumber; } if (currNodeNumber > nodeNumber) { break; } resultElement = resultElement.nextSiblingElement(); } } } } return resultSubsumption; } bool CReasonerEvaluationGroupRequestReasonerResultSubsumptionAnalyser::analyseGroupedEvaluationData(const QStringList& groubList, const QStringList& requestList, const QStringList& reasonerList, const QString& outputDirectory, CReasonerEvaluationAnalyseContext* context, CReasonerEvaluationGroupRequestSelector* selector) { mOutputFileName = QString("%1").arg(selector->getOutputString()); bool hasSelectedRequestFile = false; QStringList titleList; titleList.append("Requests"); titleList.append(reasonerList); titleList.append("Average"); titleList.append("Min"); titleList.append("Max"); QDir outDir(outputDirectory); outDir.mkpath(mOutputFileName); QString outputFileString(outputDirectory+mOutputFileName+"result-subsumption-over-other-reasoners"); QString detailedOutputReportingDirectory = QString(outputDirectory + mOutputFileName + "/result-subsumption-reports/"); outDir.mkpath(mOutputFileName + "/result-subsumption-reports"); CReasonerEvaluationTableMultiFormatOutputWriter tableMultiFormatOutputWriter; tableMultiFormatOutputWriter.addColumnTitles(titleList); foreach (QString groupString, groubList) { if (selector->isGroupSelected(groupString)) { QSet* groupRequestList = getGroupRequests(groupString); foreach (QString requestString, *groupRequestList) { if (selector->isRequestSelected(groupString,requestString)) { bool allDataAvailable = true; foreach (QString reasonerString, reasonerList) { CReasonerEvaluationDataValue* evalData = getGroupRequestReasonerEvaluationData(groupString,requestString,reasonerString); if (!evalData || evalData->hasDataNotAvailableFlag()) { allDataAvailable = false; } } if (allDataAvailable) { hasSelectedRequestFile = true; double min = DOUBLE_MAX; double max = DOUBLE_MIN; double mean = 0.; cint64 count = 0; tableMultiFormatOutputWriter.addNewTableRow(requestString); foreach (QString reasonerString, reasonerList) { CReasonerEvaluationDataValue* evalData = getGroupRequestReasonerEvaluationData(groupString,requestString,reasonerString); CReasonerEvaluationStringListValue* stringListDataValue = dynamic_cast(evalData); if (stringListDataValue) { QStringList responseFiles = stringListDataValue->getValue(); double subsumptionCount = compareResultsSubsumptionWithOtherReasoners(groupString,requestString,reasonerString,stringListDataValue,reasonerList, detailedOutputReportingDirectory); tableMultiFormatOutputWriter.addValuesToLastTableRow(QList()<()<isGNUPlottingActivated()) { reasonerCompGNUPlotFileList += tableMultiFormatOutputWriter.writeGNUPlotDataTable(outputFileString); reasonerCompGNUPlotFileList += tableMultiFormatOutputWriter.writeGNUPlotDataTablesSorted(outputFileString); foreach (QString plotFileString, reasonerCompGNUPlotFileList) { if (true /*plotFileString.contains("-by-Average-")*/) { QString outputFile = plotFileString; outputFile.remove(".dat"); QString title = getPrettyTitleFromOutputFile(outputFile); context->getGNUPlotPlotter()->createReasonerComparisonPlot(plotFileString,context->getAdditionalTitleString()+title+"[# subsumptions]",reasonerList,outputFile); } } } if (context->isHighchartPlottingActivated()) { foreach (QString plotFileString, reasonerCompHighchartPlotFileList) { if (true /*plotFileString.contains("-by-Average-")*/) { QString outputFile = plotFileString; outputFile.remove(".csv"); QString title = getPrettyTitleFromOutputFile(outputFile); context->getHighchartPlotter()->createReasonerSubsumptionPlot(plotFileString,context->getAdditionalTitleString()+title+"[# subsumptions]",reasonerList,outputFile,selector->getNameString()+" :: "+context->getAdditionalTitleString()); } } } } return true; } QString CReasonerEvaluationGroupRequestReasonerResultSubsumptionAnalyser::getPrettyTitleFromOutputFile(const QString& outputFile) { QString title = outputFile; title.remove("table"); cint64 slashPos = title.lastIndexOf("/"); if (slashPos >= 0) { title = title.mid(slashPos+1); } title.replace("-"," "); title.replace("result subsumptions over other reasoners","Result Subsumptions over other Reasoners, "); title.replace("sorted","Sorted"); if (title.contains(" ascending")) { title.replace(" ascending",""); title.append(" in Ascending Order "); } if (title.contains(" descending")) { title.replace(" descending",""); title.append(" in Descending Order "); } return title; } }; // end namespace Evaluation }; // end namespace Test }; // end namespace Konclude