/*
* Copyright (C) 2013-2015, 2019 by the Konclude Developer Team.
*
* This file is part of the reasoning system Konclude.
* For details and support, see .
*
* Konclude is free software: you can redistribute it and/or modify
* it under the terms of version 3 of the GNU Lesser General Public
* License (LGPLv3) as published by the Free Software Foundation.
*
* Konclude is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU (Lesser) General Public License for more details.
*
* You should have received a copy of the GNU (Lesser) General Public
* License along with Konclude. If not, see .
*
*/
#include "CReasonerEvaluationManagerThread.h"
namespace Konclude {
namespace Test {
namespace Evaluation {
CReasonerEvaluationManagerThread::CReasonerEvaluationManagerThread(CConfiguration *config) : CThread("ReasonerEvaluationManagerThread"), CLogIdentifier("::Konclude::Test::Evaluation::ReasonerEvaluationManagerThread",this) {
mConfig = config;
mSatisticBasePath = "Evaluation/Infos/Statistics/Requests/";
startThread();
}
CReasonerEvaluationManagerThread::~CReasonerEvaluationManagerThread() {
}
CReasonerEvaluationManager* CReasonerEvaluationManagerThread::realizeEvaluationCommand(CEvaluateTestsuiteCommand* evalTestsuiteComm) {
postEvent(new CReasonerEvaluationProgramEvent(evalTestsuiteComm));
return this;
}
bool CReasonerEvaluationManagerThread::processCustomsEvents(QEvent::Type type, CCustomEvent* event) {
if (CThread::processCustomsEvents(type,event)) {
return true;
} else {
if (type == CReasonerEvaluationProgramEvent::EVENTTYPE) {
CReasonerEvaluationProgramEvent* repe = (CReasonerEvaluationProgramEvent*)event;
CEvaluateTestsuiteCommand* evalTestComm = repe->getEvaluationTestsuiteCommand();
CCommandRecordRouter commandRecordRouter(evalTestComm,this);
CStartProcessCommandRecord::makeRecord(&commandRecordRouter);
initEvaluationProgram(evalTestComm);
evaluationProgram(evalTestComm);
analysingProgram(evalTestComm);
CStopProcessCommandRecord::makeRecord(&commandRecordRouter);
CFinishProcessCommandRecord::makeRecord(&commandRecordRouter);
return true;
}
}
return false;
}
CReasonerEvaluationManager* CReasonerEvaluationManagerThread::initEvaluationProgram(CEvaluateTestsuiteCommand* evalTestsuiteComm) {
CReasonerEvaluationProgram* evalProg = evalTestsuiteComm->getEvaluationTestsuite();
mReasonerNameStringList.clear();
mReasonerOutputDirStringList.clear();
mTestTimeout = 0;
mTestCount = 0;
mOneReasonerFailed = false;
mAllReasonerFailed = true;
mFiltering = new CReasonerEvaluationFiltering(mSatisticBasePath);
if (!evalProg->getFilterStringList().isEmpty()) {
mFitlerStringList = evalProg->getFilterStringList();
mFiltering->loadFiltersFromFile(mFitlerStringList);
} else {
mFiltering->loadEmptyFilter();
}
return this;
}
bool CReasonerEvaluationManagerThread::evaluationProgram(CEvaluateTestsuiteCommand* evalTestsuiteComm) {
CReasonerEvaluationProgram* evalProg = evalTestsuiteComm->getEvaluationTestsuite();
QStringList reasonerConfigList = evalProg->getReasonerStringList();
QString testInputDirectory = evalProg->getInputDirectoryString();
QString testOutputDirectory = evalProg->getOutputDirectoryString();
QString plattform = CConfigDataReader::readConfigString(mConfig,"Konclude.Evaluation.Platform");
QString globalConfig = evalProg->getConfigFileString();
bool evaluationRunned = false;
if (plattform != "UNSPECIFIED") {
QTime totalTimer;
totalTimer.start();
LOG(INFO,getLogDomain(),logTr("Starting evaluation."),this);
foreach (QString reasonerConfig, reasonerConfigList) {
QTime timer;
timer.start();
LOG(INFO,getLogDomain(),logTr("Starting evaluation for reasoner '%1'.").arg(reasonerConfig),this);
CReasonerEvaluationTestsuite* reasonerEvalTestsuite = new CReasonerEvaluationTestsuite();
reasonerEvalTestsuite->initEvaluationTestsuite(globalConfig,reasonerConfig,testInputDirectory,testOutputDirectory,plattform,mFiltering);
CBlockingCallbackData blockCallData;
CReasonerEvaluationExecutorThread* reasonerExecutorThread = new CReasonerEvaluationExecutorThread(mConfig);
reasonerExecutorThread->executeEvaluation(reasonerEvalTestsuite,&blockCallData);
blockCallData.waitForCallback();
CReasonerEvaluationEvaluatedCallbackContextData* evaluatedCallbackContextData = (CReasonerEvaluationEvaluatedCallbackContextData*)blockCallData.getCallbackDataContext();
if (!evaluatedCallbackContextData->getEvaluationFailed()) {
evaluationRunned |= evaluatedCallbackContextData->getEvaluationRunned();
mReasonerNameStringList.append(evaluatedCallbackContextData->getReasonerNameString());
mReasonerOutputDirStringList.append(evaluatedCallbackContextData->getReasonerOutputDirectoryString());
mTestTimeout = evaluatedCallbackContextData->getTestTimeout();
mTestCount = evaluatedCallbackContextData->getTestCount();
mAllReasonerFailed = false;
} else {
mOneReasonerFailed = true;
mFailedReasonerStringList.append(reasonerConfig);
}
delete reasonerExecutorThread;
cint64 elapsedTime = timer.elapsed();
LOG(INFO,getLogDomain(),logTr("Finished evaluation for reasoner '%1' in '%2' seconds.").arg(reasonerConfig).arg(elapsedTime/1000),this);
}
cint64 totalElapsedTime = totalTimer.elapsed();
LOG(INFO,getLogDomain(),logTr("Finished evaluation in '%1' seconds.").arg(totalElapsedTime/1000),this);
} else {
LOG(ERROR,getLogDomain(),logTr("Evaluation not possible, platform not specified."),this);
}
return evaluationRunned;
}
CReasonerEvaluationGroupRequestSelector* CReasonerEvaluationManagerThread::getSelectors(const QString& requestDirectory) {
CReasonerEvaluationGroupRequestSelector* selectors = nullptr;
CReasonerEvaluationExpressivitySelectorFactory expEvalSelFactory("ExpressivenessGrouped","ExpressivenessGrouped/",mSatisticBasePath);
selectors = expEvalSelFactory.getSelectors(selectors);
CReasonerEvaluationDirectoryStructureSelectorFactory dirEvalSelFactory(requestDirectory,requestDirectory,"DirectoryGrouped","DirectoryGrouped/");
selectors = dirEvalSelFactory.getSelectors(selectors);
CReasonerEvaluationProfileSelectorFactory profileEvalSelFactory("ProfileGrouped","ProfileGrouped/",mSatisticBasePath);
selectors = profileEvalSelFactory.getSelectors(selectors);
return selectors;
}
QString CReasonerEvaluationManagerThread::getAnalyserOutputDirectory(const QString& baseDirectory, const QString& evaluationProgramName, const QString& platformName, const QString& analyserName) {
QString outputDirectory = baseDirectory;
if (outputDirectory.contains("$$_ANALYSER_NAME_$$")) {
outputDirectory = outputDirectory.replace("$$_ANALYSER_NAME_$$",analyserName);
outputDirectory = outputDirectory.replace("$$_EVALUATION_PROGRAM_NAME_$$",evaluationProgramName);
outputDirectory = outputDirectory.replace("$$_PLATFORM_$$",platformName);
} else {
outputDirectory = outputDirectory+evaluationProgramName+"/"+platformName+"/"+analyserName+"/";
}
QDir currDir(".");
currDir.mkpath(outputDirectory);
return outputDirectory;
}
bool CReasonerEvaluationManagerThread::analyseTimes(const QString& analysingDirectoryString, const QString& evaluationProgramName, const QString& plattform, const QString& analyserOutputDirectoryName, const QString& requestDirectory, cint64 testCountCut, cint64 timeoutCut, cint64 errorPunishmentTime, CReasonerEvaluationDataValueCacher* dataValueCacher, CReasonerEvaluationExtractor::EXTRACTORTYPE extractorType, const QString& extractionName, const QString& contextSpecificationName) {
bool analysed = false;
QString outputDirectory = getAnalyserOutputDirectory(analysingDirectoryString,evaluationProgramName,plattform,analyserOutputDirectoryName);
CReasonerEvaluationAnalyserChecker analysingUpdateChecker;
if (analysingUpdateChecker.checkAnalysingUpdateNecessary(mReasonerOutputDirStringList,outputDirectory+"AnalysingUpdateCheckingData.dat",testCountCut)) {
LOG(INFO,getLogDomain(),logTr("Analysing %1 times for '%2' into directory '%3'.").arg(extractionName).arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
CReasonerEvaluationSpecifiedTimeExtractor* timeExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(extractorType,timeoutCut,errorPunishmentTime,dataValueCacher);
CReasonerEvaluationAvaragerSummarizer* avarageSummarize = new CReasonerEvaluationAvaragerSummarizer();
CReasonerEvaluationAnalyseContext* reasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig,contextSpecificationName);
CReasonerEvaluationCollector* collector = new CReasonerEvaluationCollector(timeExtractor,avarageSummarize,mFiltering);
CReasonerEvaluationDataValueGroupCollectionReasonerComparison* reasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
for (QStringList::const_iterator it1 = mReasonerNameStringList.constBegin(), it2 = mReasonerOutputDirStringList.constBegin(), it1End = mReasonerNameStringList.constEnd(), it2End = mReasonerOutputDirStringList.constEnd(); it1 != it1End && it2 != it2End; ++it1, ++it2) {
QString reasonerName(*it1);
QString reasonerPath(*it2);
LOG(INFO,getLogDomain(),logTr("Extracting %1 times for reasoner '%2' from responses in directory '%3'.").arg(extractionName).arg(reasonerName).arg(reasonerPath),this);
collector->collectReasonerEvaluationDataValues(reasonerComp,reasonerName,reasonerPath);
}
LOG(INFO,getLogDomain(),logTr("Comparing %1 times for '%2'.").arg(extractionName).arg(mReasonerNameStringList.join(", ")),this);
CReasonerEvaluationGroupRequestSelector* selectors = getSelectors(requestDirectory);
CReasonerEvaluationGroupRequestReasonerCSVComparisonAnalyser* analyser1 = new CReasonerEvaluationGroupRequestReasonerCSVComparisonAnalyser();
analyser1->analyseEvaluationData(reasonerComp,reasonerEvaluationContext,outputDirectory,selectors);
CReasonerEvaluationGroupRequestReasonerWithMeanMinMaxCSVComparisonAnalyser* analyser2 = new CReasonerEvaluationGroupRequestReasonerWithMeanMinMaxCSVComparisonAnalyser();
analyser2->analyseEvaluationData(reasonerComp,reasonerEvaluationContext,outputDirectory,selectors);
CReasonerEvaluationGroupReasonerWinnerCountCSVComparisonAnalyser* analyser3 = new CReasonerEvaluationGroupReasonerWinnerCountCSVComparisonAnalyser();
analyser3->analyseEvaluationData(reasonerComp,reasonerEvaluationContext,outputDirectory,selectors);
CReasonerEvaluationGroupRequestReasonerBucketCSVComparisonAnalyser* analyser4 = new CReasonerEvaluationGroupRequestReasonerBucketCSVComparisonAnalyser(100.,timeoutCut,qSqrt(10.),timeoutCut);
analyser4->analyseEvaluationData(reasonerComp,reasonerEvaluationContext,outputDirectory,selectors);
CReasonerEvaluationGroupRequestReasonerBucketCSVComparisonAnalyser* analyser5 = new CReasonerEvaluationGroupRequestReasonerBucketCSVComparisonAnalyser(100.,timeoutCut,10.,timeoutCut);
analyser5->analyseEvaluationData(reasonerComp,reasonerEvaluationContext,outputDirectory,selectors);
LOG(INFO,getLogDomain(),logTr("Compared %1 times for '%2' into directory '%3'.").arg(extractionName).arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
analysingUpdateChecker.saveAnalysingUpdateCheckFile(outputDirectory+"AnalysingUpdateCheckingData.dat");
analysed = true;
} else {
LOG(WARNING,getLogDomain(),logTr("Compared %1 times for '%2' in directory '%3' are already up to date.").arg(extractionName).arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
}
return analysed;
}
CReasonerEvaluationManager* CReasonerEvaluationManagerThread::analysingProgram(CEvaluateTestsuiteCommand* evalTestsuiteComm) {
CReasonerEvaluationProgram* evalProg = evalTestsuiteComm->getEvaluationTestsuite();
bool analysed = false;
QString plattform = CConfigDataReader::readConfigString(mConfig,"Konclude.Evaluation.Platform");
QString analysingDirectoryString = CReasonerEvaluationPathConverter::getCorrectedPathString(evalProg->getAnalyserDirectoryString());
if (!analysingDirectoryString.endsWith("/")) {
analysingDirectoryString += "/";
}
QString requestDirectory = evalProg->getInputDirectoryString();
if (!requestDirectory.endsWith("/")) {
requestDirectory += "/";
}
QString evaluationProgramName = evalProg->getProgramName();
cint64 testCountCut = mTestCount;
cint64 timeoutCut = mTestTimeout;
cint64 errorPunishmentTime = 1;
CReasonerEvaluationDataValueCacher* dataValueCacher = new CReasonerEvaluationDataValueCacher();
if (!mAllReasonerFailed) {
QString dataString = QDateTime::currentDateTime().date().toString(Qt::ISODate);
if (plattform != "UNSPECIFIED") {
QTime totalTimer;
totalTimer.start();
LOG(INFO,getLogDomain(),logTr("Starting analysing."),this);
QStringList analysingStringList = evalProg->getAnalyserStringList();
foreach (QString ananysingString, analysingStringList) {
QTime timer;
timer.start();
LOG(INFO,getLogDomain(),logTr("Starting analysing for '%1'.").arg(ananysingString),this);
if (ananysingString == "ReasonerEvaluationClassificationTimesComparer") {
analyseTimes(analysingDirectoryString,evaluationProgramName,plattform,"ClassificationTimeComparison",requestDirectory,testCountCut,timeoutCut,errorPunishmentTime,dataValueCacher,CReasonerEvaluationExtractor::CLASSIFICATIONTIMEEXTRACTOR,"classification","Classification ");
}
if (ananysingString == "ReasonerEvaluationComplexQueryingTimesComparer") {
analyseTimes(analysingDirectoryString, evaluationProgramName, plattform, "ComplexQueryingTimeComparison", requestDirectory, testCountCut, timeoutCut, errorPunishmentTime, dataValueCacher, CReasonerEvaluationExtractor::COMPLEXQUERYINGTIMEEXTRACTOR, "complex-querying", "Complex Querying ");
}
if (ananysingString == "ReasonerEvaluationRealizationTimesComparer") {
analyseTimes(analysingDirectoryString,evaluationProgramName,plattform,"RealizationTimeComparison",requestDirectory,testCountCut,timeoutCut,errorPunishmentTime,dataValueCacher,CReasonerEvaluationExtractor::REALIZATIONTIMEEXTRACTOR,"realization","Realization ");
}
if (ananysingString == "ReasonerEvaluationConsistencyTimesComparer") {
analyseTimes(analysingDirectoryString,evaluationProgramName,plattform,"ConsistencyTimeComparison",requestDirectory,testCountCut,timeoutCut,errorPunishmentTime,dataValueCacher,CReasonerEvaluationExtractor::CONSISTENCYTIMEEXTRACTOR,"consistency","Consistency ");
}
if (ananysingString == "ReasonerEvaluationSatisfiabilityTimesComparer") {
analyseTimes(analysingDirectoryString,evaluationProgramName,plattform,"SatisfiabilityTimeComparison",requestDirectory,testCountCut,timeoutCut,errorPunishmentTime,dataValueCacher,CReasonerEvaluationExtractor::SATISFIABILITYTIMEEXTRACTOR,"satisfiability","Satisfiability ");
}
if (ananysingString == "ReasonerEvaluationLoadingTimesComparer") {
analyseTimes(analysingDirectoryString,evaluationProgramName,plattform,"LoadingTimeComparison",requestDirectory,testCountCut,timeoutCut,errorPunishmentTime,dataValueCacher,CReasonerEvaluationExtractor::LOADINGTIMEEXTRACTOR,"loading","Loading ");
}
if (ananysingString == "ReasonerEvaluationResponseTimesComparer") {
analyseTimes(analysingDirectoryString,evaluationProgramName,plattform,"ResponseTimeComparison",requestDirectory,testCountCut,timeoutCut,errorPunishmentTime,dataValueCacher,CReasonerEvaluationExtractor::TOTALRESPONSETIMEEXTRACTOR,"response","Response ");
}
if (ananysingString == "ReasonerEvaluationAccumulatedTimesComparer") {
QString outputDirectory = getAnalyserOutputDirectory(analysingDirectoryString,evaluationProgramName,plattform,"AccumulatedTimeComparison");
CReasonerEvaluationAnalyserChecker analysingUpdateChecker;
if (analysingUpdateChecker.checkAnalysingUpdateNecessary(mReasonerOutputDirStringList,outputDirectory+"AnalysingUpdateCheckingData.dat",testCountCut)) {
LOG(INFO,getLogDomain(),logTr("Analysing accumulated times for '%1' to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
CReasonerEvaluationSpecifiedTimeExtractor* classTimeExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::CLASSIFICATIONTIMEEXTRACTOR, timeoutCut, errorPunishmentTime, dataValueCacher);
CReasonerEvaluationSpecifiedTimeExtractor* complexQueryingTimeExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::COMPLEXQUERYINGTIMEEXTRACTOR, timeoutCut, errorPunishmentTime, dataValueCacher);
CReasonerEvaluationSpecifiedTimeExtractor* consTimeExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::CONSISTENCYTIMEEXTRACTOR,timeoutCut,errorPunishmentTime,dataValueCacher);
CReasonerEvaluationSpecifiedTimeExtractor* satTimeExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::SATISFIABILITYTIMEEXTRACTOR,timeoutCut,errorPunishmentTime,dataValueCacher);
CReasonerEvaluationSpecifiedTimeExtractor* realTimeExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::REALIZATIONTIMEEXTRACTOR,timeoutCut,errorPunishmentTime,dataValueCacher);
CReasonerEvaluationSpecifiedTimeExtractor* loadTimeExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::LOADINGTIMEEXTRACTOR,timeoutCut,errorPunishmentTime,dataValueCacher);
CReasonerEvaluationSpecifiedTimeExtractor* reponseTimeExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::TOTALRESPONSETIMEEXTRACTOR,timeoutCut,errorPunishmentTime,dataValueCacher);
CReasonerEvaluationSpecifiedTimeExtractor* errorExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::ERROREXTRACTOR,timeoutCut,errorPunishmentTime,dataValueCacher);
CReasonerEvaluationSpecifiedTimeExtractor* timeoutExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::TIMEOUTEXTRACTOR,timeoutCut,errorPunishmentTime,dataValueCacher);
CReasonerEvaluationAvaragerSummarizer* avarageSummarize = new CReasonerEvaluationAvaragerSummarizer();
CReasonerEvaluationCollector* responseCollector = new CReasonerEvaluationCollector(reponseTimeExtractor,avarageSummarize,mFiltering);
CReasonerEvaluationDataValueGroupCollectionReasonerComparison* responceReasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
CReasonerEvaluationCollector* loadingCollector = new CReasonerEvaluationCollector(loadTimeExtractor,avarageSummarize,mFiltering);
CReasonerEvaluationDataValueGroupCollectionReasonerComparison* loadingReasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
CReasonerEvaluationCollector* classificationCollector = new CReasonerEvaluationCollector(classTimeExtractor,avarageSummarize,mFiltering);
CReasonerEvaluationDataValueGroupCollectionReasonerComparison* classificationReasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
CReasonerEvaluationCollector* complexQueryingCollector = new CReasonerEvaluationCollector(complexQueryingTimeExtractor, avarageSummarize, mFiltering);
CReasonerEvaluationDataValueGroupCollectionReasonerComparison* complexQueryingReasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
CReasonerEvaluationCollector* realizationCollector = new CReasonerEvaluationCollector(realTimeExtractor,avarageSummarize,mFiltering);
CReasonerEvaluationDataValueGroupCollectionReasonerComparison* realizationReasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
CReasonerEvaluationCollector* consistencyCollector = new CReasonerEvaluationCollector(consTimeExtractor,avarageSummarize,mFiltering);
CReasonerEvaluationDataValueGroupCollectionReasonerComparison* consistencyReasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
CReasonerEvaluationCollector* satisfiabilityCollector = new CReasonerEvaluationCollector(satTimeExtractor,avarageSummarize,mFiltering);
CReasonerEvaluationDataValueGroupCollectionReasonerComparison* satisfiabilityReasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
CReasonerEvaluationCollector* errorCollector = new CReasonerEvaluationCollector(errorExtractor,avarageSummarize,mFiltering);
CReasonerEvaluationDataValueGroupCollectionReasonerComparison* errorReasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
CReasonerEvaluationCollector* timeoutCollector = new CReasonerEvaluationCollector(timeoutExtractor,avarageSummarize,mFiltering);
CReasonerEvaluationDataValueGroupCollectionReasonerComparison* timeoutReasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
for (QStringList::const_iterator it1 = mReasonerNameStringList.constBegin(), it2 = mReasonerOutputDirStringList.constBegin(), it1End = mReasonerNameStringList.constEnd(), it2End = mReasonerOutputDirStringList.constEnd(); it1 != it1End && it2 != it2End; ++it1, ++it2) {
QString reasonerName(*it1);
QString reasonerPath(*it2);
LOG(INFO,getLogDomain(),logTr("Extracting accumulated times for '%1' reasoner from responses in directory '%2'.").arg(reasonerName).arg(reasonerPath),this);
responseCollector->collectReasonerEvaluationDataValues(responceReasonerComp,reasonerName,reasonerPath);
loadingCollector->collectReasonerEvaluationDataValues(loadingReasonerComp,reasonerName,reasonerPath);
classificationCollector->collectReasonerEvaluationDataValues(classificationReasonerComp, reasonerName, reasonerPath);
complexQueryingCollector->collectReasonerEvaluationDataValues(complexQueryingReasonerComp, reasonerName, reasonerPath);
realizationCollector->collectReasonerEvaluationDataValues(realizationReasonerComp,reasonerName,reasonerPath);
consistencyCollector->collectReasonerEvaluationDataValues(consistencyReasonerComp,reasonerName,reasonerPath);
satisfiabilityCollector->collectReasonerEvaluationDataValues(satisfiabilityReasonerComp,reasonerName,reasonerPath);
errorCollector->collectReasonerEvaluationDataValues(errorReasonerComp,reasonerName,reasonerPath);
timeoutCollector->collectReasonerEvaluationDataValues(timeoutReasonerComp,reasonerName,reasonerPath);
}
LOG(INFO,getLogDomain(),logTr("Comparing accumulated times for '%1'.").arg(mReasonerNameStringList.join(", ")),this);
CReasonerEvaluationGroupRequestSelector* selectors = getSelectors(requestDirectory);
CReasonerEvaluationAnalyseContext* responseReasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig,"Response ","response-");
CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser* responseAnalyser1 = new CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser();
responseAnalyser1->analyseEvaluationData(responceReasonerComp,responseReasonerEvaluationContext,outputDirectory,selectors);
CReasonerEvaluationAnalyseContext* loadingReasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig,"Loading ","loading-");
CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser* loadingAnalyser1 = new CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser();
loadingAnalyser1->analyseEvaluationData(loadingReasonerComp,loadingReasonerEvaluationContext,outputDirectory,selectors);
CReasonerEvaluationAnalyseContext* classificationReasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig,"Classification ","classification-");
CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser* classificationAnalyser1 = new CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser();
classificationAnalyser1->analyseEvaluationData(classificationReasonerComp,classificationReasonerEvaluationContext,outputDirectory,selectors);
CReasonerEvaluationAnalyseContext* complexQueryingReasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig, "Complex Querying ", "complex-querying-");
CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser* complexQueryingAnalyser1 = new CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser();
complexQueryingAnalyser1->analyseEvaluationData(complexQueryingReasonerComp, complexQueryingReasonerEvaluationContext, outputDirectory, selectors);
CReasonerEvaluationAnalyseContext* realizationReasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig,"Realization ","realization-");
CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser* realizationAnalyser1 = new CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser();
realizationAnalyser1->analyseEvaluationData(realizationReasonerComp,realizationReasonerEvaluationContext,outputDirectory,selectors);
CReasonerEvaluationAnalyseContext* consistencyReasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig,"Consistency ","consistency-");
CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser* consistencyAnalyser1 = new CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser();
consistencyAnalyser1->analyseEvaluationData(consistencyReasonerComp,consistencyReasonerEvaluationContext,outputDirectory,selectors);
CReasonerEvaluationAnalyseContext* satisfiabilityReasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig,"Satisfiability ","satisfiability-");
CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser* satisfiabilityAnalyser1 = new CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser();
satisfiabilityAnalyser1->analyseEvaluationData(satisfiabilityReasonerComp,satisfiabilityReasonerEvaluationContext,outputDirectory,selectors);
CReasonerEvaluationAnalyseContext* errorReasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig,"Error ","error-");
CReasonerEvaluationGroupRequestReasonerAccumulatedCountCSVComparisonAnalyser* errorAnalyser1 = new CReasonerEvaluationGroupRequestReasonerAccumulatedCountCSVComparisonAnalyser();
errorAnalyser1->analyseEvaluationData(errorReasonerComp,errorReasonerEvaluationContext,outputDirectory,selectors);
CReasonerEvaluationAnalyseContext* timeoutReasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig,"Timeout ","timeout-");
CReasonerEvaluationGroupRequestReasonerAccumulatedCountCSVComparisonAnalyser* timeoutAnalyser1 = new CReasonerEvaluationGroupRequestReasonerAccumulatedCountCSVComparisonAnalyser();
timeoutAnalyser1->analyseEvaluationData(timeoutReasonerComp,timeoutReasonerEvaluationContext,outputDirectory,selectors);
LOG(INFO,getLogDomain(),logTr("Accumulated times for '%1' compared to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
analysingUpdateChecker.saveAnalysingUpdateCheckFile(outputDirectory+"AnalysingUpdateCheckingData.dat");
analysed = true;
} else {
LOG(WARNING,getLogDomain(),logTr("Accumulated times for '%1' in '%2' already up to date.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
}
}
if (ananysingString == "ReasonerEvaluationStatisticValuesComparer") {
QString outputDirectory = getAnalyserOutputDirectory(analysingDirectoryString,evaluationProgramName,plattform,"StatisticValuesComparison");
CReasonerEvaluationAnalyserChecker analysingUpdateChecker;
if (analysingUpdateChecker.checkAnalysingUpdateNecessary(mReasonerOutputDirStringList,outputDirectory+"AnalysingUpdateCheckingData.dat",testCountCut)) {
LOG(INFO,getLogDomain(),logTr("Collecting statistic names for '%1' to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
QSet staticticNamesSet;
CReasonerEvaluationStatisticsNameCollectingExtractor* statNameCollectingExtractor = new CReasonerEvaluationStatisticsNameCollectingExtractor(&staticticNamesSet,dataValueCacher);
CReasonerEvaluationAvaragerSummarizer* avarageSummarize = new CReasonerEvaluationAvaragerSummarizer();
CReasonerEvaluationCollector* statNameCollector = new CReasonerEvaluationCollector(statNameCollectingExtractor,avarageSummarize,mFiltering);
CReasonerEvaluationDataValueGroupCollectionReasonerComparison* statNameReasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
for (QStringList::const_iterator it1 = mReasonerNameStringList.constBegin(), it2 = mReasonerOutputDirStringList.constBegin(), it1End = mReasonerNameStringList.constEnd(), it2End = mReasonerOutputDirStringList.constEnd(); it1 != it1End && it2 != it2End; ++it1, ++it2) {
QString reasonerName(*it1);
QString reasonerPath(*it2);
LOG(INFO,getLogDomain(),logTr("Extracting statistic names for '%1' reasoner from responses in directory '%2'.").arg(reasonerName).arg(reasonerPath),this);
statNameCollector->collectReasonerEvaluationDataValues(statNameReasonerComp,reasonerName,reasonerPath);
}
LOG(INFO,getLogDomain(),logTr("Analysing statistic values for '%1' to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
CReasonerEvaluationGroupRequestReasonerStatisticValueCollectingCSVComparisonAnalyser* analyser4 = new CReasonerEvaluationGroupRequestReasonerStatisticValueCollectingCSVComparisonAnalyser();
CReasonerEvaluationGroupRequestSelector* selectors = getSelectors(requestDirectory);
for (QSet::const_iterator it = staticticNamesSet.constBegin(), itEnd = staticticNamesSet.constEnd(); it != itEnd; ++it) {
QString statName(*it);
analyser4->addNextStatisticName(statName);
CReasonerEvaluationStatisticsValueExtractor* statValueExtractor = new CReasonerEvaluationStatisticsValueExtractor(statName,&staticticNamesSet,timeoutCut,errorPunishmentTime,dataValueCacher);
CReasonerEvaluationCollector* statValueCollector = new CReasonerEvaluationCollector(statValueExtractor,avarageSummarize,mFiltering);
CReasonerEvaluationDataValueGroupCollectionReasonerComparison* statValueReasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
for (QStringList::const_iterator it1 = mReasonerNameStringList.constBegin(), it2 = mReasonerOutputDirStringList.constBegin(), it1End = mReasonerNameStringList.constEnd(), it2End = mReasonerOutputDirStringList.constEnd(); it1 != it1End && it2 != it2End; ++it1, ++it2) {
QString reasonerName(*it1);
QString reasonerPath(*it2);
LOG(INFO,getLogDomain(),logTr("Extracting statistic values for '%1' reasoner from responses in directory '%2'.").arg(reasonerName).arg(reasonerPath),this);
statValueCollector->collectReasonerEvaluationDataValues(statValueReasonerComp,reasonerName,reasonerPath);
}
LOG(INFO,getLogDomain(),logTr("Comparing statistic values for '%1'.").arg(mReasonerNameStringList.join(", ")),this);
CReasonerEvaluationAnalyseContext* reasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig,QString("%1 Statistics").arg(statName.toUpper()),statName+"-");
CReasonerEvaluationGroupRequestReasonerCSVComparisonAnalyser* analyser1 = new CReasonerEvaluationGroupRequestReasonerCSVComparisonAnalyser();
analyser1->analyseEvaluationData(statValueReasonerComp,reasonerEvaluationContext,outputDirectory,selectors);
CReasonerEvaluationGroupRequestReasonerStatisticValueCSVComparisonAnalyser* analyser2 = new CReasonerEvaluationGroupRequestReasonerStatisticValueCSVComparisonAnalyser();
analyser2->analyseEvaluationData(statValueReasonerComp,reasonerEvaluationContext,outputDirectory,selectors);
CReasonerEvaluationGroupRequestReasonerAccumulatedCountCSVComparisonAnalyser* analyser3 = new CReasonerEvaluationGroupRequestReasonerAccumulatedCountCSVComparisonAnalyser();
analyser3->analyseEvaluationData(statValueReasonerComp,reasonerEvaluationContext,outputDirectory,selectors);
analyser4->analyseEvaluationData(statValueReasonerComp,reasonerEvaluationContext,outputDirectory,selectors);
LOG(INFO,getLogDomain(),logTr("Statistics values for '%1' compared to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
analysingUpdateChecker.saveAnalysingUpdateCheckFile(outputDirectory+"AnalysingUpdateCheckingData.dat");
analysed = true;
}
} else {
LOG(WARNING,getLogDomain(),logTr("Statistic values for '%1' in '%2' already up to date.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
}
}
if (ananysingString == "ReasonerEvaluationResultDifferencesComparer") {
QString outputDirectory = getAnalyserOutputDirectory(analysingDirectoryString,evaluationProgramName,plattform,"ResultDifferenceComparison");
CReasonerEvaluationAnalyserChecker analysingUpdateChecker;
if (analysingUpdateChecker.checkAnalysingUpdateNecessary(mReasonerOutputDirStringList,outputDirectory+"AnalysingUpdateCheckingData.dat",testCountCut)) {
LOG(INFO,getLogDomain(),logTr("Analysing result differences for '%1' to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
CReasonerEvaluationSpecifiedTimeExtractor* respFileExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::RESPONSEFILEEXTRACTOR,timeoutCut,errorPunishmentTime,dataValueCacher);
CReasonerEvaluationResponseFileListSummarizer* respFileListSummarizer = new CReasonerEvaluationResponseFileListSummarizer();
CReasonerEvaluationAnalyseContext* reasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig);
CReasonerEvaluationCollector* collector = new CReasonerEvaluationCollector(respFileExtractor,respFileListSummarizer,mFiltering);
CReasonerEvaluationDataValueGroupCollectionReasonerComparison* reasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
for (QStringList::const_iterator it1 = mReasonerNameStringList.constBegin(), it2 = mReasonerOutputDirStringList.constBegin(), it1End = mReasonerNameStringList.constEnd(), it2End = mReasonerOutputDirStringList.constEnd(); it1 != it1End && it2 != it2End; ++it1, ++it2) {
QString reasonerName(*it1);
QString reasonerPath(*it2);
LOG(INFO,getLogDomain(),logTr("Collecting response files for '%1' reasoner in directory '%2'.").arg(reasonerName).arg(reasonerPath),this);
collector->collectReasonerEvaluationDataValues(reasonerComp,reasonerName,reasonerPath);
}
LOG(INFO,getLogDomain(),logTr("Comparing result differences for '%1'.").arg(mReasonerNameStringList.join(", ")),this);
CReasonerEvaluationGroupRequestSelector* selectors = getSelectors(requestDirectory);
CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser* analyser1 = new CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser();
analyser1->analyseEvaluationData(reasonerComp,reasonerEvaluationContext,outputDirectory,selectors);
LOG(INFO,getLogDomain(),logTr("Result differences for '%1' compared to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
analysingUpdateChecker.saveAnalysingUpdateCheckFile(outputDirectory+"AnalysingUpdateCheckingData.dat");
analysed = true;
} else {
LOG(WARNING,getLogDomain(),logTr("Result differences for '%1' in '%2' already up to date.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
}
}
if (ananysingString == "ReasonerEvaluationResultSubsumptionsComparer") {
QString outputDirectory = getAnalyserOutputDirectory(analysingDirectoryString, evaluationProgramName, plattform, "ResultSubsumptionComparison");
CReasonerEvaluationAnalyserChecker analysingUpdateChecker;
if (analysingUpdateChecker.checkAnalysingUpdateNecessary(mReasonerOutputDirStringList, outputDirectory + "AnalysingUpdateCheckingData.dat", testCountCut)) {
LOG(INFO, getLogDomain(), logTr("Analysing result subsumptions for '%1' to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory), this);
CReasonerEvaluationSpecifiedTimeExtractor* respFileExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::RESPONSEFILEEXTRACTOR, timeoutCut, errorPunishmentTime, dataValueCacher);
CReasonerEvaluationResponseFileListSummarizer* respFileListSummarizer = new CReasonerEvaluationResponseFileListSummarizer();
CReasonerEvaluationAnalyseContext* reasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig);
CReasonerEvaluationCollector* collector = new CReasonerEvaluationCollector(respFileExtractor, respFileListSummarizer, mFiltering);
CReasonerEvaluationDataValueGroupCollectionReasonerComparison* reasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
for (QStringList::const_iterator it1 = mReasonerNameStringList.constBegin(), it2 = mReasonerOutputDirStringList.constBegin(), it1End = mReasonerNameStringList.constEnd(), it2End = mReasonerOutputDirStringList.constEnd(); it1 != it1End && it2 != it2End; ++it1, ++it2) {
QString reasonerName(*it1);
QString reasonerPath(*it2);
LOG(INFO, getLogDomain(), logTr("Collecting response files for '%1' reasoner in directory '%2'.").arg(reasonerName).arg(reasonerPath), this);
collector->collectReasonerEvaluationDataValues(reasonerComp, reasonerName, reasonerPath);
}
LOG(INFO, getLogDomain(), logTr("Comparing result subsumptions for '%1'.").arg(mReasonerNameStringList.join(", ")), this);
CReasonerEvaluationGroupRequestSelector* selectors = getSelectors(requestDirectory);
CReasonerEvaluationGroupRequestReasonerResultSubsumptionAnalyser* analyser1 = new CReasonerEvaluationGroupRequestReasonerResultSubsumptionAnalyser();
analyser1->analyseEvaluationData(reasonerComp, reasonerEvaluationContext, outputDirectory, selectors);
LOG(INFO, getLogDomain(), logTr("Result subsumptions for '%1' compared to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory), this);
analysingUpdateChecker.saveAnalysingUpdateCheckFile(outputDirectory + "AnalysingUpdateCheckingData.dat");
analysed = true;
} else {
LOG(WARNING, getLogDomain(), logTr("Result subsumptions for '%1' in '%2' already up to date.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory), this);
}
}
if (ananysingString == "ReasonerEvaluationResultCountingComparer") {
QString outputDirectory = getAnalyserOutputDirectory(analysingDirectoryString, evaluationProgramName, plattform, "ResultCountingComparison");
CReasonerEvaluationAnalyserChecker analysingUpdateChecker;
if (analysingUpdateChecker.checkAnalysingUpdateNecessary(mReasonerOutputDirStringList, outputDirectory + "AnalysingUpdateCheckingData.dat", testCountCut)) {
LOG(INFO, getLogDomain(), logTr("Analysing result counting for '%1' to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory), this);
CReasonerEvaluationSpecifiedTimeExtractor* respFileExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::RESPONSECOUNTEXTRACTOR, timeoutCut, errorPunishmentTime, dataValueCacher);
CReasonerEvaluationAvaragerSummarizer* avarageSummarize = new CReasonerEvaluationAvaragerSummarizer();
CReasonerEvaluationAnalyseContext* reasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig);
CReasonerEvaluationCollector* collector = new CReasonerEvaluationCollector(respFileExtractor, avarageSummarize, mFiltering);
CReasonerEvaluationDataValueGroupCollectionReasonerComparison* reasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
for (QStringList::const_iterator it1 = mReasonerNameStringList.constBegin(), it2 = mReasonerOutputDirStringList.constBegin(), it1End = mReasonerNameStringList.constEnd(), it2End = mReasonerOutputDirStringList.constEnd(); it1 != it1End && it2 != it2End; ++it1, ++it2) {
QString reasonerName(*it1);
QString reasonerPath(*it2);
LOG(INFO, getLogDomain(), logTr("Collecting response files for '%1' reasoner in directory '%2'.").arg(reasonerName).arg(reasonerPath), this);
collector->collectReasonerEvaluationDataValues(reasonerComp, reasonerName, reasonerPath);
}
LOG(INFO, getLogDomain(), logTr("Comparing result counting for '%1'.").arg(mReasonerNameStringList.join(", ")), this);
CReasonerEvaluationGroupRequestSelector* selectors = getSelectors(requestDirectory);
CReasonerEvaluationGroupRequestReasonerResultCountingAnalyser* analyser1 = new CReasonerEvaluationGroupRequestReasonerResultCountingAnalyser();
analyser1->analyseEvaluationData(reasonerComp, reasonerEvaluationContext, outputDirectory, selectors);
LOG(INFO, getLogDomain(), logTr("Result counting for '%1' compared to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory), this);
analysingUpdateChecker.saveAnalysingUpdateCheckFile(outputDirectory + "AnalysingUpdateCheckingData.dat");
analysed = true;
} else {
LOG(WARNING, getLogDomain(), logTr("Result counting for '%1' in '%2' already up to date.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory), this);
}
}
if (ananysingString == "ReasonerEvaluationErrorTimeoutComparer") {
QString outputDirectory = getAnalyserOutputDirectory(analysingDirectoryString,evaluationProgramName,plattform,"ErrorTimeoutComparison");
CReasonerEvaluationAnalyserChecker analysingUpdateChecker;
if (analysingUpdateChecker.checkAnalysingUpdateNecessary(mReasonerOutputDirStringList,outputDirectory+"AnalysingUpdateCheckingData.dat",testCountCut)) {
LOG(INFO,getLogDomain(),logTr("Analysing errors and timeouts for '%1' to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
CReasonerEvaluationSpecifiedTimeExtractor* errorExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::TIMEOUTERROREXTRACTOR,timeoutCut,errorPunishmentTime,dataValueCacher);
CReasonerEvaluationAvaragerSummarizer* avarageSummarize = new CReasonerEvaluationAvaragerSummarizer();
CReasonerEvaluationAnalyseContext* reasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig);
CReasonerEvaluationCollector* collector = new CReasonerEvaluationCollector(errorExtractor,avarageSummarize,mFiltering);
CReasonerEvaluationDataValueGroupCollectionReasonerComparison* reasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
for (QStringList::const_iterator it1 = mReasonerNameStringList.constBegin(), it2 = mReasonerOutputDirStringList.constBegin(), it1End = mReasonerNameStringList.constEnd(), it2End = mReasonerOutputDirStringList.constEnd(); it1 != it1End && it2 != it2End; ++it1, ++it2) {
QString reasonerName(*it1);
QString reasonerPath(*it2);
LOG(INFO,getLogDomain(),logTr("Collecting errors/timeouts for '%1' reasoner in directory '%2'.").arg(reasonerName).arg(reasonerPath),this);
collector->collectReasonerEvaluationDataValues(reasonerComp,reasonerName,reasonerPath);
}
LOG(INFO,getLogDomain(),logTr("Comparing errors/timeouts for '%1'.").arg(mReasonerNameStringList.join(", ")),this);
CReasonerEvaluationGroupRequestSelector* selectors = getSelectors(requestDirectory);
CReasonerEvaluationGroupRequestReasonerErrorAnalyser* analyser1 = new CReasonerEvaluationGroupRequestReasonerErrorAnalyser();
analyser1->analyseEvaluationData(reasonerComp,reasonerEvaluationContext,outputDirectory,selectors);
LOG(INFO,getLogDomain(),logTr("Errors/timeouts for '%1' compared to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
analysingUpdateChecker.saveAnalysingUpdateCheckFile(outputDirectory+"AnalysingUpdateCheckingData.dat");
analysed = true;
} else {
LOG(WARNING,getLogDomain(),logTr("Errors/timeouts for '%1' in '%2' already up to date.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
}
}
if (ananysingString == "ReasonerEvaluationErrorComparer") {
QString outputDirectory = getAnalyserOutputDirectory(analysingDirectoryString, evaluationProgramName, plattform, "ErrorComparison");
CReasonerEvaluationAnalyserChecker analysingUpdateChecker;
if (analysingUpdateChecker.checkAnalysingUpdateNecessary(mReasonerOutputDirStringList, outputDirectory + "AnalysingUpdateCheckingData.dat", testCountCut)) {
LOG(INFO, getLogDomain(), logTr("Analysing errors for '%1' to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory), this);
CReasonerEvaluationSpecifiedTimeExtractor* errorExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::ERROREXTRACTOR, timeoutCut, errorPunishmentTime, dataValueCacher);
CReasonerEvaluationAvaragerSummarizer* avarageSummarize = new CReasonerEvaluationAvaragerSummarizer();
CReasonerEvaluationAnalyseContext* reasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig);
CReasonerEvaluationCollector* collector = new CReasonerEvaluationCollector(errorExtractor, avarageSummarize, mFiltering);
CReasonerEvaluationDataValueGroupCollectionReasonerComparison* reasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
for (QStringList::const_iterator it1 = mReasonerNameStringList.constBegin(), it2 = mReasonerOutputDirStringList.constBegin(), it1End = mReasonerNameStringList.constEnd(), it2End = mReasonerOutputDirStringList.constEnd(); it1 != it1End && it2 != it2End; ++it1, ++it2) {
QString reasonerName(*it1);
QString reasonerPath(*it2);
LOG(INFO, getLogDomain(), logTr("Collecting errors for '%1' reasoner in directory '%2'.").arg(reasonerName).arg(reasonerPath), this);
collector->collectReasonerEvaluationDataValues(reasonerComp, reasonerName, reasonerPath);
}
LOG(INFO, getLogDomain(), logTr("Comparing errors for '%1'.").arg(mReasonerNameStringList.join(", ")), this);
CReasonerEvaluationGroupRequestSelector* selectors = getSelectors(requestDirectory);
CReasonerEvaluationGroupRequestReasonerErrorAnalyser* analyser1 = new CReasonerEvaluationGroupRequestReasonerErrorAnalyser();
analyser1->analyseEvaluationData(reasonerComp, reasonerEvaluationContext, outputDirectory, selectors);
LOG(INFO, getLogDomain(), logTr("Errors for '%1' compared to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory), this);
analysingUpdateChecker.saveAnalysingUpdateCheckFile(outputDirectory + "AnalysingUpdateCheckingData.dat");
analysed = true;
} else {
LOG(WARNING, getLogDomain(), logTr("Errors for '%1' in '%2' already up to date.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory), this);
}
}
if (ananysingString == "ReasonerEvaluationMemoryUsageComparer") {
QString outputDirectory = getAnalyserOutputDirectory(analysingDirectoryString, evaluationProgramName, plattform, "MemoryUsageComparison");
CReasonerEvaluationAnalyserChecker analysingUpdateChecker;
if (analysingUpdateChecker.checkAnalysingUpdateNecessary(mReasonerOutputDirStringList, outputDirectory + "AnalysingUpdateCheckingData.dat", testCountCut)) {
LOG(INFO, getLogDomain(), logTr("Analysing memory usage for '%1' to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory), this);
CReasonerEvaluationSpecifiedTimeExtractor* memoryUsageaExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::MEMORYUSAGEEXTRACTOR, timeoutCut, errorPunishmentTime, dataValueCacher);
CReasonerEvaluationAvaragerSummarizer* avarageSummarize = new CReasonerEvaluationAvaragerSummarizer();
CReasonerEvaluationAnalyseContext* reasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig);
CReasonerEvaluationCollector* collector = new CReasonerEvaluationCollector(memoryUsageaExtractor, avarageSummarize, mFiltering);
CReasonerEvaluationDataValueGroupCollectionReasonerComparison* reasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
for (QStringList::const_iterator it1 = mReasonerNameStringList.constBegin(), it2 = mReasonerOutputDirStringList.constBegin(), it1End = mReasonerNameStringList.constEnd(), it2End = mReasonerOutputDirStringList.constEnd(); it1 != it1End && it2 != it2End; ++it1, ++it2) {
QString reasonerName(*it1);
QString reasonerPath(*it2);
LOG(INFO, getLogDomain(), logTr("Collecting memory usage for '%1' reasoner in directory '%2'.").arg(reasonerName).arg(reasonerPath), this);
collector->collectReasonerEvaluationDataValues(reasonerComp, reasonerName, reasonerPath);
}
LOG(INFO, getLogDomain(), logTr("Comparing memory usage for '%1'.").arg(mReasonerNameStringList.join(", ")), this);
CReasonerEvaluationGroupRequestSelector* selectors = getSelectors(requestDirectory);
CReasonerEvaluationGroupRequestReasonerMemoryUsageAnalyser* analyser1 = new CReasonerEvaluationGroupRequestReasonerMemoryUsageAnalyser();
analyser1->analyseEvaluationData(reasonerComp, reasonerEvaluationContext, outputDirectory, selectors);
LOG(INFO, getLogDomain(), logTr("Memory usage for '%1' compared to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory), this);
analysingUpdateChecker.saveAnalysingUpdateCheckFile(outputDirectory + "AnalysingUpdateCheckingData.dat");
analysed = true;
} else {
LOG(WARNING, getLogDomain(), logTr("Memory usage for '%1' in '%2' already up to date.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory), this);
}
}
if (ananysingString == "RequestStatisticsComparer") {
QString outputDirectory = getAnalyserOutputDirectory(analysingDirectoryString,evaluationProgramName,plattform,"RequestStatisticsComparison");
CReasonerEvaluationAnalyserChecker analysingUpdateChecker;
if (analysingUpdateChecker.checkAnalysingUpdateNecessary(mReasonerOutputDirStringList,outputDirectory+"AnalysingUpdateCheckingData.dat",testCountCut)) {
LOG(INFO,getLogDomain(),logTr("Analysing request files of '%1'.").arg(outputDirectory),this);
CReasonerEvaluationSpecifiedTimeExtractor* requestFileExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::REQUESTFILEEXTRACTOR,timeoutCut,errorPunishmentTime,dataValueCacher);
CReasonerEvaluationResponseFileListSummarizer* respFileListSummarizer = new CReasonerEvaluationResponseFileListSummarizer();
CReasonerEvaluationAnalyseContext* reasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig);
CReasonerEvaluationCollector* collector = new CReasonerEvaluationCollector(requestFileExtractor,respFileListSummarizer,mFiltering);
CReasonerEvaluationDataValueGroupCollectionReasonerComparison* reasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
for (QStringList::const_iterator it1 = mReasonerNameStringList.constBegin(), it2 = mReasonerOutputDirStringList.constBegin(), it1End = mReasonerNameStringList.constEnd(), it2End = mReasonerOutputDirStringList.constEnd(); it1 != it1End && it2 != it2End; ++it1, ++it2) {
QString reasonerName(*it1);
QString reasonerPath(*it2);
LOG(INFO,getLogDomain(),logTr("Collecting request files for '%1' reasoner in directory '%2'.").arg(reasonerName).arg(reasonerPath),this);
collector->collectReasonerEvaluationDataValues(reasonerComp,reasonerName,reasonerPath);
}
CReasonerEvaluationGroupRequestSelector* selectors = getSelectors(requestDirectory);
CReasonerEvaluationGroupRequestStatisticsCSVComparisonAnalyser* analyser1 = new CReasonerEvaluationGroupRequestStatisticsCSVComparisonAnalyser(mFiltering);
analyser1->analyseEvaluationData(reasonerComp,reasonerEvaluationContext,outputDirectory,selectors);
LOG(INFO,getLogDomain(),logTr("Request statistics evaluated to '%1'.").arg(outputDirectory),this);
analysingUpdateChecker.saveAnalysingUpdateCheckFile(outputDirectory+"AnalysingUpdateCheckingData.dat");
analysed = true;
} else {
LOG(WARNING,getLogDomain(),logTr("Request statistics evaluation of '%1' already up to date.").arg(outputDirectory),this);
}
}
cint64 elapsedTime = timer.elapsed();
LOG(INFO,getLogDomain(),logTr("Finished analysing for '%1' in '%2' seconds.").arg(ananysingString).arg(elapsedTime/1000),this);
}
cint64 totalElapsedTime = totalTimer.elapsed();
LOG(INFO,getLogDomain(),logTr("Finished analysing in '%1' seconds.").arg(totalElapsedTime/1000),this);
} else {
LOG(ERROR,getLogDomain(),logTr("Analysing not possible, platform not specified."),this);
}
if (analysed) {
LOG(INFO,getLogDomain(),logTr("Creation HTML navigation overview site."),this);
CReasonerEvaluationHTMLNavigationOverviewSummarizer* htmlNavOvSumm = new CReasonerEvaluationHTMLNavigationOverviewSummarizer();
QString outputDirectory = getAnalyserOutputDirectory(analysingDirectoryString,evaluationProgramName,plattform,".");
htmlNavOvSumm->createHTMLNavigationOverview(outputDirectory,evaluationProgramName,mReasonerNameStringList,mFitlerStringList,mTestTimeout,mTestCount);
delete htmlNavOvSumm;
}
}
if (!mFailedReasonerStringList.isEmpty()) {
LOG(WARNING,getLogDomain(),logTr("Evaluation failed for the following reasoner configuration files:\r\n%1.").arg(mFailedReasonerStringList.join(",\r\n")),this);
}
return this;
}
}; // end namespace Evaluation
}; // end namespace Test
}; // end namespace Konclude