/*
* Copyright (C) 2013-2015, 2019 by the Konclude Developer Team.
*
* This file is part of the reasoning system Konclude.
* For details and support, see .
*
* Konclude is free software: you can redistribute it and/or modify
* it under the terms of version 3 of the GNU Lesser General Public
* License (LGPLv3) as published by the Free Software Foundation.
*
* Konclude is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU (Lesser) General Public License for more details.
*
* You should have received a copy of the GNU (Lesser) General Public
* License along with Konclude. If not, see .
*
*/
#include "CReasonerEvaluationExecutorThread.h"
#include "CReasonerEvaluationConfigurationLoader.h"
namespace Konclude {
namespace Test {
namespace Evaluation {
CReasonerEvaluationExecutorThread::CReasonerEvaluationExecutorThread(CConfiguration *config, CWatchDog *watchDog)
: CThread("Reasoner-Evaluation-Executor-Thread",watchDog), CLogIdentifier("::Konclude::Test::ReasonerEvaluationExecutor",this) {
mLoaderConfig = config;
mCommanderManager = CConfigManagerReader::readCommanderManagerConfig(mLoaderConfig);
mReasonerEvalFactory = new CConfigDependendReasonerEvaluationFactory();
mReasonerEvaluatorOWLlink = new CReasonerEvaluationEvaluatorOWLlink();
mReasonerEvaluatorSPARQL = new CReasonerEvaluationEvaluatorSPARQL();
mReasonerOWLlinkClient = new CReasonerEvaluationRequestClientOWLlinkThread();
mReasonerSPARQLClient = new CReasonerEvaluationRequestClientSPARQLThread();
mReasonerProvider = nullptr;
mNumberStringSize = 4;
mNextTestRemainingExecutionCount = 0;
mNextTestExecutionNumber = 0;
mTestCount = 0;
mErrorCommBreakCount = 2;
mFirstTest = false;
mCriticalProcessesTester = nullptr;
mConfTestcaseEvaluationExceptionLimit = -1;
mCurrentErrorTestRetryCount = 0;
mErrorTestMaximalRetryCount = -1;
mErrorTestMaximalRetryWriting = false;
startThread();
}
CReasonerEvaluationExecutorThread::~CReasonerEvaluationExecutorThread() {
delete mReasonerEvalFactory;
delete mReasonerEvaluatorOWLlink;
delete mReasonerEvaluatorSPARQL;
delete mReasonerProvider;
delete mReasonerOWLlinkClient;
delete mReasonerSPARQLClient;
delete mCriticalProcessesTester;
}
bool CReasonerEvaluationExecutorThread::executeEvaluation(CReasonerEvaluationTestsuite* evaluationTestsuite, CCallbackData* callbackData) {
QCoreApplication::postEvent(this,new CReasonerEvaluationTestsuiteEvent(evaluationTestsuite,callbackData));
return true;
}
bool CReasonerEvaluationExecutorThread::processCustomsEvents(QEvent::Type type, CCustomEvent *event) {
if (CThread::processCustomsEvents(type,event)) {
return true;
} else if (type == CReasonerEvaluationTestsuiteEvent::EVENTTYPE) {
CReasonerEvaluationTestsuiteEvent* rete = (CReasonerEvaluationTestsuiteEvent*)event;
mEvalTestsuite = rete->getEvaluationTestsuite();
mExecutedCallback = rete->getCallbackData();
mPlatformName = mEvalTestsuite->getPlatformString();
mExecutedTest = false;
mFiltering = mEvalTestsuite->getEvaluationFiltering();
bool configsSuccessfullyLoaded = loadConfigurations();
if (!configsSuccessfullyLoaded) {
failedEvaluationTests();
} else {
if (mCommanderManager) {
CGetSettingsCommand* getConfigCommand = new CGetSettingsCommand();
getConfigCommand->addProcessedCallback(new CCommandProcessedCallbackEvent(this,getConfigCommand));
mCommanderManager->realizeCommand(getConfigCommand);
}
}
return true;
} else if (type == CCommandProcessedCallbackEvent::EVENTTYPE) {
CCommandProcessedCallbackEvent *commProCallbackEvent = (CCommandProcessedCallbackEvent*)event;
CGetSettingsCommand* getConfigCommand = (CGetSettingsCommand*)commProCallbackEvent->getCommand();
CConfigurationBase* configurationBase = getConfigCommand->getConfiguration();
mConfiguration = configurationBase;
if (!mConfiguration) {
mConfiguration = mLoaderConfig;
}
initEvaluationTests();
if (prepareNextEvaluationTest()) {
executeNextEvaluationTest(mReasonerInitFileString,mNextTestInputFile);
} else {
finishEvaluationTests();
}
} else if (type == CReasonerEvaluationNextEvent::EVENTTYPE) {
CReasonerEvaluationNextEvent* nextEvalEvent = (CReasonerEvaluationNextEvent*)event;
LOG(INFO, getLogDomain(), logTr("Collecting responses."), this);
CReasonerEvaluationRequestResult* requestResults = mCurrentReasonerClient->getReasonerEvaluationResult();
LOG(INFO, getLogDomain(), logTr("Destroying reasoner."), this);
CReasonerEvaluationTerminationResult* terminationResults = mReasonerProvider->destroyReasoner();
QString outputFileString = getNextOutputFileString();
if (mFirstTest) {
mFirstTest = false;
executeNextEvaluationTest(mReasonerInitFileString,mNextTestInputFile);
if (mWaitingTimeBetweenTests > 0) {
LOG(INFO,getLogDomain(),logTr("Waiting %1 ms before continuation.").arg(mWaitingTimeBetweenTests),this);
QThread::msleep(mWaitingTimeBetweenTests);
}
delete terminationResults;
delete requestResults;
} else {
bool succ = evaluateExecutedTest(terminationResults,requestResults,outputFileString, false);
if (mWaitingTimeBetweenTests > 0) {
LOG(INFO,getLogDomain(),logTr("Waiting %1 ms before continuation.").arg(mWaitingTimeBetweenTests),this);
QThread::msleep(mWaitingTimeBetweenTests);
}
delete terminationResults;
delete requestResults;
if (mConfTestcaseEvaluationExceptionLimit == 0) {
LOG(WARN, getLogDomain(), logTr("Reached testcase evaluation limit, terminating with exception."), this);
QCoreApplication::exit(-1);
//TODO: fix, find a better solution
int* x = 0;
*x = 1;
} else {
if (mConfTestcaseEvaluationExceptionLimit != -1) {
mConfTestcaseEvaluationExceptionLimit--;
}
if (succ) {
mCurrentErrorTestRetryCount = 0;
if (--mNextTestRemainingExecutionCount > 0) {
++mNextTestExecutionNumber;
executeNextEvaluationTest(mReasonerInitFileString, mNextTestInputFile);
} else {
if (prepareNextEvaluationTest()) {
executeNextEvaluationTest(mReasonerInitFileString, mNextTestInputFile);
} else {
finishEvaluationTests();
}
}
} else {
if (--mErrorCommBreakCount > 0) {
if (mErrorTestMaximalRetryCount != -1 && mCurrentErrorTestRetryCount++ > mErrorTestMaximalRetryCount) {
LOG(ERROR, getLogDomain(), logTr("Reached maximum number of retries for test case '%1'.").arg(mNextTestInputFile), this);
if (mErrorTestMaximalRetryWriting) {
evaluateExecutedTest(terminationResults, requestResults, outputFileString, true);
}
mCurrentErrorTestRetryCount = 0;
if (prepareNextEvaluationTest()) {
executeNextEvaluationTest(mReasonerInitFileString, mNextTestInputFile);
} else {
failedEvaluationTests();
}
} else {
LOG(ERROR, getLogDomain(), logTr("Communication failure %1 with reasoner for test case '%2'.").arg(mCurrentErrorTestRetryCount).arg(mNextTestInputFile), this);
executeNextEvaluationTest(mReasonerInitFileString, mNextTestInputFile);
}
} else {
failedEvaluationTests();
}
}
}
}
}
return false;
}
bool CReasonerEvaluationExecutorThread::loadConfigurations() {
bool configsSuccessfullyLoaded = true;
QString globalConfString = getCorrectedPathString(mEvalTestsuite->getGlobalConfigString());
if (!globalConfString.isEmpty()) {
CReasonerEvaluationConfigurationLoader* globalConfLoader = new CReasonerEvaluationConfigurationLoader();
LOG(INFO,getLogDomain(),logTr("Loading global configuration '%1'.").arg(globalConfString),this);
configsSuccessfullyLoaded &= globalConfLoader->loadConfiguration(globalConfString,mLoaderConfig);
delete globalConfLoader;
} else {
configsSuccessfullyLoaded = false;
}
QString reasonerConfString = getCorrectedPathString(mEvalTestsuite->getReasonerConfigString());
if (!reasonerConfString.isEmpty()) {
CReasonerEvaluationConfigurationLoader* reasonerConfLoader = new CReasonerEvaluationConfigurationLoader();
LOG(INFO,getLogDomain(),logTr("Loading reasoner configuration '%1'.").arg(reasonerConfString),this);
configsSuccessfullyLoaded &= reasonerConfLoader->loadConfiguration(reasonerConfString,mLoaderConfig);
delete reasonerConfLoader;
} else {
configsSuccessfullyLoaded = false;
}
return configsSuccessfullyLoaded;
}
bool CReasonerEvaluationExecutorThread::initEvaluationTests() {
mInputDir = mEvalTestsuite->getTestsDirectoryString();
mOutputDir = mEvalTestsuite->getEvaluationDirectoryString();
if (!mInputDir.endsWith("/")) {
mInputDir += "/";
}
if (!mOutputDir.endsWith("/")) {
mOutputDir += "/";
}
mRemainingSubDirs.append(".");
mReasonerInitFileString = getCorrectedPathString(CConfigDataReader::readConfigString(mConfiguration,"Konclude.Evaluation.Reasoner.InitializationFile"));
mConfigName = CConfigDataReader::readConfigString(mConfiguration,"Konclude.Evaluation.Config.Name");
mReasonerName = CConfigDataReader::readConfigString(mConfiguration,"Konclude.Evaluation.Reasoner.Name");
mReasonerOutputDirectory = getCorrectedPathString(CConfigDataReader::readConfigString(mConfiguration,"Konclude.Evaluation.Reasoner.OutputDirectory"));
mTestCount = CConfigDataReader::readConfigInteger(mConfiguration,"Konclude.Evaluation.TestingCount");
mErrorCommBreakCount = CConfigDataReader::readConfigInteger(mConfiguration,"Konclude.Evaluation.CancelReasonerErrorCount");
mErrorTestMaximalRetryCount = CConfigDataReader::readConfigInteger(mConfiguration, "Konclude.Evaluation.ReasonerTestErrorMaximalRetryCount");
mErrorTestMaximalRetryWriting = CConfigDataReader::readConfigBoolean(mConfiguration, "Konclude.Evaluation.ReasonerTestErrorMaximalRetryWriting");
mConfTestcaseEvaluationExceptionLimit = CConfigDataReader::readConfigInteger(mConfiguration, "Konclude.Evaluation.TestcaseEvaluationExceptionLimit");
mTestTimeout = CConfigDataReader::readConfigInteger(mConfiguration,"Konclude.Evaluation.RequestTimeout");
mWaitingTimeBetweenTests = CConfigDataReader::readConfigInteger(mConfiguration,"Konclude.Evaluation.WaitingTimeBetweenTests");
mWaitingTimeAfterReasonerCreation = CConfigDataReader::readConfigInteger(mConfiguration,"Konclude.Evaluation.WaitingTimeAfterReasonerCreation");
mSystemReadyRetestTimeForTests = CConfigDataReader::readConfigInteger(mConfiguration,"Konclude.Evaluation.SystemReadyRetestTimeForTests");
mFirstTestWarmUp = CConfigDataReader::readConfigBoolean(mConfiguration,"Konclude.Evaluation.FirstTestsWarmingUp");
mAllTestsWarmUp = CConfigDataReader::readConfigBoolean(mConfiguration,"Konclude.Evaluation.AllTestsWarmingUp");
mCriticalProcessesTester = new CCriticalSystemProcessTester(mConfiguration);
if (mFirstTestWarmUp) {
mFirstTest = true;
}
LOG(INFO,getLogDomain(),logTr("Preparing reasoner '%1' evaluation with config file '%2' and test suite '%3'.").arg(mReasonerName).arg(mConfigName).arg(mInputDir),this);
LOG(INFO,getLogDomain(),logTr("Using init file '%1' and output directory '%2'.").arg(mReasonerInitFileString).arg(mReasonerOutputDirectory),this);
return true;
}
bool CReasonerEvaluationExecutorThread::finishEvaluationTests() {
LOG(INFO,getLogDomain(),logTr("Finished reasoner '%1' evaluation with config file '%2' and test suite '%3'.").arg(mReasonerName).arg(mConfigName).arg(mInputDir),this);
if (mExecutedCallback) {
mExecutedCallback->setCallbackDataContext(new CReasonerEvaluationEvaluatedCallbackContextData(mExecutedTest,mReasonerName,mOutputDir+mConfigName+"/"+mReasonerOutputDirectory,mTestTimeout,mTestCount));
mExecutedCallback->doCallback();
}
delete mEvalTestsuite;
return true;
}
bool CReasonerEvaluationExecutorThread::failedEvaluationTests() {
LOG(INFO,getLogDomain(),logTr("Failed evaluation for reasoner with config file '%2' and test suite '%3'.").arg(mEvalTestsuite->getReasonerConfigString()).arg(mEvalTestsuite->getTestsDirectoryString()),this);
if (mExecutedCallback) {
mExecutedCallback->setCallbackDataContext(new CReasonerEvaluationEvaluatedCallbackContextData(true));
mExecutedCallback->doCallback();
}
delete mEvalTestsuite;
return true;
}
bool CReasonerEvaluationExecutorThread::prepareNextEvaluationTest() {
while (mNextTestRemainingExecutionCount <= 0) {
QString testFile = nextEvaluationTestfile();
if (testFile.isEmpty()) {
return false;
} else {
if (mAllTestsWarmUp) {
mFirstTest = true;
}
QString reasonerString = mReasonerOutputDirectory;
QString configString = mConfigName;
QDir genPath(mOutputDir);
if (genPath.mkpath(configString+"/"+reasonerString+testFile)) {
QDir outputDir(mOutputDir+configString+"/"+reasonerString+testFile);
QStringList responseList = outputDir.entryList(QDir::Files);
mNextTestExecutionNumber = responseList.count();
mNextTestRemainingExecutionCount = mTestCount - mNextTestExecutionNumber;
if (mNextTestRemainingExecutionCount > 0) {
mNextTestInputFile = mInputDir+testFile;
mNextTestOutputDir = mOutputDir+configString+"/"+reasonerString+testFile+"/";
return true;
}
}
}
}
return false;
}
QString CReasonerEvaluationExecutorThread::getCorrectedPathString(const QString& string) {
QString corrString = string;
corrString = corrString.replace("$$_PLATFORM_$$",mPlatformName);
corrString = CReasonerEvaluationPathConverter::getCorrectedPathString(corrString);
return corrString;
}
bool CReasonerEvaluationExecutorThread::testExistFile(const QString& fileString) {
QFile file(fileString);
if (file.exists()) {
return true;
}
return false;
}
QString CReasonerEvaluationExecutorThread::getNextOutputFileString() {
QString outputFileString = QString("%2%1-response.xml").arg(getNumberString(mNextTestExecutionNumber)).arg(mNextTestOutputDir);
while (testExistFile(outputFileString)) {
++mNextTestExecutionNumber;
outputFileString = QString("%2%1-response.xml").arg(getNumberString(mNextTestExecutionNumber)).arg(mNextTestOutputDir);
}
return outputFileString;
}
QString CReasonerEvaluationExecutorThread::getNumberString(cint64 number) {
QString numberString = QString::number(number);
while (numberString.size() < mNumberStringSize) {
numberString = QString("0%1").arg(numberString);
}
return numberString;
}
CReasonerEvaluationEvaluator* CReasonerEvaluationExecutorThread::getReasonerEvaluatorForTestfile(const QString& testFileNameString, CReasonerEvaluationRequestClientBaseThread* reasonerClient) {
if (dynamic_cast(reasonerClient)) {
return mReasonerEvaluatorOWLlink;
} else if (dynamic_cast(reasonerClient)) {
return mReasonerEvaluatorSPARQL;
}
LOG(WARN, getLogDomain(), logTr("No supported evaluator for request file '%1'.").arg(mWaitingTimeAfterReasonerCreation), this);
return mReasonerEvaluatorOWLlink;
}
bool CReasonerEvaluationExecutorThread::evaluateExecutedTest(CReasonerEvaluationTerminationResult* terminationResults, CReasonerEvaluationRequestResult* requestResults, const QString& testcaseOutput, bool forceWriting) {
if (requestResults->hasSucessfullReasonerCommunication() || forceWriting) {
LOG(INFO,getLogDomain(),logTr("Evaluate executed test case to '%1'.").arg(testcaseOutput),this);
mCurrentReasonerEvaluator = getReasonerEvaluatorForTestfile(mNextTestInputFile, mCurrentReasonerClient);
mCurrentReasonerEvaluator->evaluateResults(requestResults,terminationResults,mNextTestInputFile,testcaseOutput,mConfiguration);
mExecutedTest = true;
return true;
} else {
LOG(ERROR,getLogDomain(),logTr("Communication with reasoner failed for test case '%1'.").arg(testcaseOutput),this);
}
return false;
}
bool CReasonerEvaluationExecutorThread::testSystemReadyForNextEvaluationTest() {
if (mCriticalProcessesTester->hasCriticalSystemProcesses()) {
LOG(WARNING,getLogDomain(),logTr("Critical processes are running on the test system."),this);
return false;
}
return true;
}
void CReasonerEvaluationExecutorThread::delayNextEvaluationTestUntilSystemReady() {
while (!testSystemReadyForNextEvaluationTest()) {
LOG(WARNING,getLogDomain(),logTr("Test system not ready, retest after %1 ms.").arg(mSystemReadyRetestTimeForTests),this);
QThread::msleep(mSystemReadyRetestTimeForTests);
}
}
bool CReasonerEvaluationExecutorThread::executeNextEvaluationTest(const QString& testcaseInit, const QString& testcaseInput) {
delayNextEvaluationTestUntilSystemReady();
if (!mReasonerProvider) {
mReasonerProvider = mReasonerEvalFactory->createReasonerProvider(mConfiguration);
}
mReasonerProvider->createReasoner(mConfiguration);
QString addressString = mReasonerProvider->getReasonerAddressString();
if (mFirstTest) {
LOG(INFO,getLogDomain(),logTr("Warming up for reasoner '%1' with test case '%2'.").arg(mReasonerName).arg(testcaseInput),this);
} else {
LOG(INFO,getLogDomain(),logTr("Testing reasoner '%1' with test case '%2'.").arg(mReasonerName).arg(testcaseInput),this);
}
if (mWaitingTimeAfterReasonerCreation > 0) {
LOG(INFO,getLogDomain(),logTr("Waiting %1 ms after created reasoner.").arg(mWaitingTimeAfterReasonerCreation),this);
QThread::msleep(mWaitingTimeAfterReasonerCreation);
}
mCurrentReasonerClient = getReasonerClientForTestfile(testcaseInput);
mCurrentReasonerClient->evaluateReasoner(testcaseInit,testcaseInput, mReasonerProvider,mConfiguration,new CReasonerEvaluationNextEvent(this,0));
LOG(INFO, getLogDomain(), logTr("Executing reasoner for test case '%1'.").arg(testcaseInput), this);
return true;
}
CReasonerEvaluationRequestClientBaseThread* CReasonerEvaluationExecutorThread::getReasonerClientForTestfile(const QString& testFileNameString) {
if (mReasonerOWLlinkClient->canHandleRequestFile(testFileNameString)) {
return mReasonerOWLlinkClient;
}
if (mReasonerSPARQLClient->canHandleRequestFile(testFileNameString)) {
return mReasonerSPARQLClient;
}
LOG(WARN, getLogDomain(), logTr("No supported client handler for request file '%1'.").arg(mWaitingTimeAfterReasonerCreation), this);
return mReasonerOWLlinkClient;
}
bool CReasonerEvaluationExecutorThread::testFileFiltered(const QString& testFileNameString, const QString& testFilePathString, const QString& subDirPathString) {
if (!mFiltering) {
return true;
} else {
QStringList statisticList = mFiltering->loadStatistics(testFileNameString,subDirPathString);
return mFiltering->testSatisfiesFilter(testFileNameString,testFilePathString,subDirPathString,statisticList);
}
}
QString CReasonerEvaluationExecutorThread::nextEvaluationTestfile() {
if (mRemainingFiles.isEmpty()) {
while (mRemainingFiles.isEmpty() && !mRemainingSubDirs.isEmpty()) {
QString nextDir = mRemainingSubDirs.takeFirst();
QDir dir(mInputDir+nextDir);
QStringList subDirList = dir.entryList(QDir::Dirs,QDir::Name|QDir::IgnoreCase);
foreach (QString subDir, subDirList) {
if (!subDir.startsWith("-") && subDir != "." && subDir != "..") {
mRemainingSubDirs.prepend(QString("%1").arg(nextDir+"/"+subDir));
}
}
QStringList fileList = dir.entryList(QDir::Files,QDir::Name|QDir::IgnoreCase);
foreach (QString fileString, fileList) {
if (!fileString.startsWith("-")) {
if (testFileFiltered(fileString,mInputDir,nextDir+"/")) {
mRemainingFiles.append(nextDir+"/"+fileString);
} else {
//LOG(INFO,getLogDomain(),logTr("File '%1' in directory '%2' filtered out.").arg(fileString).arg(nextDir),this);
}
}
}
}
}
QString nextTestfile;
if (!mRemainingFiles.isEmpty()) {
nextTestfile = mRemainingFiles.takeFirst();
}
return nextTestfile;
}
}; // end namespace Evaluation
}; // end namespace Test
}; // end namespace Konclude