Debian Codegraph @

Source/Test/Evaluation/CReasonerEvaluationExecutorThread.cpp

raw
1 /*
2 * Copyright (C) 2013-2015, 2019 by the Konclude Developer Team.
3 *
4 * This file is part of the reasoning system Konclude.
5 * For details and support, see <http://konclude.com/>.
6 *
7 * Konclude is free software: you can redistribute it and/or modify
8 * it under the terms of version 3 of the GNU Lesser General Public
9 * License (LGPLv3) as published by the Free Software Foundation.
10 *
11 * Konclude is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU (Lesser) General Public License for more details.
15 *
16 * You should have received a copy of the GNU (Lesser) General Public
17 * License along with Konclude. If not, see <http://www.gnu.org/licenses/>.
18 *
19 */
20
21 #include "CReasonerEvaluationExecutorThread.h"
22 #include "CReasonerEvaluationConfigurationLoader.h"
23
24
25 namespace Konclude {
26
27 namespace Test {
28
29 namespace Evaluation {
30
31
32
33 CReasonerEvaluationExecutorThread::CReasonerEvaluationExecutorThread(CConfiguration *config, CWatchDog *watchDog)
34 : CThread("Reasoner-Evaluation-Executor-Thread",watchDog), CLogIdentifier("::Konclude::Test::ReasonerEvaluationExecutor",this) {
35
36 mLoaderConfig = config;
37 mCommanderManager = CConfigManagerReader::readCommanderManagerConfig(mLoaderConfig);
38 mReasonerEvalFactory = new CConfigDependendReasonerEvaluationFactory();
39 mReasonerEvaluatorOWLlink = new CReasonerEvaluationEvaluatorOWLlink();
40 mReasonerEvaluatorSPARQL = new CReasonerEvaluationEvaluatorSPARQL();
41 mReasonerOWLlinkClient = new CReasonerEvaluationRequestClientOWLlinkThread();
42 mReasonerSPARQLClient = new CReasonerEvaluationRequestClientSPARQLThread();
43 mReasonerProvider = nullptr;
44 mNumberStringSize = 4;
45 mNextTestRemainingExecutionCount = 0;
46 mNextTestExecutionNumber = 0;
47 mTestCount = 0;
48 mErrorCommBreakCount = 2;
49 mFirstTest = false;
50 mCriticalProcessesTester = nullptr;
51 mConfTestcaseEvaluationExceptionLimit = -1;
52 mCurrentErrorTestRetryCount = 0;
53 mErrorTestMaximalRetryCount = -1;
54 mErrorTestMaximalRetryWriting = false;
55 startThread();
56 }
57
58
59 CReasonerEvaluationExecutorThread::~CReasonerEvaluationExecutorThread() {
60 delete mReasonerEvalFactory;
61 delete mReasonerEvaluatorOWLlink;
62 delete mReasonerEvaluatorSPARQL;
63 delete mReasonerProvider;
64 delete mReasonerOWLlinkClient;
65 delete mReasonerSPARQLClient;
66 delete mCriticalProcessesTester;
67 }
68
69 bool CReasonerEvaluationExecutorThread::executeEvaluation(CReasonerEvaluationTestsuite* evaluationTestsuite, CCallbackData* callbackData) {
70 QCoreApplication::postEvent(this,new CReasonerEvaluationTestsuiteEvent(evaluationTestsuite,callbackData));
71 return true;
72 }
73
74 bool CReasonerEvaluationExecutorThread::processCustomsEvents(QEvent::Type type, CCustomEvent *event) {
75 if (CThread::processCustomsEvents(type,event)) {
76 return true;
77 } else if (type == CReasonerEvaluationTestsuiteEvent::EVENTTYPE) {
78 CReasonerEvaluationTestsuiteEvent* rete = (CReasonerEvaluationTestsuiteEvent*)event;
79
80
81 mEvalTestsuite = rete->getEvaluationTestsuite();
82 mExecutedCallback = rete->getCallbackData();
83 mPlatformName = mEvalTestsuite->getPlatformString();
84 mExecutedTest = false;
85
86
87 mFiltering = mEvalTestsuite->getEvaluationFiltering();
88
89 bool configsSuccessfullyLoaded = loadConfigurations();
90 if (!configsSuccessfullyLoaded) {
91 failedEvaluationTests();
92 } else {
93 if (mCommanderManager) {
94 CGetSettingsCommand* getConfigCommand = new CGetSettingsCommand();
95 getConfigCommand->addProcessedCallback(new CCommandProcessedCallbackEvent(this,getConfigCommand));
96 mCommanderManager->realizeCommand(getConfigCommand);
97 }
98 }
99
100 return true;
101 } else if (type == CCommandProcessedCallbackEvent::EVENTTYPE) {
102 CCommandProcessedCallbackEvent *commProCallbackEvent = (CCommandProcessedCallbackEvent*)event;
103 CGetSettingsCommand* getConfigCommand = (CGetSettingsCommand*)commProCallbackEvent->getCommand();
104 CConfigurationBase* configurationBase = getConfigCommand->getConfiguration();
105 mConfiguration = configurationBase;
106 if (!mConfiguration) {
107 mConfiguration = mLoaderConfig;
108 }
109 initEvaluationTests();
110
111 if (prepareNextEvaluationTest()) {
112 executeNextEvaluationTest(mReasonerInitFileString,mNextTestInputFile);
113 } else {
114 finishEvaluationTests();
115 }
116 } else if (type == CReasonerEvaluationNextEvent::EVENTTYPE) {
117 CReasonerEvaluationNextEvent* nextEvalEvent = (CReasonerEvaluationNextEvent*)event;
118
119 LOG(INFO, getLogDomain(), logTr("Collecting responses."), this);
120 CReasonerEvaluationRequestResult* requestResults = mCurrentReasonerClient->getReasonerEvaluationResult();
121 LOG(INFO, getLogDomain(), logTr("Destroying reasoner."), this);
122 CReasonerEvaluationTerminationResult* terminationResults = mReasonerProvider->destroyReasoner();
123
124 QString outputFileString = getNextOutputFileString();
125
126 if (mFirstTest) {
127 mFirstTest = false;
128 executeNextEvaluationTest(mReasonerInitFileString,mNextTestInputFile);
129
130 if (mWaitingTimeBetweenTests > 0) {
131 LOG(INFO,getLogDomain(),logTr("Waiting %1 ms before continuation.").arg(mWaitingTimeBetweenTests),this);
132 QThread::msleep(mWaitingTimeBetweenTests);
133 }
134
135 delete terminationResults;
136 delete requestResults;
137
138 } else {
139 bool succ = evaluateExecutedTest(terminationResults,requestResults,outputFileString, false);
140
141 if (mWaitingTimeBetweenTests > 0) {
142 LOG(INFO,getLogDomain(),logTr("Waiting %1 ms before continuation.").arg(mWaitingTimeBetweenTests),this);
143 QThread::msleep(mWaitingTimeBetweenTests);
144 }
145
146 delete terminationResults;
147 delete requestResults;
148
149
150
151 if (mConfTestcaseEvaluationExceptionLimit == 0) {
152 LOG(WARN, getLogDomain(), logTr("Reached testcase evaluation limit, terminating with exception."), this);
153 QCoreApplication::exit(-1);
154
155 //TODO: fix, find a better solution
156 int* x = 0;
157 *x = 1;
158 } else {
159
160 if (mConfTestcaseEvaluationExceptionLimit != -1) {
161 mConfTestcaseEvaluationExceptionLimit--;
162 }
163
164 if (succ) {
165 mCurrentErrorTestRetryCount = 0;
166 if (--mNextTestRemainingExecutionCount > 0) {
167 ++mNextTestExecutionNumber;
168 executeNextEvaluationTest(mReasonerInitFileString, mNextTestInputFile);
169 } else {
170 if (prepareNextEvaluationTest()) {
171 executeNextEvaluationTest(mReasonerInitFileString, mNextTestInputFile);
172 } else {
173 finishEvaluationTests();
174 }
175 }
176 } else {
177 if (--mErrorCommBreakCount > 0) {
178 if (mErrorTestMaximalRetryCount != -1 && mCurrentErrorTestRetryCount++ > mErrorTestMaximalRetryCount) {
179
180 LOG(ERROR, getLogDomain(), logTr("Reached maximum number of retries for test case '%1'.").arg(mNextTestInputFile), this);
181 if (mErrorTestMaximalRetryWriting) {
182 evaluateExecutedTest(terminationResults, requestResults, outputFileString, true);
183 }
184
185 mCurrentErrorTestRetryCount = 0;
186 if (prepareNextEvaluationTest()) {
187 executeNextEvaluationTest(mReasonerInitFileString, mNextTestInputFile);
188 } else {
189 failedEvaluationTests();
190 }
191 } else {
192 LOG(ERROR, getLogDomain(), logTr("Communication failure %1 with reasoner for test case '%2'.").arg(mCurrentErrorTestRetryCount).arg(mNextTestInputFile), this);
193 executeNextEvaluationTest(mReasonerInitFileString, mNextTestInputFile);
194 }
195 } else {
196 failedEvaluationTests();
197 }
198 }
199 }
200 }
201 }
202
203 return false;
204 }
205
206 bool CReasonerEvaluationExecutorThread::loadConfigurations() {
207 bool configsSuccessfullyLoaded = true;
208 QString globalConfString = getCorrectedPathString(mEvalTestsuite->getGlobalConfigString());
209 if (!globalConfString.isEmpty()) {
210 CReasonerEvaluationConfigurationLoader* globalConfLoader = new CReasonerEvaluationConfigurationLoader();
211 LOG(INFO,getLogDomain(),logTr("Loading global configuration '%1'.").arg(globalConfString),this);
212 configsSuccessfullyLoaded &= globalConfLoader->loadConfiguration(globalConfString,mLoaderConfig);
213 delete globalConfLoader;
214 } else {
215 configsSuccessfullyLoaded = false;
216 }
217 QString reasonerConfString = getCorrectedPathString(mEvalTestsuite->getReasonerConfigString());
218 if (!reasonerConfString.isEmpty()) {
219 CReasonerEvaluationConfigurationLoader* reasonerConfLoader = new CReasonerEvaluationConfigurationLoader();
220 LOG(INFO,getLogDomain(),logTr("Loading reasoner configuration '%1'.").arg(reasonerConfString),this);
221 configsSuccessfullyLoaded &= reasonerConfLoader->loadConfiguration(reasonerConfString,mLoaderConfig);
222 delete reasonerConfLoader;
223 } else {
224 configsSuccessfullyLoaded = false;
225 }
226 return configsSuccessfullyLoaded;
227 }
228
229
230 bool CReasonerEvaluationExecutorThread::initEvaluationTests() {
231 mInputDir = mEvalTestsuite->getTestsDirectoryString();
232 mOutputDir = mEvalTestsuite->getEvaluationDirectoryString();
233
234 if (!mInputDir.endsWith("/")) {
235 mInputDir += "/";
236 }
237 if (!mOutputDir.endsWith("/")) {
238 mOutputDir += "/";
239 }
240
241 mRemainingSubDirs.append(".");
242
243 mReasonerInitFileString = getCorrectedPathString(CConfigDataReader::readConfigString(mConfiguration,"Konclude.Evaluation.Reasoner.InitializationFile"));
244 mConfigName = CConfigDataReader::readConfigString(mConfiguration,"Konclude.Evaluation.Config.Name");
245 mReasonerName = CConfigDataReader::readConfigString(mConfiguration,"Konclude.Evaluation.Reasoner.Name");
246 mReasonerOutputDirectory = getCorrectedPathString(CConfigDataReader::readConfigString(mConfiguration,"Konclude.Evaluation.Reasoner.OutputDirectory"));
247 mTestCount = CConfigDataReader::readConfigInteger(mConfiguration,"Konclude.Evaluation.TestingCount");
248
249 mErrorCommBreakCount = CConfigDataReader::readConfigInteger(mConfiguration,"Konclude.Evaluation.CancelReasonerErrorCount");
250 mErrorTestMaximalRetryCount = CConfigDataReader::readConfigInteger(mConfiguration, "Konclude.Evaluation.ReasonerTestErrorMaximalRetryCount");
251 mErrorTestMaximalRetryWriting = CConfigDataReader::readConfigBoolean(mConfiguration, "Konclude.Evaluation.ReasonerTestErrorMaximalRetryWriting");
252
253 mConfTestcaseEvaluationExceptionLimit = CConfigDataReader::readConfigInteger(mConfiguration, "Konclude.Evaluation.TestcaseEvaluationExceptionLimit");
254
255 mTestTimeout = CConfigDataReader::readConfigInteger(mConfiguration,"Konclude.Evaluation.RequestTimeout");
256
257 mWaitingTimeBetweenTests = CConfigDataReader::readConfigInteger(mConfiguration,"Konclude.Evaluation.WaitingTimeBetweenTests");
258 mWaitingTimeAfterReasonerCreation = CConfigDataReader::readConfigInteger(mConfiguration,"Konclude.Evaluation.WaitingTimeAfterReasonerCreation");
259 mSystemReadyRetestTimeForTests = CConfigDataReader::readConfigInteger(mConfiguration,"Konclude.Evaluation.SystemReadyRetestTimeForTests");
260
261 mFirstTestWarmUp = CConfigDataReader::readConfigBoolean(mConfiguration,"Konclude.Evaluation.FirstTestsWarmingUp");
262 mAllTestsWarmUp = CConfigDataReader::readConfigBoolean(mConfiguration,"Konclude.Evaluation.AllTestsWarmingUp");
263
264 mCriticalProcessesTester = new CCriticalSystemProcessTester(mConfiguration);
265
266 if (mFirstTestWarmUp) {
267 mFirstTest = true;
268 }
269
270 LOG(INFO,getLogDomain(),logTr("Preparing reasoner '%1' evaluation with config file '%2' and test suite '%3'.").arg(mReasonerName).arg(mConfigName).arg(mInputDir),this);
271 LOG(INFO,getLogDomain(),logTr("Using init file '%1' and output directory '%2'.").arg(mReasonerInitFileString).arg(mReasonerOutputDirectory),this);
272
273 return true;
274 }
275
276
277 bool CReasonerEvaluationExecutorThread::finishEvaluationTests() {
278 LOG(INFO,getLogDomain(),logTr("Finished reasoner '%1' evaluation with config file '%2' and test suite '%3'.").arg(mReasonerName).arg(mConfigName).arg(mInputDir),this);
279 if (mExecutedCallback) {
280 mExecutedCallback->setCallbackDataContext(new CReasonerEvaluationEvaluatedCallbackContextData(mExecutedTest,mReasonerName,mOutputDir+mConfigName+"/"+mReasonerOutputDirectory,mTestTimeout,mTestCount));
281 mExecutedCallback->doCallback();
282 }
283 delete mEvalTestsuite;
284 return true;
285 }
286
287
288
289 bool CReasonerEvaluationExecutorThread::failedEvaluationTests() {
290 LOG(INFO,getLogDomain(),logTr("Failed evaluation for reasoner with config file '%2' and test suite '%3'.").arg(mEvalTestsuite->getReasonerConfigString()).arg(mEvalTestsuite->getTestsDirectoryString()),this);
291 if (mExecutedCallback) {
292 mExecutedCallback->setCallbackDataContext(new CReasonerEvaluationEvaluatedCallbackContextData(true));
293 mExecutedCallback->doCallback();
294 }
295 delete mEvalTestsuite;
296 return true;
297 }
298
299
300
301
302 bool CReasonerEvaluationExecutorThread::prepareNextEvaluationTest() {
303 while (mNextTestRemainingExecutionCount <= 0) {
304 QString testFile = nextEvaluationTestfile();
305 if (testFile.isEmpty()) {
306 return false;
307 } else {
308 if (mAllTestsWarmUp) {
309 mFirstTest = true;
310 }
311 QString reasonerString = mReasonerOutputDirectory;
312 QString configString = mConfigName;
313 QDir genPath(mOutputDir);
314 if (genPath.mkpath(configString+"/"+reasonerString+testFile)) {
315 QDir outputDir(mOutputDir+configString+"/"+reasonerString+testFile);
316 QStringList responseList = outputDir.entryList(QDir::Files);
317 mNextTestExecutionNumber = responseList.count();
318 mNextTestRemainingExecutionCount = mTestCount - mNextTestExecutionNumber;
319 if (mNextTestRemainingExecutionCount > 0) {
320 mNextTestInputFile = mInputDir+testFile;
321 mNextTestOutputDir = mOutputDir+configString+"/"+reasonerString+testFile+"/";
322 return true;
323 }
324 }
325 }
326 }
327 return false;
328 }
329
330
331
332 QString CReasonerEvaluationExecutorThread::getCorrectedPathString(const QString& string) {
333 QString corrString = string;
334 corrString = corrString.replace("$$_PLATFORM_$$",mPlatformName);
335 corrString = CReasonerEvaluationPathConverter::getCorrectedPathString(corrString);
336 return corrString;
337 }
338
339
340 bool CReasonerEvaluationExecutorThread::testExistFile(const QString& fileString) {
341 QFile file(fileString);
342 if (file.exists()) {
343 return true;
344 }
345 return false;
346 }
347
348
349
350 QString CReasonerEvaluationExecutorThread::getNextOutputFileString() {
351 QString outputFileString = QString("%2%1-response.xml").arg(getNumberString(mNextTestExecutionNumber)).arg(mNextTestOutputDir);
352 while (testExistFile(outputFileString)) {
353 ++mNextTestExecutionNumber;
354 outputFileString = QString("%2%1-response.xml").arg(getNumberString(mNextTestExecutionNumber)).arg(mNextTestOutputDir);
355 }
356 return outputFileString;
357 }
358
359 QString CReasonerEvaluationExecutorThread::getNumberString(cint64 number) {
360 QString numberString = QString::number(number);
361 while (numberString.size() < mNumberStringSize) {
362 numberString = QString("0%1").arg(numberString);
363 }
364 return numberString;
365 }
366
367
368
369 CReasonerEvaluationEvaluator* CReasonerEvaluationExecutorThread::getReasonerEvaluatorForTestfile(const QString& testFileNameString, CReasonerEvaluationRequestClientBaseThread* reasonerClient) {
370 if (dynamic_cast<CReasonerEvaluationRequestClientOWLlinkThread*>(reasonerClient)) {
371 return mReasonerEvaluatorOWLlink;
372 } else if (dynamic_cast<CReasonerEvaluationRequestClientSPARQLThread*>(reasonerClient)) {
373 return mReasonerEvaluatorSPARQL;
374 }
375 LOG(WARN, getLogDomain(), logTr("No supported evaluator for request file '%1'.").arg(mWaitingTimeAfterReasonerCreation), this);
376 return mReasonerEvaluatorOWLlink;
377 }
378
379
380 bool CReasonerEvaluationExecutorThread::evaluateExecutedTest(CReasonerEvaluationTerminationResult* terminationResults, CReasonerEvaluationRequestResult* requestResults, const QString& testcaseOutput, bool forceWriting) {
381 if (requestResults->hasSucessfullReasonerCommunication() || forceWriting) {
382 LOG(INFO,getLogDomain(),logTr("Evaluate executed test case to '%1'.").arg(testcaseOutput),this);
383 mCurrentReasonerEvaluator = getReasonerEvaluatorForTestfile(mNextTestInputFile, mCurrentReasonerClient);
384 mCurrentReasonerEvaluator->evaluateResults(requestResults,terminationResults,mNextTestInputFile,testcaseOutput,mConfiguration);
385 mExecutedTest = true;
386 return true;
387 } else {
388 LOG(ERROR,getLogDomain(),logTr("Communication with reasoner failed for test case '%1'.").arg(testcaseOutput),this);
389 }
390 return false;
391 }
392
393
394 bool CReasonerEvaluationExecutorThread::testSystemReadyForNextEvaluationTest() {
395 if (mCriticalProcessesTester->hasCriticalSystemProcesses()) {
396 LOG(WARNING,getLogDomain(),logTr("Critical processes are running on the test system."),this);
397 return false;
398 }
399 return true;
400 }
401
402 void CReasonerEvaluationExecutorThread::delayNextEvaluationTestUntilSystemReady() {
403 while (!testSystemReadyForNextEvaluationTest()) {
404 LOG(WARNING,getLogDomain(),logTr("Test system not ready, retest after %1 ms.").arg(mSystemReadyRetestTimeForTests),this);
405 QThread::msleep(mSystemReadyRetestTimeForTests);
406 }
407 }
408
409
410
411 bool CReasonerEvaluationExecutorThread::executeNextEvaluationTest(const QString& testcaseInit, const QString& testcaseInput) {
412
413 delayNextEvaluationTestUntilSystemReady();
414
415 if (!mReasonerProvider) {
416 mReasonerProvider = mReasonerEvalFactory->createReasonerProvider(mConfiguration);
417 }
418 mReasonerProvider->createReasoner(mConfiguration);
419
420 QString addressString = mReasonerProvider->getReasonerAddressString();
421
422 if (mFirstTest) {
423 LOG(INFO,getLogDomain(),logTr("Warming up for reasoner '%1' with test case '%2'.").arg(mReasonerName).arg(testcaseInput),this);
424 } else {
425 LOG(INFO,getLogDomain(),logTr("Testing reasoner '%1' with test case '%2'.").arg(mReasonerName).arg(testcaseInput),this);
426 }
427 if (mWaitingTimeAfterReasonerCreation > 0) {
428 LOG(INFO,getLogDomain(),logTr("Waiting %1 ms after created reasoner.").arg(mWaitingTimeAfterReasonerCreation),this);
429 QThread::msleep(mWaitingTimeAfterReasonerCreation);
430 }
431 mCurrentReasonerClient = getReasonerClientForTestfile(testcaseInput);
432 mCurrentReasonerClient->evaluateReasoner(testcaseInit,testcaseInput, mReasonerProvider,mConfiguration,new CReasonerEvaluationNextEvent(this,0));
433 LOG(INFO, getLogDomain(), logTr("Executing reasoner for test case '%1'.").arg(testcaseInput), this);
434 return true;
435 }
436
437
438
439 CReasonerEvaluationRequestClientBaseThread* CReasonerEvaluationExecutorThread::getReasonerClientForTestfile(const QString& testFileNameString) {
440 if (mReasonerOWLlinkClient->canHandleRequestFile(testFileNameString)) {
441 return mReasonerOWLlinkClient;
442 }
443 if (mReasonerSPARQLClient->canHandleRequestFile(testFileNameString)) {
444 return mReasonerSPARQLClient;
445 }
446 LOG(WARN, getLogDomain(), logTr("No supported client handler for request file '%1'.").arg(mWaitingTimeAfterReasonerCreation), this);
447 return mReasonerOWLlinkClient;
448 }
449
450
451
452 bool CReasonerEvaluationExecutorThread::testFileFiltered(const QString& testFileNameString, const QString& testFilePathString, const QString& subDirPathString) {
453 if (!mFiltering) {
454 return true;
455 } else {
456 QStringList statisticList = mFiltering->loadStatistics(testFileNameString,subDirPathString);
457 return mFiltering->testSatisfiesFilter(testFileNameString,testFilePathString,subDirPathString,statisticList);
458 }
459 }
460
461
462 QString CReasonerEvaluationExecutorThread::nextEvaluationTestfile() {
463 if (mRemainingFiles.isEmpty()) {
464 while (mRemainingFiles.isEmpty() && !mRemainingSubDirs.isEmpty()) {
465 QString nextDir = mRemainingSubDirs.takeFirst();
466 QDir dir(mInputDir+nextDir);
467 QStringList subDirList = dir.entryList(QDir::Dirs,QDir::Name|QDir::IgnoreCase);
468 foreach (QString subDir, subDirList) {
469 if (!subDir.startsWith("-") && subDir != "." && subDir != "..") {
470 mRemainingSubDirs.prepend(QString("%1").arg(nextDir+"/"+subDir));
471 }
472 }
473 QStringList fileList = dir.entryList(QDir::Files,QDir::Name|QDir::IgnoreCase);
474 foreach (QString fileString, fileList) {
475 if (!fileString.startsWith("-")) {
476 if (testFileFiltered(fileString,mInputDir,nextDir+"/")) {
477 mRemainingFiles.append(nextDir+"/"+fileString);
478 } else {
479 //LOG(INFO,getLogDomain(),logTr("File '%1' in directory '%2' filtered out.").arg(fileString).arg(nextDir),this);
480 }
481 }
482 }
483 }
484 }
485 QString nextTestfile;
486 if (!mRemainingFiles.isEmpty()) {
487 nextTestfile = mRemainingFiles.takeFirst();
488 }
489 return nextTestfile;
490 }
491
492 }; // end namespace Evaluation
493
494 }; // end namespace Test
495
496 }; // end namespace Konclude