Debian Codegraph @

Source/Test/Evaluation/CReasonerEvaluationManagerThread.cpp

raw
1 /*
2 * Copyright (C) 2013-2015, 2019 by the Konclude Developer Team.
3 *
4 * This file is part of the reasoning system Konclude.
5 * For details and support, see <http://konclude.com/>.
6 *
7 * Konclude is free software: you can redistribute it and/or modify
8 * it under the terms of version 3 of the GNU Lesser General Public
9 * License (LGPLv3) as published by the Free Software Foundation.
10 *
11 * Konclude is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU (Lesser) General Public License for more details.
15 *
16 * You should have received a copy of the GNU (Lesser) General Public
17 * License along with Konclude. If not, see <http://www.gnu.org/licenses/>.
18 *
19 */
20
21 #include "CReasonerEvaluationManagerThread.h"
22
23
24 namespace Konclude {
25
26 namespace Test {
27
28 namespace Evaluation {
29
30
31
32 CReasonerEvaluationManagerThread::CReasonerEvaluationManagerThread(CConfiguration *config) : CThread("ReasonerEvaluationManagerThread"), CLogIdentifier("::Konclude::Test::Evaluation::ReasonerEvaluationManagerThread",this) {
33 mConfig = config;
34 mSatisticBasePath = "Evaluation/Infos/Statistics/Requests/";
35 startThread();
36 }
37
38
39 CReasonerEvaluationManagerThread::~CReasonerEvaluationManagerThread() {
40 }
41
42
43 CReasonerEvaluationManager* CReasonerEvaluationManagerThread::realizeEvaluationCommand(CEvaluateTestsuiteCommand* evalTestsuiteComm) {
44 postEvent(new CReasonerEvaluationProgramEvent(evalTestsuiteComm));
45 return this;
46 }
47
48
49 bool CReasonerEvaluationManagerThread::processCustomsEvents(QEvent::Type type, CCustomEvent* event) {
50 if (CThread::processCustomsEvents(type,event)) {
51 return true;
52 } else {
53 if (type == CReasonerEvaluationProgramEvent::EVENTTYPE) {
54 CReasonerEvaluationProgramEvent* repe = (CReasonerEvaluationProgramEvent*)event;
55 CEvaluateTestsuiteCommand* evalTestComm = repe->getEvaluationTestsuiteCommand();
56
57 CCommandRecordRouter commandRecordRouter(evalTestComm,this);
58 CStartProcessCommandRecord::makeRecord(&commandRecordRouter);
59
60 initEvaluationProgram(evalTestComm);
61 evaluationProgram(evalTestComm);
62 analysingProgram(evalTestComm);
63
64 CStopProcessCommandRecord::makeRecord(&commandRecordRouter);
65 CFinishProcessCommandRecord::makeRecord(&commandRecordRouter);
66
67 return true;
68 }
69 }
70 return false;
71 }
72
73
74 CReasonerEvaluationManager* CReasonerEvaluationManagerThread::initEvaluationProgram(CEvaluateTestsuiteCommand* evalTestsuiteComm) {
75 CReasonerEvaluationProgram* evalProg = evalTestsuiteComm->getEvaluationTestsuite();
76 mReasonerNameStringList.clear();
77 mReasonerOutputDirStringList.clear();
78 mTestTimeout = 0;
79 mTestCount = 0;
80 mOneReasonerFailed = false;
81 mAllReasonerFailed = true;
82 mFiltering = new CReasonerEvaluationFiltering(mSatisticBasePath);
83 if (!evalProg->getFilterStringList().isEmpty()) {
84 mFitlerStringList = evalProg->getFilterStringList();
85 mFiltering->loadFiltersFromFile(mFitlerStringList);
86 } else {
87 mFiltering->loadEmptyFilter();
88 }
89 return this;
90 }
91
92
93
94 bool CReasonerEvaluationManagerThread::evaluationProgram(CEvaluateTestsuiteCommand* evalTestsuiteComm) {
95 CReasonerEvaluationProgram* evalProg = evalTestsuiteComm->getEvaluationTestsuite();
96
97 QStringList reasonerConfigList = evalProg->getReasonerStringList();
98
99 QString testInputDirectory = evalProg->getInputDirectoryString();
100 QString testOutputDirectory = evalProg->getOutputDirectoryString();
101
102 QString plattform = CConfigDataReader::readConfigString(mConfig,"Konclude.Evaluation.Platform");
103
104 QString globalConfig = evalProg->getConfigFileString();
105
106 bool evaluationRunned = false;
107
108
109
110
111
112 if (plattform != "UNSPECIFIED") {
113
114
115 QTime totalTimer;
116 totalTimer.start();
117 LOG(INFO,getLogDomain(),logTr("Starting evaluation."),this);
118
119 foreach (QString reasonerConfig, reasonerConfigList) {
120
121 QTime timer;
122 timer.start();
123
124 LOG(INFO,getLogDomain(),logTr("Starting evaluation for reasoner '%1'.").arg(reasonerConfig),this);
125
126
127 CReasonerEvaluationTestsuite* reasonerEvalTestsuite = new CReasonerEvaluationTestsuite();
128 reasonerEvalTestsuite->initEvaluationTestsuite(globalConfig,reasonerConfig,testInputDirectory,testOutputDirectory,plattform,mFiltering);
129
130 CBlockingCallbackData blockCallData;
131
132 CReasonerEvaluationExecutorThread* reasonerExecutorThread = new CReasonerEvaluationExecutorThread(mConfig);
133 reasonerExecutorThread->executeEvaluation(reasonerEvalTestsuite,&blockCallData);
134 blockCallData.waitForCallback();
135 CReasonerEvaluationEvaluatedCallbackContextData* evaluatedCallbackContextData = (CReasonerEvaluationEvaluatedCallbackContextData*)blockCallData.getCallbackDataContext();
136 if (!evaluatedCallbackContextData->getEvaluationFailed()) {
137 evaluationRunned |= evaluatedCallbackContextData->getEvaluationRunned();
138 mReasonerNameStringList.append(evaluatedCallbackContextData->getReasonerNameString());
139 mReasonerOutputDirStringList.append(evaluatedCallbackContextData->getReasonerOutputDirectoryString());
140 mTestTimeout = evaluatedCallbackContextData->getTestTimeout();
141 mTestCount = evaluatedCallbackContextData->getTestCount();
142 mAllReasonerFailed = false;
143 } else {
144 mOneReasonerFailed = true;
145 mFailedReasonerStringList.append(reasonerConfig);
146 }
147 delete reasonerExecutorThread;
148
149 cint64 elapsedTime = timer.elapsed();
150 LOG(INFO,getLogDomain(),logTr("Finished evaluation for reasoner '%1' in '%2' seconds.").arg(reasonerConfig).arg(elapsedTime/1000),this);
151
152 }
153
154 cint64 totalElapsedTime = totalTimer.elapsed();
155 LOG(INFO,getLogDomain(),logTr("Finished evaluation in '%1' seconds.").arg(totalElapsedTime/1000),this);
156 } else {
157 LOG(ERROR,getLogDomain(),logTr("Evaluation not possible, platform not specified."),this);
158 }
159
160 return evaluationRunned;
161 }
162
163
164
165
166 CReasonerEvaluationGroupRequestSelector* CReasonerEvaluationManagerThread::getSelectors(const QString& requestDirectory) {
167 CReasonerEvaluationGroupRequestSelector* selectors = nullptr;
168 CReasonerEvaluationExpressivitySelectorFactory expEvalSelFactory("ExpressivenessGrouped","ExpressivenessGrouped/",mSatisticBasePath);
169 selectors = expEvalSelFactory.getSelectors(selectors);
170 CReasonerEvaluationDirectoryStructureSelectorFactory dirEvalSelFactory(requestDirectory,requestDirectory,"DirectoryGrouped","DirectoryGrouped/");
171 selectors = dirEvalSelFactory.getSelectors(selectors);
172 CReasonerEvaluationProfileSelectorFactory profileEvalSelFactory("ProfileGrouped","ProfileGrouped/",mSatisticBasePath);
173 selectors = profileEvalSelFactory.getSelectors(selectors);
174 return selectors;
175 }
176
177
178
179 QString CReasonerEvaluationManagerThread::getAnalyserOutputDirectory(const QString& baseDirectory, const QString& evaluationProgramName, const QString& platformName, const QString& analyserName) {
180 QString outputDirectory = baseDirectory;
181 if (outputDirectory.contains("$$_ANALYSER_NAME_$$")) {
182 outputDirectory = outputDirectory.replace("$$_ANALYSER_NAME_$$",analyserName);
183 outputDirectory = outputDirectory.replace("$$_EVALUATION_PROGRAM_NAME_$$",evaluationProgramName);
184 outputDirectory = outputDirectory.replace("$$_PLATFORM_$$",platformName);
185 } else {
186 outputDirectory = outputDirectory+evaluationProgramName+"/"+platformName+"/"+analyserName+"/";
187 }
188 QDir currDir(".");
189 currDir.mkpath(outputDirectory);
190 return outputDirectory;
191 }
192
193
194
195
196 bool CReasonerEvaluationManagerThread::analyseTimes(const QString& analysingDirectoryString, const QString& evaluationProgramName, const QString& plattform, const QString& analyserOutputDirectoryName, const QString& requestDirectory, cint64 testCountCut, cint64 timeoutCut, cint64 errorPunishmentTime, CReasonerEvaluationDataValueCacher* dataValueCacher, CReasonerEvaluationExtractor::EXTRACTORTYPE extractorType, const QString& extractionName, const QString& contextSpecificationName) {
197 bool analysed = false;
198 QString outputDirectory = getAnalyserOutputDirectory(analysingDirectoryString,evaluationProgramName,plattform,analyserOutputDirectoryName);
199
200 CReasonerEvaluationAnalyserChecker analysingUpdateChecker;
201 if (analysingUpdateChecker.checkAnalysingUpdateNecessary(mReasonerOutputDirStringList,outputDirectory+"AnalysingUpdateCheckingData.dat",testCountCut)) {
202
203 LOG(INFO,getLogDomain(),logTr("Analysing %1 times for '%2' into directory '%3'.").arg(extractionName).arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
204
205 CReasonerEvaluationSpecifiedTimeExtractor* timeExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(extractorType,timeoutCut,errorPunishmentTime,dataValueCacher);
206 CReasonerEvaluationAvaragerSummarizer* avarageSummarize = new CReasonerEvaluationAvaragerSummarizer();
207
208 CReasonerEvaluationAnalyseContext* reasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig,contextSpecificationName);
209
210 CReasonerEvaluationCollector* collector = new CReasonerEvaluationCollector(timeExtractor,avarageSummarize,mFiltering);
211
212 CReasonerEvaluationDataValueGroupCollectionReasonerComparison* reasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
213
214 for (QStringList::const_iterator it1 = mReasonerNameStringList.constBegin(), it2 = mReasonerOutputDirStringList.constBegin(), it1End = mReasonerNameStringList.constEnd(), it2End = mReasonerOutputDirStringList.constEnd(); it1 != it1End && it2 != it2End; ++it1, ++it2) {
215 QString reasonerName(*it1);
216 QString reasonerPath(*it2);
217 LOG(INFO,getLogDomain(),logTr("Extracting %1 times for reasoner '%2' from responses in directory '%3'.").arg(extractionName).arg(reasonerName).arg(reasonerPath),this);
218 collector->collectReasonerEvaluationDataValues(reasonerComp,reasonerName,reasonerPath);
219 }
220
221
222 LOG(INFO,getLogDomain(),logTr("Comparing %1 times for '%2'.").arg(extractionName).arg(mReasonerNameStringList.join(", ")),this);
223
224
225 CReasonerEvaluationGroupRequestSelector* selectors = getSelectors(requestDirectory);
226
227
228 CReasonerEvaluationGroupRequestReasonerCSVComparisonAnalyser* analyser1 = new CReasonerEvaluationGroupRequestReasonerCSVComparisonAnalyser();
229 analyser1->analyseEvaluationData(reasonerComp,reasonerEvaluationContext,outputDirectory,selectors);
230
231 CReasonerEvaluationGroupRequestReasonerWithMeanMinMaxCSVComparisonAnalyser* analyser2 = new CReasonerEvaluationGroupRequestReasonerWithMeanMinMaxCSVComparisonAnalyser();
232 analyser2->analyseEvaluationData(reasonerComp,reasonerEvaluationContext,outputDirectory,selectors);
233
234 CReasonerEvaluationGroupReasonerWinnerCountCSVComparisonAnalyser* analyser3 = new CReasonerEvaluationGroupReasonerWinnerCountCSVComparisonAnalyser();
235 analyser3->analyseEvaluationData(reasonerComp,reasonerEvaluationContext,outputDirectory,selectors);
236
237 CReasonerEvaluationGroupRequestReasonerBucketCSVComparisonAnalyser* analyser4 = new CReasonerEvaluationGroupRequestReasonerBucketCSVComparisonAnalyser(100.,timeoutCut,qSqrt(10.),timeoutCut);
238 analyser4->analyseEvaluationData(reasonerComp,reasonerEvaluationContext,outputDirectory,selectors);
239 CReasonerEvaluationGroupRequestReasonerBucketCSVComparisonAnalyser* analyser5 = new CReasonerEvaluationGroupRequestReasonerBucketCSVComparisonAnalyser(100.,timeoutCut,10.,timeoutCut);
240 analyser5->analyseEvaluationData(reasonerComp,reasonerEvaluationContext,outputDirectory,selectors);
241
242 LOG(INFO,getLogDomain(),logTr("Compared %1 times for '%2' into directory '%3'.").arg(extractionName).arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
243
244 analysingUpdateChecker.saveAnalysingUpdateCheckFile(outputDirectory+"AnalysingUpdateCheckingData.dat");
245 analysed = true;
246 } else {
247 LOG(WARNING,getLogDomain(),logTr("Compared %1 times for '%2' in directory '%3' are already up to date.").arg(extractionName).arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
248 }
249 return analysed;
250 }
251
252
253
254 CReasonerEvaluationManager* CReasonerEvaluationManagerThread::analysingProgram(CEvaluateTestsuiteCommand* evalTestsuiteComm) {
255 CReasonerEvaluationProgram* evalProg = evalTestsuiteComm->getEvaluationTestsuite();
256
257 bool analysed = false;
258
259 QString plattform = CConfigDataReader::readConfigString(mConfig,"Konclude.Evaluation.Platform");
260
261 QString analysingDirectoryString = CReasonerEvaluationPathConverter::getCorrectedPathString(evalProg->getAnalyserDirectoryString());
262 if (!analysingDirectoryString.endsWith("/")) {
263 analysingDirectoryString += "/";
264 }
265
266 QString requestDirectory = evalProg->getInputDirectoryString();
267 if (!requestDirectory.endsWith("/")) {
268 requestDirectory += "/";
269 }
270
271 QString evaluationProgramName = evalProg->getProgramName();
272 cint64 testCountCut = mTestCount;
273 cint64 timeoutCut = mTestTimeout;
274 cint64 errorPunishmentTime = 1;
275
276
277 CReasonerEvaluationDataValueCacher* dataValueCacher = new CReasonerEvaluationDataValueCacher();
278
279 if (!mAllReasonerFailed) {
280
281 QString dataString = QDateTime::currentDateTime().date().toString(Qt::ISODate);
282
283 if (plattform != "UNSPECIFIED") {
284
285 QTime totalTimer;
286 totalTimer.start();
287 LOG(INFO,getLogDomain(),logTr("Starting analysing."),this);
288
289
290 QStringList analysingStringList = evalProg->getAnalyserStringList();
291
292 foreach (QString ananysingString, analysingStringList) {
293
294
295 QTime timer;
296 timer.start();
297
298 LOG(INFO,getLogDomain(),logTr("Starting analysing for '%1'.").arg(ananysingString),this);
299
300
301 if (ananysingString == "ReasonerEvaluationClassificationTimesComparer") {
302 analyseTimes(analysingDirectoryString,evaluationProgramName,plattform,"ClassificationTimeComparison",requestDirectory,testCountCut,timeoutCut,errorPunishmentTime,dataValueCacher,CReasonerEvaluationExtractor::CLASSIFICATIONTIMEEXTRACTOR,"classification","Classification ");
303 }
304
305 if (ananysingString == "ReasonerEvaluationComplexQueryingTimesComparer") {
306 analyseTimes(analysingDirectoryString, evaluationProgramName, plattform, "ComplexQueryingTimeComparison", requestDirectory, testCountCut, timeoutCut, errorPunishmentTime, dataValueCacher, CReasonerEvaluationExtractor::COMPLEXQUERYINGTIMEEXTRACTOR, "complex-querying", "Complex Querying ");
307 }
308
309 if (ananysingString == "ReasonerEvaluationRealizationTimesComparer") {
310 analyseTimes(analysingDirectoryString,evaluationProgramName,plattform,"RealizationTimeComparison",requestDirectory,testCountCut,timeoutCut,errorPunishmentTime,dataValueCacher,CReasonerEvaluationExtractor::REALIZATIONTIMEEXTRACTOR,"realization","Realization ");
311 }
312
313 if (ananysingString == "ReasonerEvaluationConsistencyTimesComparer") {
314 analyseTimes(analysingDirectoryString,evaluationProgramName,plattform,"ConsistencyTimeComparison",requestDirectory,testCountCut,timeoutCut,errorPunishmentTime,dataValueCacher,CReasonerEvaluationExtractor::CONSISTENCYTIMEEXTRACTOR,"consistency","Consistency ");
315 }
316
317 if (ananysingString == "ReasonerEvaluationSatisfiabilityTimesComparer") {
318 analyseTimes(analysingDirectoryString,evaluationProgramName,plattform,"SatisfiabilityTimeComparison",requestDirectory,testCountCut,timeoutCut,errorPunishmentTime,dataValueCacher,CReasonerEvaluationExtractor::SATISFIABILITYTIMEEXTRACTOR,"satisfiability","Satisfiability ");
319 }
320
321 if (ananysingString == "ReasonerEvaluationLoadingTimesComparer") {
322 analyseTimes(analysingDirectoryString,evaluationProgramName,plattform,"LoadingTimeComparison",requestDirectory,testCountCut,timeoutCut,errorPunishmentTime,dataValueCacher,CReasonerEvaluationExtractor::LOADINGTIMEEXTRACTOR,"loading","Loading ");
323 }
324
325 if (ananysingString == "ReasonerEvaluationResponseTimesComparer") {
326 analyseTimes(analysingDirectoryString,evaluationProgramName,plattform,"ResponseTimeComparison",requestDirectory,testCountCut,timeoutCut,errorPunishmentTime,dataValueCacher,CReasonerEvaluationExtractor::TOTALRESPONSETIMEEXTRACTOR,"response","Response ");
327 }
328
329
330 if (ananysingString == "ReasonerEvaluationAccumulatedTimesComparer") {
331
332 QString outputDirectory = getAnalyserOutputDirectory(analysingDirectoryString,evaluationProgramName,plattform,"AccumulatedTimeComparison");
333
334 CReasonerEvaluationAnalyserChecker analysingUpdateChecker;
335 if (analysingUpdateChecker.checkAnalysingUpdateNecessary(mReasonerOutputDirStringList,outputDirectory+"AnalysingUpdateCheckingData.dat",testCountCut)) {
336
337 LOG(INFO,getLogDomain(),logTr("Analysing accumulated times for '%1' to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
338
339
340 CReasonerEvaluationSpecifiedTimeExtractor* classTimeExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::CLASSIFICATIONTIMEEXTRACTOR, timeoutCut, errorPunishmentTime, dataValueCacher);
341 CReasonerEvaluationSpecifiedTimeExtractor* complexQueryingTimeExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::COMPLEXQUERYINGTIMEEXTRACTOR, timeoutCut, errorPunishmentTime, dataValueCacher);
342 CReasonerEvaluationSpecifiedTimeExtractor* consTimeExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::CONSISTENCYTIMEEXTRACTOR,timeoutCut,errorPunishmentTime,dataValueCacher);
343 CReasonerEvaluationSpecifiedTimeExtractor* satTimeExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::SATISFIABILITYTIMEEXTRACTOR,timeoutCut,errorPunishmentTime,dataValueCacher);
344 CReasonerEvaluationSpecifiedTimeExtractor* realTimeExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::REALIZATIONTIMEEXTRACTOR,timeoutCut,errorPunishmentTime,dataValueCacher);
345 CReasonerEvaluationSpecifiedTimeExtractor* loadTimeExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::LOADINGTIMEEXTRACTOR,timeoutCut,errorPunishmentTime,dataValueCacher);
346 CReasonerEvaluationSpecifiedTimeExtractor* reponseTimeExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::TOTALRESPONSETIMEEXTRACTOR,timeoutCut,errorPunishmentTime,dataValueCacher);
347 CReasonerEvaluationSpecifiedTimeExtractor* errorExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::ERROREXTRACTOR,timeoutCut,errorPunishmentTime,dataValueCacher);
348 CReasonerEvaluationSpecifiedTimeExtractor* timeoutExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::TIMEOUTEXTRACTOR,timeoutCut,errorPunishmentTime,dataValueCacher);
349
350 CReasonerEvaluationAvaragerSummarizer* avarageSummarize = new CReasonerEvaluationAvaragerSummarizer();
351
352
353 CReasonerEvaluationCollector* responseCollector = new CReasonerEvaluationCollector(reponseTimeExtractor,avarageSummarize,mFiltering);
354 CReasonerEvaluationDataValueGroupCollectionReasonerComparison* responceReasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
355 CReasonerEvaluationCollector* loadingCollector = new CReasonerEvaluationCollector(loadTimeExtractor,avarageSummarize,mFiltering);
356 CReasonerEvaluationDataValueGroupCollectionReasonerComparison* loadingReasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
357 CReasonerEvaluationCollector* classificationCollector = new CReasonerEvaluationCollector(classTimeExtractor,avarageSummarize,mFiltering);
358 CReasonerEvaluationDataValueGroupCollectionReasonerComparison* classificationReasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
359
360 CReasonerEvaluationCollector* complexQueryingCollector = new CReasonerEvaluationCollector(complexQueryingTimeExtractor, avarageSummarize, mFiltering);
361 CReasonerEvaluationDataValueGroupCollectionReasonerComparison* complexQueryingReasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
362
363 CReasonerEvaluationCollector* realizationCollector = new CReasonerEvaluationCollector(realTimeExtractor,avarageSummarize,mFiltering);
364 CReasonerEvaluationDataValueGroupCollectionReasonerComparison* realizationReasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
365 CReasonerEvaluationCollector* consistencyCollector = new CReasonerEvaluationCollector(consTimeExtractor,avarageSummarize,mFiltering);
366 CReasonerEvaluationDataValueGroupCollectionReasonerComparison* consistencyReasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
367 CReasonerEvaluationCollector* satisfiabilityCollector = new CReasonerEvaluationCollector(satTimeExtractor,avarageSummarize,mFiltering);
368 CReasonerEvaluationDataValueGroupCollectionReasonerComparison* satisfiabilityReasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
369 CReasonerEvaluationCollector* errorCollector = new CReasonerEvaluationCollector(errorExtractor,avarageSummarize,mFiltering);
370 CReasonerEvaluationDataValueGroupCollectionReasonerComparison* errorReasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
371 CReasonerEvaluationCollector* timeoutCollector = new CReasonerEvaluationCollector(timeoutExtractor,avarageSummarize,mFiltering);
372 CReasonerEvaluationDataValueGroupCollectionReasonerComparison* timeoutReasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
373
374
375 for (QStringList::const_iterator it1 = mReasonerNameStringList.constBegin(), it2 = mReasonerOutputDirStringList.constBegin(), it1End = mReasonerNameStringList.constEnd(), it2End = mReasonerOutputDirStringList.constEnd(); it1 != it1End && it2 != it2End; ++it1, ++it2) {
376 QString reasonerName(*it1);
377 QString reasonerPath(*it2);
378 LOG(INFO,getLogDomain(),logTr("Extracting accumulated times for '%1' reasoner from responses in directory '%2'.").arg(reasonerName).arg(reasonerPath),this);
379 responseCollector->collectReasonerEvaluationDataValues(responceReasonerComp,reasonerName,reasonerPath);
380 loadingCollector->collectReasonerEvaluationDataValues(loadingReasonerComp,reasonerName,reasonerPath);
381 classificationCollector->collectReasonerEvaluationDataValues(classificationReasonerComp, reasonerName, reasonerPath);
382 complexQueryingCollector->collectReasonerEvaluationDataValues(complexQueryingReasonerComp, reasonerName, reasonerPath);
383 realizationCollector->collectReasonerEvaluationDataValues(realizationReasonerComp,reasonerName,reasonerPath);
384 consistencyCollector->collectReasonerEvaluationDataValues(consistencyReasonerComp,reasonerName,reasonerPath);
385 satisfiabilityCollector->collectReasonerEvaluationDataValues(satisfiabilityReasonerComp,reasonerName,reasonerPath);
386 errorCollector->collectReasonerEvaluationDataValues(errorReasonerComp,reasonerName,reasonerPath);
387 timeoutCollector->collectReasonerEvaluationDataValues(timeoutReasonerComp,reasonerName,reasonerPath);
388 }
389
390
391 LOG(INFO,getLogDomain(),logTr("Comparing accumulated times for '%1'.").arg(mReasonerNameStringList.join(", ")),this);
392
393 CReasonerEvaluationGroupRequestSelector* selectors = getSelectors(requestDirectory);
394
395
396 CReasonerEvaluationAnalyseContext* responseReasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig,"Response ","response-");
397 CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser* responseAnalyser1 = new CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser();
398 responseAnalyser1->analyseEvaluationData(responceReasonerComp,responseReasonerEvaluationContext,outputDirectory,selectors);
399
400 CReasonerEvaluationAnalyseContext* loadingReasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig,"Loading ","loading-");
401 CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser* loadingAnalyser1 = new CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser();
402 loadingAnalyser1->analyseEvaluationData(loadingReasonerComp,loadingReasonerEvaluationContext,outputDirectory,selectors);
403
404 CReasonerEvaluationAnalyseContext* classificationReasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig,"Classification ","classification-");
405 CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser* classificationAnalyser1 = new CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser();
406 classificationAnalyser1->analyseEvaluationData(classificationReasonerComp,classificationReasonerEvaluationContext,outputDirectory,selectors);
407
408 CReasonerEvaluationAnalyseContext* complexQueryingReasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig, "Complex Querying ", "complex-querying-");
409 CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser* complexQueryingAnalyser1 = new CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser();
410 complexQueryingAnalyser1->analyseEvaluationData(complexQueryingReasonerComp, complexQueryingReasonerEvaluationContext, outputDirectory, selectors);
411
412 CReasonerEvaluationAnalyseContext* realizationReasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig,"Realization ","realization-");
413 CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser* realizationAnalyser1 = new CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser();
414 realizationAnalyser1->analyseEvaluationData(realizationReasonerComp,realizationReasonerEvaluationContext,outputDirectory,selectors);
415
416 CReasonerEvaluationAnalyseContext* consistencyReasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig,"Consistency ","consistency-");
417 CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser* consistencyAnalyser1 = new CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser();
418 consistencyAnalyser1->analyseEvaluationData(consistencyReasonerComp,consistencyReasonerEvaluationContext,outputDirectory,selectors);
419
420 CReasonerEvaluationAnalyseContext* satisfiabilityReasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig,"Satisfiability ","satisfiability-");
421 CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser* satisfiabilityAnalyser1 = new CReasonerEvaluationGroupRequestReasonerAccumulatedCSVComparisonAnalyser();
422 satisfiabilityAnalyser1->analyseEvaluationData(satisfiabilityReasonerComp,satisfiabilityReasonerEvaluationContext,outputDirectory,selectors);
423
424 CReasonerEvaluationAnalyseContext* errorReasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig,"Error ","error-");
425 CReasonerEvaluationGroupRequestReasonerAccumulatedCountCSVComparisonAnalyser* errorAnalyser1 = new CReasonerEvaluationGroupRequestReasonerAccumulatedCountCSVComparisonAnalyser();
426 errorAnalyser1->analyseEvaluationData(errorReasonerComp,errorReasonerEvaluationContext,outputDirectory,selectors);
427
428 CReasonerEvaluationAnalyseContext* timeoutReasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig,"Timeout ","timeout-");
429 CReasonerEvaluationGroupRequestReasonerAccumulatedCountCSVComparisonAnalyser* timeoutAnalyser1 = new CReasonerEvaluationGroupRequestReasonerAccumulatedCountCSVComparisonAnalyser();
430 timeoutAnalyser1->analyseEvaluationData(timeoutReasonerComp,timeoutReasonerEvaluationContext,outputDirectory,selectors);
431
432 LOG(INFO,getLogDomain(),logTr("Accumulated times for '%1' compared to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
433 analysingUpdateChecker.saveAnalysingUpdateCheckFile(outputDirectory+"AnalysingUpdateCheckingData.dat");
434 analysed = true;
435 } else {
436 LOG(WARNING,getLogDomain(),logTr("Accumulated times for '%1' in '%2' already up to date.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
437 }
438 }
439
440
441
442
443
444
445
446
447 if (ananysingString == "ReasonerEvaluationStatisticValuesComparer") {
448
449 QString outputDirectory = getAnalyserOutputDirectory(analysingDirectoryString,evaluationProgramName,plattform,"StatisticValuesComparison");
450
451 CReasonerEvaluationAnalyserChecker analysingUpdateChecker;
452 if (analysingUpdateChecker.checkAnalysingUpdateNecessary(mReasonerOutputDirStringList,outputDirectory+"AnalysingUpdateCheckingData.dat",testCountCut)) {
453
454
455
456 LOG(INFO,getLogDomain(),logTr("Collecting statistic names for '%1' to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
457
458
459 QSet<QString> staticticNamesSet;
460 CReasonerEvaluationStatisticsNameCollectingExtractor* statNameCollectingExtractor = new CReasonerEvaluationStatisticsNameCollectingExtractor(&staticticNamesSet,dataValueCacher);
461
462 CReasonerEvaluationAvaragerSummarizer* avarageSummarize = new CReasonerEvaluationAvaragerSummarizer();
463
464 CReasonerEvaluationCollector* statNameCollector = new CReasonerEvaluationCollector(statNameCollectingExtractor,avarageSummarize,mFiltering);
465 CReasonerEvaluationDataValueGroupCollectionReasonerComparison* statNameReasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
466
467
468 for (QStringList::const_iterator it1 = mReasonerNameStringList.constBegin(), it2 = mReasonerOutputDirStringList.constBegin(), it1End = mReasonerNameStringList.constEnd(), it2End = mReasonerOutputDirStringList.constEnd(); it1 != it1End && it2 != it2End; ++it1, ++it2) {
469 QString reasonerName(*it1);
470 QString reasonerPath(*it2);
471 LOG(INFO,getLogDomain(),logTr("Extracting statistic names for '%1' reasoner from responses in directory '%2'.").arg(reasonerName).arg(reasonerPath),this);
472 statNameCollector->collectReasonerEvaluationDataValues(statNameReasonerComp,reasonerName,reasonerPath);
473 }
474
475
476 LOG(INFO,getLogDomain(),logTr("Analysing statistic values for '%1' to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
477
478
479 CReasonerEvaluationGroupRequestReasonerStatisticValueCollectingCSVComparisonAnalyser* analyser4 = new CReasonerEvaluationGroupRequestReasonerStatisticValueCollectingCSVComparisonAnalyser();
480 CReasonerEvaluationGroupRequestSelector* selectors = getSelectors(requestDirectory);
481
482
483 for (QSet<QString>::const_iterator it = staticticNamesSet.constBegin(), itEnd = staticticNamesSet.constEnd(); it != itEnd; ++it) {
484 QString statName(*it);
485
486 analyser4->addNextStatisticName(statName);
487
488 CReasonerEvaluationStatisticsValueExtractor* statValueExtractor = new CReasonerEvaluationStatisticsValueExtractor(statName,&staticticNamesSet,timeoutCut,errorPunishmentTime,dataValueCacher);
489
490 CReasonerEvaluationCollector* statValueCollector = new CReasonerEvaluationCollector(statValueExtractor,avarageSummarize,mFiltering);
491 CReasonerEvaluationDataValueGroupCollectionReasonerComparison* statValueReasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
492
493
494 for (QStringList::const_iterator it1 = mReasonerNameStringList.constBegin(), it2 = mReasonerOutputDirStringList.constBegin(), it1End = mReasonerNameStringList.constEnd(), it2End = mReasonerOutputDirStringList.constEnd(); it1 != it1End && it2 != it2End; ++it1, ++it2) {
495 QString reasonerName(*it1);
496 QString reasonerPath(*it2);
497 LOG(INFO,getLogDomain(),logTr("Extracting statistic values for '%1' reasoner from responses in directory '%2'.").arg(reasonerName).arg(reasonerPath),this);
498 statValueCollector->collectReasonerEvaluationDataValues(statValueReasonerComp,reasonerName,reasonerPath);
499 }
500
501
502
503
504 LOG(INFO,getLogDomain(),logTr("Comparing statistic values for '%1'.").arg(mReasonerNameStringList.join(", ")),this);
505
506
507 CReasonerEvaluationAnalyseContext* reasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig,QString("%1 Statistics").arg(statName.toUpper()),statName+"-");
508
509
510 CReasonerEvaluationGroupRequestReasonerCSVComparisonAnalyser* analyser1 = new CReasonerEvaluationGroupRequestReasonerCSVComparisonAnalyser();
511 analyser1->analyseEvaluationData(statValueReasonerComp,reasonerEvaluationContext,outputDirectory,selectors);
512
513 CReasonerEvaluationGroupRequestReasonerStatisticValueCSVComparisonAnalyser* analyser2 = new CReasonerEvaluationGroupRequestReasonerStatisticValueCSVComparisonAnalyser();
514 analyser2->analyseEvaluationData(statValueReasonerComp,reasonerEvaluationContext,outputDirectory,selectors);
515
516 CReasonerEvaluationGroupRequestReasonerAccumulatedCountCSVComparisonAnalyser* analyser3 = new CReasonerEvaluationGroupRequestReasonerAccumulatedCountCSVComparisonAnalyser();
517 analyser3->analyseEvaluationData(statValueReasonerComp,reasonerEvaluationContext,outputDirectory,selectors);
518
519 analyser4->analyseEvaluationData(statValueReasonerComp,reasonerEvaluationContext,outputDirectory,selectors);
520
521 LOG(INFO,getLogDomain(),logTr("Statistics values for '%1' compared to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
522 analysingUpdateChecker.saveAnalysingUpdateCheckFile(outputDirectory+"AnalysingUpdateCheckingData.dat");
523
524
525 analysed = true;
526 }
527
528
529
530 } else {
531 LOG(WARNING,getLogDomain(),logTr("Statistic values for '%1' in '%2' already up to date.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
532 }
533 }
534
535
536
537
538
539
540
541
542
543 if (ananysingString == "ReasonerEvaluationResultDifferencesComparer") {
544
545 QString outputDirectory = getAnalyserOutputDirectory(analysingDirectoryString,evaluationProgramName,plattform,"ResultDifferenceComparison");
546
547 CReasonerEvaluationAnalyserChecker analysingUpdateChecker;
548 if (analysingUpdateChecker.checkAnalysingUpdateNecessary(mReasonerOutputDirStringList,outputDirectory+"AnalysingUpdateCheckingData.dat",testCountCut)) {
549
550 LOG(INFO,getLogDomain(),logTr("Analysing result differences for '%1' to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
551
552 CReasonerEvaluationSpecifiedTimeExtractor* respFileExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::RESPONSEFILEEXTRACTOR,timeoutCut,errorPunishmentTime,dataValueCacher);
553
554 CReasonerEvaluationResponseFileListSummarizer* respFileListSummarizer = new CReasonerEvaluationResponseFileListSummarizer();
555
556 CReasonerEvaluationAnalyseContext* reasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig);
557
558 CReasonerEvaluationCollector* collector = new CReasonerEvaluationCollector(respFileExtractor,respFileListSummarizer,mFiltering);
559
560 CReasonerEvaluationDataValueGroupCollectionReasonerComparison* reasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
561
562 for (QStringList::const_iterator it1 = mReasonerNameStringList.constBegin(), it2 = mReasonerOutputDirStringList.constBegin(), it1End = mReasonerNameStringList.constEnd(), it2End = mReasonerOutputDirStringList.constEnd(); it1 != it1End && it2 != it2End; ++it1, ++it2) {
563 QString reasonerName(*it1);
564 QString reasonerPath(*it2);
565 LOG(INFO,getLogDomain(),logTr("Collecting response files for '%1' reasoner in directory '%2'.").arg(reasonerName).arg(reasonerPath),this);
566 collector->collectReasonerEvaluationDataValues(reasonerComp,reasonerName,reasonerPath);
567 }
568
569
570 LOG(INFO,getLogDomain(),logTr("Comparing result differences for '%1'.").arg(mReasonerNameStringList.join(", ")),this);
571
572 CReasonerEvaluationGroupRequestSelector* selectors = getSelectors(requestDirectory);
573
574
575 CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser* analyser1 = new CReasonerEvaluationGroupRequestReasonerResultDifferenceAnalyser();
576 analyser1->analyseEvaluationData(reasonerComp,reasonerEvaluationContext,outputDirectory,selectors);
577
578
579 LOG(INFO,getLogDomain(),logTr("Result differences for '%1' compared to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
580 analysingUpdateChecker.saveAnalysingUpdateCheckFile(outputDirectory+"AnalysingUpdateCheckingData.dat");
581 analysed = true;
582 } else {
583 LOG(WARNING,getLogDomain(),logTr("Result differences for '%1' in '%2' already up to date.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
584 }
585 }
586
587
588
589
590
591
592
593
594
595 if (ananysingString == "ReasonerEvaluationResultSubsumptionsComparer") {
596
597 QString outputDirectory = getAnalyserOutputDirectory(analysingDirectoryString, evaluationProgramName, plattform, "ResultSubsumptionComparison");
598
599 CReasonerEvaluationAnalyserChecker analysingUpdateChecker;
600 if (analysingUpdateChecker.checkAnalysingUpdateNecessary(mReasonerOutputDirStringList, outputDirectory + "AnalysingUpdateCheckingData.dat", testCountCut)) {
601
602 LOG(INFO, getLogDomain(), logTr("Analysing result subsumptions for '%1' to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory), this);
603
604 CReasonerEvaluationSpecifiedTimeExtractor* respFileExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::RESPONSEFILEEXTRACTOR, timeoutCut, errorPunishmentTime, dataValueCacher);
605
606 CReasonerEvaluationResponseFileListSummarizer* respFileListSummarizer = new CReasonerEvaluationResponseFileListSummarizer();
607
608 CReasonerEvaluationAnalyseContext* reasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig);
609
610 CReasonerEvaluationCollector* collector = new CReasonerEvaluationCollector(respFileExtractor, respFileListSummarizer, mFiltering);
611
612 CReasonerEvaluationDataValueGroupCollectionReasonerComparison* reasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
613
614 for (QStringList::const_iterator it1 = mReasonerNameStringList.constBegin(), it2 = mReasonerOutputDirStringList.constBegin(), it1End = mReasonerNameStringList.constEnd(), it2End = mReasonerOutputDirStringList.constEnd(); it1 != it1End && it2 != it2End; ++it1, ++it2) {
615 QString reasonerName(*it1);
616 QString reasonerPath(*it2);
617 LOG(INFO, getLogDomain(), logTr("Collecting response files for '%1' reasoner in directory '%2'.").arg(reasonerName).arg(reasonerPath), this);
618 collector->collectReasonerEvaluationDataValues(reasonerComp, reasonerName, reasonerPath);
619 }
620
621
622 LOG(INFO, getLogDomain(), logTr("Comparing result subsumptions for '%1'.").arg(mReasonerNameStringList.join(", ")), this);
623
624 CReasonerEvaluationGroupRequestSelector* selectors = getSelectors(requestDirectory);
625
626
627 CReasonerEvaluationGroupRequestReasonerResultSubsumptionAnalyser* analyser1 = new CReasonerEvaluationGroupRequestReasonerResultSubsumptionAnalyser();
628 analyser1->analyseEvaluationData(reasonerComp, reasonerEvaluationContext, outputDirectory, selectors);
629
630
631 LOG(INFO, getLogDomain(), logTr("Result subsumptions for '%1' compared to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory), this);
632 analysingUpdateChecker.saveAnalysingUpdateCheckFile(outputDirectory + "AnalysingUpdateCheckingData.dat");
633 analysed = true;
634 } else {
635 LOG(WARNING, getLogDomain(), logTr("Result subsumptions for '%1' in '%2' already up to date.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory), this);
636 }
637 }
638
639
640
641
642
643
644
645 if (ananysingString == "ReasonerEvaluationResultCountingComparer") {
646
647 QString outputDirectory = getAnalyserOutputDirectory(analysingDirectoryString, evaluationProgramName, plattform, "ResultCountingComparison");
648
649 CReasonerEvaluationAnalyserChecker analysingUpdateChecker;
650 if (analysingUpdateChecker.checkAnalysingUpdateNecessary(mReasonerOutputDirStringList, outputDirectory + "AnalysingUpdateCheckingData.dat", testCountCut)) {
651
652 LOG(INFO, getLogDomain(), logTr("Analysing result counting for '%1' to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory), this);
653
654 CReasonerEvaluationSpecifiedTimeExtractor* respFileExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::RESPONSECOUNTEXTRACTOR, timeoutCut, errorPunishmentTime, dataValueCacher);
655
656 CReasonerEvaluationAvaragerSummarizer* avarageSummarize = new CReasonerEvaluationAvaragerSummarizer();
657
658 CReasonerEvaluationAnalyseContext* reasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig);
659
660 CReasonerEvaluationCollector* collector = new CReasonerEvaluationCollector(respFileExtractor, avarageSummarize, mFiltering);
661
662 CReasonerEvaluationDataValueGroupCollectionReasonerComparison* reasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
663
664 for (QStringList::const_iterator it1 = mReasonerNameStringList.constBegin(), it2 = mReasonerOutputDirStringList.constBegin(), it1End = mReasonerNameStringList.constEnd(), it2End = mReasonerOutputDirStringList.constEnd(); it1 != it1End && it2 != it2End; ++it1, ++it2) {
665 QString reasonerName(*it1);
666 QString reasonerPath(*it2);
667 LOG(INFO, getLogDomain(), logTr("Collecting response files for '%1' reasoner in directory '%2'.").arg(reasonerName).arg(reasonerPath), this);
668 collector->collectReasonerEvaluationDataValues(reasonerComp, reasonerName, reasonerPath);
669 }
670
671
672 LOG(INFO, getLogDomain(), logTr("Comparing result counting for '%1'.").arg(mReasonerNameStringList.join(", ")), this);
673
674 CReasonerEvaluationGroupRequestSelector* selectors = getSelectors(requestDirectory);
675
676
677 CReasonerEvaluationGroupRequestReasonerResultCountingAnalyser* analyser1 = new CReasonerEvaluationGroupRequestReasonerResultCountingAnalyser();
678 analyser1->analyseEvaluationData(reasonerComp, reasonerEvaluationContext, outputDirectory, selectors);
679
680
681 LOG(INFO, getLogDomain(), logTr("Result counting for '%1' compared to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory), this);
682 analysingUpdateChecker.saveAnalysingUpdateCheckFile(outputDirectory + "AnalysingUpdateCheckingData.dat");
683 analysed = true;
684 } else {
685 LOG(WARNING, getLogDomain(), logTr("Result counting for '%1' in '%2' already up to date.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory), this);
686 }
687 }
688
689
690
691
692
693
694
695 if (ananysingString == "ReasonerEvaluationErrorTimeoutComparer") {
696
697 QString outputDirectory = getAnalyserOutputDirectory(analysingDirectoryString,evaluationProgramName,plattform,"ErrorTimeoutComparison");
698
699 CReasonerEvaluationAnalyserChecker analysingUpdateChecker;
700 if (analysingUpdateChecker.checkAnalysingUpdateNecessary(mReasonerOutputDirStringList,outputDirectory+"AnalysingUpdateCheckingData.dat",testCountCut)) {
701
702 LOG(INFO,getLogDomain(),logTr("Analysing errors and timeouts for '%1' to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
703
704 CReasonerEvaluationSpecifiedTimeExtractor* errorExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::TIMEOUTERROREXTRACTOR,timeoutCut,errorPunishmentTime,dataValueCacher);
705
706 CReasonerEvaluationAvaragerSummarizer* avarageSummarize = new CReasonerEvaluationAvaragerSummarizer();
707
708 CReasonerEvaluationAnalyseContext* reasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig);
709
710 CReasonerEvaluationCollector* collector = new CReasonerEvaluationCollector(errorExtractor,avarageSummarize,mFiltering);
711
712 CReasonerEvaluationDataValueGroupCollectionReasonerComparison* reasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
713
714 for (QStringList::const_iterator it1 = mReasonerNameStringList.constBegin(), it2 = mReasonerOutputDirStringList.constBegin(), it1End = mReasonerNameStringList.constEnd(), it2End = mReasonerOutputDirStringList.constEnd(); it1 != it1End && it2 != it2End; ++it1, ++it2) {
715 QString reasonerName(*it1);
716 QString reasonerPath(*it2);
717 LOG(INFO,getLogDomain(),logTr("Collecting errors/timeouts for '%1' reasoner in directory '%2'.").arg(reasonerName).arg(reasonerPath),this);
718 collector->collectReasonerEvaluationDataValues(reasonerComp,reasonerName,reasonerPath);
719 }
720
721
722 LOG(INFO,getLogDomain(),logTr("Comparing errors/timeouts for '%1'.").arg(mReasonerNameStringList.join(", ")),this);
723
724 CReasonerEvaluationGroupRequestSelector* selectors = getSelectors(requestDirectory);
725
726
727 CReasonerEvaluationGroupRequestReasonerErrorAnalyser* analyser1 = new CReasonerEvaluationGroupRequestReasonerErrorAnalyser();
728 analyser1->analyseEvaluationData(reasonerComp,reasonerEvaluationContext,outputDirectory,selectors);
729
730
731 LOG(INFO,getLogDomain(),logTr("Errors/timeouts for '%1' compared to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
732 analysingUpdateChecker.saveAnalysingUpdateCheckFile(outputDirectory+"AnalysingUpdateCheckingData.dat");
733 analysed = true;
734 } else {
735 LOG(WARNING,getLogDomain(),logTr("Errors/timeouts for '%1' in '%2' already up to date.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory),this);
736 }
737 }
738
739
740
741 if (ananysingString == "ReasonerEvaluationErrorComparer") {
742
743 QString outputDirectory = getAnalyserOutputDirectory(analysingDirectoryString, evaluationProgramName, plattform, "ErrorComparison");
744
745 CReasonerEvaluationAnalyserChecker analysingUpdateChecker;
746 if (analysingUpdateChecker.checkAnalysingUpdateNecessary(mReasonerOutputDirStringList, outputDirectory + "AnalysingUpdateCheckingData.dat", testCountCut)) {
747
748 LOG(INFO, getLogDomain(), logTr("Analysing errors for '%1' to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory), this);
749
750 CReasonerEvaluationSpecifiedTimeExtractor* errorExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::ERROREXTRACTOR, timeoutCut, errorPunishmentTime, dataValueCacher);
751
752 CReasonerEvaluationAvaragerSummarizer* avarageSummarize = new CReasonerEvaluationAvaragerSummarizer();
753
754 CReasonerEvaluationAnalyseContext* reasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig);
755
756 CReasonerEvaluationCollector* collector = new CReasonerEvaluationCollector(errorExtractor, avarageSummarize, mFiltering);
757
758 CReasonerEvaluationDataValueGroupCollectionReasonerComparison* reasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
759
760 for (QStringList::const_iterator it1 = mReasonerNameStringList.constBegin(), it2 = mReasonerOutputDirStringList.constBegin(), it1End = mReasonerNameStringList.constEnd(), it2End = mReasonerOutputDirStringList.constEnd(); it1 != it1End && it2 != it2End; ++it1, ++it2) {
761 QString reasonerName(*it1);
762 QString reasonerPath(*it2);
763 LOG(INFO, getLogDomain(), logTr("Collecting errors for '%1' reasoner in directory '%2'.").arg(reasonerName).arg(reasonerPath), this);
764 collector->collectReasonerEvaluationDataValues(reasonerComp, reasonerName, reasonerPath);
765 }
766
767
768 LOG(INFO, getLogDomain(), logTr("Comparing errors for '%1'.").arg(mReasonerNameStringList.join(", ")), this);
769
770 CReasonerEvaluationGroupRequestSelector* selectors = getSelectors(requestDirectory);
771
772
773 CReasonerEvaluationGroupRequestReasonerErrorAnalyser* analyser1 = new CReasonerEvaluationGroupRequestReasonerErrorAnalyser();
774 analyser1->analyseEvaluationData(reasonerComp, reasonerEvaluationContext, outputDirectory, selectors);
775
776
777 LOG(INFO, getLogDomain(), logTr("Errors for '%1' compared to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory), this);
778 analysingUpdateChecker.saveAnalysingUpdateCheckFile(outputDirectory + "AnalysingUpdateCheckingData.dat");
779 analysed = true;
780 } else {
781 LOG(WARNING, getLogDomain(), logTr("Errors for '%1' in '%2' already up to date.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory), this);
782 }
783 }
784
785
786
787
788 if (ananysingString == "ReasonerEvaluationMemoryUsageComparer") {
789
790 QString outputDirectory = getAnalyserOutputDirectory(analysingDirectoryString, evaluationProgramName, plattform, "MemoryUsageComparison");
791
792 CReasonerEvaluationAnalyserChecker analysingUpdateChecker;
793 if (analysingUpdateChecker.checkAnalysingUpdateNecessary(mReasonerOutputDirStringList, outputDirectory + "AnalysingUpdateCheckingData.dat", testCountCut)) {
794
795 LOG(INFO, getLogDomain(), logTr("Analysing memory usage for '%1' to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory), this);
796
797 CReasonerEvaluationSpecifiedTimeExtractor* memoryUsageaExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::MEMORYUSAGEEXTRACTOR, timeoutCut, errorPunishmentTime, dataValueCacher);
798
799 CReasonerEvaluationAvaragerSummarizer* avarageSummarize = new CReasonerEvaluationAvaragerSummarizer();
800
801 CReasonerEvaluationAnalyseContext* reasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig);
802
803 CReasonerEvaluationCollector* collector = new CReasonerEvaluationCollector(memoryUsageaExtractor, avarageSummarize, mFiltering);
804
805 CReasonerEvaluationDataValueGroupCollectionReasonerComparison* reasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
806
807 for (QStringList::const_iterator it1 = mReasonerNameStringList.constBegin(), it2 = mReasonerOutputDirStringList.constBegin(), it1End = mReasonerNameStringList.constEnd(), it2End = mReasonerOutputDirStringList.constEnd(); it1 != it1End && it2 != it2End; ++it1, ++it2) {
808 QString reasonerName(*it1);
809 QString reasonerPath(*it2);
810 LOG(INFO, getLogDomain(), logTr("Collecting memory usage for '%1' reasoner in directory '%2'.").arg(reasonerName).arg(reasonerPath), this);
811 collector->collectReasonerEvaluationDataValues(reasonerComp, reasonerName, reasonerPath);
812 }
813
814
815 LOG(INFO, getLogDomain(), logTr("Comparing memory usage for '%1'.").arg(mReasonerNameStringList.join(", ")), this);
816
817 CReasonerEvaluationGroupRequestSelector* selectors = getSelectors(requestDirectory);
818
819
820 CReasonerEvaluationGroupRequestReasonerMemoryUsageAnalyser* analyser1 = new CReasonerEvaluationGroupRequestReasonerMemoryUsageAnalyser();
821 analyser1->analyseEvaluationData(reasonerComp, reasonerEvaluationContext, outputDirectory, selectors);
822
823
824 LOG(INFO, getLogDomain(), logTr("Memory usage for '%1' compared to '%2'.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory), this);
825 analysingUpdateChecker.saveAnalysingUpdateCheckFile(outputDirectory + "AnalysingUpdateCheckingData.dat");
826 analysed = true;
827 } else {
828 LOG(WARNING, getLogDomain(), logTr("Memory usage for '%1' in '%2' already up to date.").arg(mReasonerNameStringList.join(", ")).arg(outputDirectory), this);
829 }
830 }
831
832
833
834
835 if (ananysingString == "RequestStatisticsComparer") {
836
837 QString outputDirectory = getAnalyserOutputDirectory(analysingDirectoryString,evaluationProgramName,plattform,"RequestStatisticsComparison");
838
839 CReasonerEvaluationAnalyserChecker analysingUpdateChecker;
840 if (analysingUpdateChecker.checkAnalysingUpdateNecessary(mReasonerOutputDirStringList,outputDirectory+"AnalysingUpdateCheckingData.dat",testCountCut)) {
841
842 LOG(INFO,getLogDomain(),logTr("Analysing request files of '%1'.").arg(outputDirectory),this);
843
844 CReasonerEvaluationSpecifiedTimeExtractor* requestFileExtractor = new CReasonerEvaluationSpecifiedTimeExtractor(CReasonerEvaluationExtractor::REQUESTFILEEXTRACTOR,timeoutCut,errorPunishmentTime,dataValueCacher);
845
846 CReasonerEvaluationResponseFileListSummarizer* respFileListSummarizer = new CReasonerEvaluationResponseFileListSummarizer();
847
848 CReasonerEvaluationAnalyseContext* reasonerEvaluationContext = new CReasonerEvaluationAnalyseContext(mConfig);
849
850 CReasonerEvaluationCollector* collector = new CReasonerEvaluationCollector(requestFileExtractor,respFileListSummarizer,mFiltering);
851
852 CReasonerEvaluationDataValueGroupCollectionReasonerComparison* reasonerComp = new CReasonerEvaluationDataValueGroupCollectionReasonerComparison(mReasonerNameStringList);
853
854 for (QStringList::const_iterator it1 = mReasonerNameStringList.constBegin(), it2 = mReasonerOutputDirStringList.constBegin(), it1End = mReasonerNameStringList.constEnd(), it2End = mReasonerOutputDirStringList.constEnd(); it1 != it1End && it2 != it2End; ++it1, ++it2) {
855 QString reasonerName(*it1);
856 QString reasonerPath(*it2);
857 LOG(INFO,getLogDomain(),logTr("Collecting request files for '%1' reasoner in directory '%2'.").arg(reasonerName).arg(reasonerPath),this);
858 collector->collectReasonerEvaluationDataValues(reasonerComp,reasonerName,reasonerPath);
859 }
860
861 CReasonerEvaluationGroupRequestSelector* selectors = getSelectors(requestDirectory);
862
863 CReasonerEvaluationGroupRequestStatisticsCSVComparisonAnalyser* analyser1 = new CReasonerEvaluationGroupRequestStatisticsCSVComparisonAnalyser(mFiltering);
864 analyser1->analyseEvaluationData(reasonerComp,reasonerEvaluationContext,outputDirectory,selectors);
865
866
867 LOG(INFO,getLogDomain(),logTr("Request statistics evaluated to '%1'.").arg(outputDirectory),this);
868 analysingUpdateChecker.saveAnalysingUpdateCheckFile(outputDirectory+"AnalysingUpdateCheckingData.dat");
869 analysed = true;
870 } else {
871 LOG(WARNING,getLogDomain(),logTr("Request statistics evaluation of '%1' already up to date.").arg(outputDirectory),this);
872 }
873 }
874
875
876
877
878
879 cint64 elapsedTime = timer.elapsed();
880 LOG(INFO,getLogDomain(),logTr("Finished analysing for '%1' in '%2' seconds.").arg(ananysingString).arg(elapsedTime/1000),this);
881
882
883 }
884
885 cint64 totalElapsedTime = totalTimer.elapsed();
886 LOG(INFO,getLogDomain(),logTr("Finished analysing in '%1' seconds.").arg(totalElapsedTime/1000),this);
887 } else {
888 LOG(ERROR,getLogDomain(),logTr("Analysing not possible, platform not specified."),this);
889 }
890
891
892
893
894 if (analysed) {
895 LOG(INFO,getLogDomain(),logTr("Creation HTML navigation overview site."),this);
896 CReasonerEvaluationHTMLNavigationOverviewSummarizer* htmlNavOvSumm = new CReasonerEvaluationHTMLNavigationOverviewSummarizer();
897 QString outputDirectory = getAnalyserOutputDirectory(analysingDirectoryString,evaluationProgramName,plattform,".");
898 htmlNavOvSumm->createHTMLNavigationOverview(outputDirectory,evaluationProgramName,mReasonerNameStringList,mFitlerStringList,mTestTimeout,mTestCount);
899 delete htmlNavOvSumm;
900 }
901 }
902
903 if (!mFailedReasonerStringList.isEmpty()) {
904 LOG(WARNING,getLogDomain(),logTr("Evaluation failed for the following reasoner configuration files:\r\n%1.").arg(mFailedReasonerStringList.join(",\r\n")),this);
905 }
906
907 return this;
908 }
909
910
911
912 }; // end namespace Evaluation
913
914 }; // end namespace Test
915
916 }; // end namespace Konclude