Debian Codegraph @

Source/Test/Evaluation/CReasonerEvaluationRequestClientSPARQLThread.cpp

raw
1 /*
2 * Copyright (C) 2013-2015, 2019 by the Konclude Developer Team.
3 *
4 * This file is part of the reasoning system Konclude.
5 * For details and support, see <http://konclude.com/>.
6 *
7 * Konclude is free software: you can redistribute it and/or modify
8 * it under the terms of version 3 of the GNU Lesser General Public
9 * License (LGPLv3) as published by the Free Software Foundation.
10 *
11 * Konclude is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU (Lesser) General Public License for more details.
15 *
16 * You should have received a copy of the GNU (Lesser) General Public
17 * License along with Konclude. If not, see <http://www.gnu.org/licenses/>.
18 *
19 */
20
21 #include "CReasonerEvaluationRequestClientSPARQLThread.h"
22
23
24 namespace Konclude {
25
26 namespace Test {
27
28 namespace Evaluation {
29
30
31
32 CReasonerEvaluationRequestClientSPARQLThread::CReasonerEvaluationRequestClientSPARQLThread() : CReasonerEvaluationRequestClientBaseThread("Reasoner-Evaluation-SPARQL-Request-Client", "::Konclude::Test::ReasonerEvaluationSPARQLRequestClient") {
33 mSPARQLOperationKeywordNameSet.insert("LOAD", "GraphLoad");
34 mSPARQLOperationKeywordNameSet.insert("SELECT", "SelectQuery");
35 mSPARQLOperationKeywordNameSet.insert("ASK", "AskQuery");
36 mSPARQLOperationKeywordNameSet.insert("PREPARE", "GraphLoad");
37
38 mConfSendPrepareBeforeQuerying = false;
39 mConfSupportsPrepareCommand = false;
40
41 mConfResponseSizeLimit = -1;
42
43 mRemainingRestartRepeatingRequests = 0;
44 }
45
46
47 CReasonerEvaluationRequestClientSPARQLThread::~CReasonerEvaluationRequestClientSPARQLThread() {
48 }
49
50
51 bool CReasonerEvaluationRequestClientSPARQLThread::canHandleRequestFile(const QString& testFileString) {
52 if (testFileString.endsWith(".sparql")) {
53 return true;
54 }
55 return false;
56 }
57
58
59 bool CReasonerEvaluationRequestClientSPARQLThread::processTimer(qint64 timerID) {
60 if (timerID == TIMERIDTOTALTIMEOUT) {
61 ++mCurrentOperationNumber;
62 closeAllRequestTimeout();
63 return true;
64 } else if (timerID == TIMERIDREQUESTTIMEOUT) {
65 ++mCurrentOperationNumber;
66 CReasonerEvaluationRequestResponseSPARQL* sprqlReqRes = (CReasonerEvaluationRequestResponseSPARQL*)mNextReqRes;
67 if (sprqlReqRes->getOperationMode() != "QUERY" || mNumberInitRequests > 0 || !mConfQueryTimeoutRestartRepeatPreparation || !hasNextRequest()) {
68 closeAllRequestTimeout();
69 } else {
70 closeRequestTimeoutContinueRestart();
71 }
72 return true;
73 }
74 return false;
75 }
76
77
78
79
80 bool CReasonerEvaluationRequestClientSPARQLThread::closeRequestTimeoutContinueRestart() {
81 cint64 elapsedTime = mNextRequestTiming.elapsed();
82 mNextReqRes->setResponseTime(elapsedTime);
83
84 LOG(INFO, getLogDomain(), logTr("Request %1 of %2 timed out, trying restarting reasoner and continuing.").arg(1 + mCurrentRequestCount++).arg(mTotalRequestCount), this);
85
86 mNextReqRes->setTimedOut(true);
87 mResponse->setResponsedInTime(false);
88 mResponse->addResponse(mNextReqRes, mNumberInitRequests-- > 0);
89 mRestartRepeatingRequests = 0;
90
91 cint64 requestNumber = 0;
92 for (CReasonerEvaluationRequestResponse* request : mAllRequestList) {
93 if (request == mNextReqRes) {
94 break;
95 }
96 CReasonerEvaluationRequestResponseSPARQL* sprqlReqRes = (CReasonerEvaluationRequestResponseSPARQL*)request;
97 if (sprqlReqRes->getOperationMode() != "QUERY" || requestNumber <= mTotalInitRequests) {
98 mRemainingRequestList.prepend(request);
99 ++mRestartRepeatingRequests;
100 } else {
101 break;
102 }
103 ++requestNumber;
104 }
105 mNumberInitRequests = mTotalInitRequests;
106 mTotalRequestCount = mRemainingRequestList.count();
107 mCurrentRequestCount = 0;
108
109 if (hasNextRequest()) {
110 if (!mReasonerProvider->restartReasoner()) {
111 closeAllRequestTimeout();
112 finishReasonerRequests();
113 return false;
114 } else {
115 LOG(INFO, getLogDomain(), logTr("Reasoner restarted, continuing evaluation."), this);
116 }
117 sendNextRequest();
118 return true;
119 } else {
120 finishReasonerRequests();
121 return false;
122 }
123 }
124
125
126
127 bool CReasonerEvaluationRequestClientSPARQLThread::closeAllRequestTimeout() {
128 cint64 elapsedTime = mNextRequestTiming.elapsed();
129
130 LOG(INFO, getLogDomain(), logTr("Request %1 of %2 timed out.").arg(1 + mCurrentRequestCount++).arg(mTotalRequestCount), this);
131
132 if (mRestartRepeatingRequests <= 0) {
133 mNextReqRes->setResponseTime(elapsedTime);
134 mNextReqRes->setTimedOut(true);
135 mResponse->setResponsedInTime(false);
136 mResponse->addResponse(mNextReqRes, mNumberInitRequests-- > 0);
137 } else {
138 --mRestartRepeatingRequests;
139 }
140
141 while (!mRemainingRequestList.isEmpty()) {
142 CReasonerEvaluationRequestResponse* nextReqResp = mRemainingRequestList.takeFirst();
143 if (mRestartRepeatingRequests <= 0) {
144 nextReqResp->setResponseError(true);
145 mResponse->addResponse(nextReqResp, mNumberInitRequests-- > 0);
146 } else {
147 --mRestartRepeatingRequests;
148 }
149 }
150
151 finishReasonerRequests();
152 return true;
153 }
154
155
156
157
158 bool CReasonerEvaluationRequestClientSPARQLThread::closeNextRequest() {
159 stopTimer(TIMERIDREQUESTTIMEOUT);
160
161 cint64 elapsedTime = mNextRequestTiming.elapsed();
162 bool responseError = false;
163
164 QIODevice* device = nullptr;
165
166 if (mRestartRepeatingRequests <= 0) {
167 LOG(INFO, getLogDomain(), logTr("Received response %1 of %2 in %3 ms.").arg(1 + mCurrentRequestCount++).arg(mTotalRequestCount).arg(elapsedTime), this);
168 mNextReqRes->setResponseTime(elapsedTime);
169 if (!mTransManager->hasFinishedSucecssfully(mWebResponse)) {
170 CHttpTransactionManager::ABORT_REASON abortReason = CHttpTransactionManager::ABORT_NO_REASON;
171 if (mTransManager->hasBeenAborted(mWebResponse, &abortReason)) {
172 mNextReqRes->setEvaluationError(true);
173 QString abortReasonString = "none given";
174 if (abortReason == CHttpTransactionManager::ABORT_DOWNLOAD_SIZE_LIMIT_REACHED) {
175 abortReasonString = QString("Response size limit of %1 bytes has been reached").arg(mDownloadSizeLimit);
176 }
177 mNextReqRes->setEvaluationErrorString(QString("Request aborted with reason: %1.").arg(abortReasonString));
178 LOG(WARN, getLogDomain(), logTr("Request has been aborted with reason: %1.").arg(abortReasonString), this);
179 } else {
180 responseError = true;
181 LOG(WARN, getLogDomain(), logTr("Response not successfully retrieved, interpreting it as error."), this);
182 }
183 } else {
184 device = mTransManager->getResponseDataReadDevice(mWebResponse);
185 if (device) {
186 responseError = parseResponse(device);
187 } else {
188 responseError = true;
189 }
190 }
191
192
193 mNextReqRes->setResponseError(responseError);
194 if (!responseError) {
195 mResponse->setSucessfullReasonerCommunication(true);
196 }
197
198 mResponse->addResponse(mNextReqRes, mNumberInitRequests-- > 0);
199 } else {
200 LOG(INFO, getLogDomain(), logTr("Received (restart repeated) response %1 of %2 in %3 ms.").arg(1 + mCurrentRequestCount++).arg(mTotalRequestCount).arg(elapsedTime), this);
201 mRestartRepeatingRequests--;
202 if (mTransManager->hasFinishedSucecssfully(mWebResponse)) {
203 device = mTransManager->getResponseDataReadDevice(mWebResponse);
204 device->readAll();
205 }
206 }
207
208
209 mNextReqRes = nullptr;
210
211 mTransManager->releaseResponse(mWebResponse);
212 return true;
213 }
214
215
216 CReasonerEvaluationRequestClientBaseThread* CReasonerEvaluationRequestClientSPARQLThread::readConfig(CConfiguration* config) {
217 CReasonerEvaluationRequestClientBaseThread::readConfig(config);
218
219 mConfSendPrepareBeforeQuerying = CConfigDataReader::readConfigBoolean(config, "Konclude.Evaluation.SPARQLSendPrepareBeforeQuerying");
220 mConfSupportsPrepareCommand = CConfigDataReader::readConfigBoolean(config, "Konclude.Evaluation.SPARQLSupportsPrepareOperation");
221
222 mConfResponseSizeLimit = CConfigDataReader::readConfigInteger(config, "Konclude.Evaluation.SPARQLResponseSizeLimit", -1);
223
224 mConfQueryTimeoutRestartRepeatPreparation = CConfigDataReader::readConfigBoolean(config, "Konclude.Evaluation.SPARQLQueryTimeoutRestartRepeatPreparation");
225 mConfSeparateQueryingTimeout = CConfigDataReader::readConfigInteger(config, "Konclude.Evaluation.SPARQLQueryingTimeout", -1);
226 return this;
227 }
228
229
230
231 bool CReasonerEvaluationRequestClientSPARQLThread::parseResponse(QIODevice* device) {
232 bool sparqlQuery = ((CReasonerEvaluationRequestResponseSPARQL*)mNextReqRes)->getRequestOperationData()->getOperationType() == CSPARQLKnowledgeBaseSplittingOperationData::SPARQL_QUERY;
233 bool responseError = false;
234 QDomDocument document;
235 QString errorMessage;
236 int errorLine = 0;
237 int errorColumn = 0;
238
239 cint64 size = device->size();
240 if (mConfResponseSizeLimit != -1 && size > mConfResponseSizeLimit) {
241 ((CReasonerEvaluationRequestResponseSPARQL*)mNextReqRes)->setEvaluationError(true);
242 ((CReasonerEvaluationRequestResponseSPARQL*)mNextReqRes)->setEvaluationErrorString(QString("Response with %1 bytes exceeds limit of %2 bytes").arg(size).arg(mConfResponseSizeLimit));
243 LOG(WARN, getLogDomain(), logTr("Response with %1 bytes exceeds limit of %2 bytes, skipping response analysis.").arg(size).arg(mConfResponseSizeLimit), this);
244 return false;
245 }
246
247 if (document.setContent(device, &errorMessage, &errorLine, &errorColumn)) {
248
249 QDomElement rootNode = document.documentElement();
250 QDomElement responseNode = rootNode.firstChildElement();
251 while (!responseNode.isNull()) {
252 ((CReasonerEvaluationRequestResponseSPARQL*)mNextReqRes)->addResponseNode(responseNode);
253 responseNode = responseNode.nextSiblingElement();
254 }
255
256 if (rootNode.isNull() && sparqlQuery) {
257 responseError = true;
258 }
259
260 } else if (sparqlQuery) {
261 responseError = true;
262 if (!errorMessage.isEmpty()) {
263 device->reset();
264 cint64 available = device->bytesAvailable();
265 QString responsePart = device->read(qMin((int)available, 10000));
266 ((CReasonerEvaluationRequestResponseSPARQL*)mNextReqRes)->setResponseErrorString(QString("Parsing error (line %2, column %3) '%1' of response '%4...' .").arg(errorMessage).arg(errorLine).arg(errorColumn).arg(responsePart));
267 LOG(INFO, getLogDomain(), logTr("Parsing of response failed due to error (line %2, column %3): %1.").arg(errorMessage).arg(errorLine).arg(errorColumn), this);
268 }
269 }
270 return responseError;
271 }
272
273
274 bool CReasonerEvaluationRequestClientSPARQLThread::sendNextRequest() {
275 if (mRemainingRequestList.isEmpty()) {
276 return false;
277 }
278 mNextReqRes = mRemainingRequestList.takeFirst();
279 CReasonerEvaluationRequestResponseSPARQL* owllinkNextReqRes = (CReasonerEvaluationRequestResponseSPARQL*)mNextReqRes;
280
281 QString requestString = getAdaptedRequestNode(owllinkNextReqRes->getRequestString(), owllinkNextReqRes->getRequestOperationData());
282 owllinkNextReqRes->setModifiedNode(requestString);
283
284 QByteArray transReqByteArray(requestString.toUtf8());
285 QString operationName = owllinkNextReqRes->getOperationName();
286
287
288 if (mNumberInitRequests == 0) {
289 mTestingTiming.start();
290 }
291
292 mWebRequest = mTransManager->createRequest(QString("http://%1").arg(mAddressString), transReqByteArray);
293 mNextRequestTiming.start();
294 CReasonerEvaluationRequestResponseSPARQL* sprqlReqRes = (CReasonerEvaluationRequestResponseSPARQL*)mNextReqRes;
295 if (mConfSeparateQueryingTimeout != -1 && sprqlReqRes->getOperationMode() == "QUERY") {
296 startTimerWithIntervalLimited(TIMERIDREQUESTTIMEOUT, mConfSeparateQueryingTimeout, 1);
297 } else {
298 startTimerWithIntervalLimited(TIMERIDREQUESTTIMEOUT, mRequestTimeout, 1);
299 }
300
301 mWebResponse = mTransManager->getResponse(mWebRequest);
302 mTransManager->callbackFinishedRequest(mWebResponse,new CReasonerEvaluationNextEvent(this,mCurrentOperationNumber));
303 LOG(INFO,getLogDomain(),logTr("Sending SPARQL operation %1 of %2 with command '%3' to '%4'.").arg(mCurrentRequestCount+1).arg(mTotalRequestCount).arg(operationName).arg(mAddressString),this);
304 return true;
305 }
306
307
308
309
310 QString CReasonerEvaluationRequestClientSPARQLThread::getAdaptedRequestNode(const QString& reqeustString, CSPARQLKnowledgeBaseSplittingOperationData* operationData) {
311 QStringList partStringList(operationData->getOperationPartStringList());
312 if (operationData->getOperationType() == CSPARQLKnowledgeBaseSplittingOperationData::SPARQL_UPDATE_MANAGE) {
313 QStringList modifiedPartStringList;
314 while (!partStringList.isEmpty()) {
315 const QString& part = partStringList.takeFirst();
316 if (!part.startsWith("#")) {
317 modifiedPartStringList.append(part);
318 QString keyword = part.toUpper();
319 if (keyword == "LOAD") {
320 bool loadingIRIPartFound = false;
321 while (!partStringList.isEmpty() && !loadingIRIPartFound) {
322 const QString& loadingIRIPart = partStringList.takeFirst();
323 if (loadingIRIPart.toUpper() != "SILENT") {
324 loadingIRIPartFound = true;
325 QString iriString = getIRI(loadingIRIPart);
326
327 if (iriString.startsWith("file:") && !iriString.startsWith("file://")) {
328 QString relFileString = iriString;
329 relFileString = relFileString.remove(0, 5);
330 QDir currFilePath(QDir::currentPath());
331 QString absFileString = currFilePath.absoluteFilePath(relFileString);
332 absFileString = QUrl::fromPercentEncoding(QUrl::fromLocalFile(absFileString).toString().toUtf8());
333
334 modifiedPartStringList.append(QString("<%1>").arg(absFileString));
335 LOG(INFO, getLogDomain(), logTr("Changed relative filepath '%1' to absolute filepath '%2'.").arg(iriString).arg(absFileString), this);
336 } else {
337 modifiedPartStringList.append(loadingIRIPart);
338 }
339 } else {
340 modifiedPartStringList.append(loadingIRIPart);
341 }
342 }
343 } else if (keyword == "PREFIX" || keyword == "@PREFIX") {
344 if (partStringList.size() >= 2) {
345 QString prefix = partStringList.takeFirst();
346 modifiedPartStringList.append(prefix);
347 QString resolvedPrefix = prefix;
348 while (resolvedPrefix.endsWith(":")) {
349 resolvedPrefix = resolvedPrefix.mid(0, resolvedPrefix.length() - 1);
350 }
351 QString iriString = partStringList.takeFirst();
352 modifiedPartStringList.append(iriString);
353 QString iri = getIRI(iriString);
354 mPrefixHash.insert(resolvedPrefix, iri);
355 }
356 }
357 }
358
359 }
360 return modifiedPartStringList.join(" ");
361 } else {
362 return reqeustString;
363 }
364 }
365
366
367 bool CReasonerEvaluationRequestClientSPARQLThread::loadReasonerRequests(const QString& initFileString, const QString& testFileString) {
368 mCurrentRequestCount = 0;
369 mTotalRequestCount = 0;
370 mRemainingRestartRepeatingRequests = 0;
371 mRestartRepeatingRequests = 0;
372
373 bool loaded = loadReasonerRequest(initFileString);
374 mTotalInitRequests = mNumberInitRequests = mRemainingRequestList.count();
375 loaded &= loadReasonerRequest(testFileString);
376 return loaded;
377 }
378
379
380 QString CReasonerEvaluationRequestClientSPARQLThread::getIRI(const QString& iriString) {
381 if (iriString.startsWith("<") && iriString.endsWith(">")) {
382 return iriString.mid(1, iriString.length() - 2);
383 } else {
384 int colonPos = iriString.indexOf(':');
385 if (colonPos != -1) {
386 QString prefix = iriString.mid(0, colonPos);
387 QString prefixResolvedString = mPrefixHash.value(prefix);
388 if (!prefixResolvedString.isEmpty()) {
389 QString remainingIRI = iriString.mid(colonPos + 1);
390 if (!remainingIRI.isEmpty()) {
391 return prefixResolvedString + remainingIRI;
392 }
393 }
394 }
395
396 return mBaseIRI + iriString;
397 }
398 }
399
400
401 bool CReasonerEvaluationRequestClientSPARQLThread::loadReasonerRequest(const QString& requestFileString) {
402 CSPARQLKnowledgeBaseSplittingOperationParser parser;
403 parser.parseQueryFile(requestFileString);
404 mBaseIRI = requestFileString;
405 cint64 loadedRequestCount = 0;
406 QList<CSPARQLKnowledgeBaseSplittingOperationData*> operationDataList = parser.getSPARQLOperationList();
407 bool hasUpdatingOperation = false;
408 for (CSPARQLKnowledgeBaseSplittingOperationData* operationData : operationDataList) {
409 CReasonerEvaluationRequestResponseSPARQL* requestRespone = new CReasonerEvaluationRequestResponseSPARQL(*operationData);
410 QStringList partStringList(operationData->getOperationPartStringList());
411 requestRespone->setRequestNode(partStringList.join(" "));
412
413
414 bool queryingOperation = false;
415 bool updatingOperation = false;
416 QString operationMode;
417 QString operationName;
418 if (requestRespone->getRequestOperationData()->getOperationType() == CSPARQLKnowledgeBaseSplittingOperationData::SPARQL_QUERY) {
419 operationMode = "QUERY";
420 queryingOperation = true;
421 } else if (requestRespone->getRequestOperationData()->getOperationType() == CSPARQLKnowledgeBaseSplittingOperationData::SPARQL_UPDATE_MANAGE) {
422 operationMode = "UPDATE";
423 updatingOperation = true;
424 } else if (requestRespone->getRequestOperationData()->getOperationType() == CSPARQLKnowledgeBaseSplittingOperationData::SPARQL_UPDATE_MODIFY) {
425 operationMode = "UPDATE";
426 updatingOperation = true;
427 }
428 requestRespone->setOperationMode(operationMode);
429 for (const QString& partString : partStringList) {
430 if (mSPARQLOperationKeywordNameSet.contains(partString.toUpper())) {
431 operationName = mSPARQLOperationKeywordNameSet.value(partString.toUpper());
432 break;
433 }
434 }
435 if (!operationName.isEmpty()) {
436 requestRespone->setOperationName(operationName);
437 } else {
438 requestRespone->setOperationMode(operationMode);
439 }
440
441 if (mConfSendPrepareBeforeQuerying && mConfSupportsPrepareCommand) {
442 if (hasUpdatingOperation && queryingOperation) {
443 CSPARQLKnowledgeBaseSplittingOperationData prepareOperation(CSPARQLKnowledgeBaseSplittingOperationData::SPARQL_UPDATE_MANAGE, operationData->getKnowledgeBaseString(), QStringList() << "PREPARE" << "QUERYING" << operationData->getKnowledgeBaseString());
444 CReasonerEvaluationRequestResponseSPARQL* prepareRequestRespone = new CReasonerEvaluationRequestResponseSPARQL(prepareOperation);
445 prepareRequestRespone->setOperationMode("PREPARE");
446 prepareRequestRespone->setOperationName(mSPARQLOperationKeywordNameSet.value("PREPARE"));
447 mRemainingRequestList.append(prepareRequestRespone);
448 mAllRequestList.append(prepareRequestRespone);
449 ++mTotalRequestCount;
450 ++loadedRequestCount;
451 hasUpdatingOperation = false;
452 } else {
453 hasUpdatingOperation |= updatingOperation;
454 }
455 }
456
457
458 mRemainingRequestList.append(requestRespone);
459 mAllRequestList.append(requestRespone);
460
461
462 ++mTotalRequestCount;
463 ++loadedRequestCount;
464 }
465 LOG(INFO, getLogDomain(), logTr("Loaded %1 SPARQL operations from file '%2'.").arg(loadedRequestCount).arg(requestFileString), this);
466
467 return true;
468 }
469
470
471
472
473
474
475 }; // end namespace Evaluation
476
477 }; // end namespace Test
478
479 }; // end namespace Konclude