/* * Copyright (C) 2013-2015, 2019 by the Konclude Developer Team. * * This file is part of the reasoning system Konclude. * For details and support, see . * * Konclude is free software: you can redistribute it and/or modify * it under the terms of version 3 of the GNU Lesser General Public * License (LGPLv3) as published by the Free Software Foundation. * * Konclude is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU (Lesser) General Public License for more details. * * You should have received a copy of the GNU (Lesser) General Public * License along with Konclude. If not, see . * */ #include "CReasonerEvaluationRequestClientSPARQLThread.h" namespace Konclude { namespace Test { namespace Evaluation { CReasonerEvaluationRequestClientSPARQLThread::CReasonerEvaluationRequestClientSPARQLThread() : CReasonerEvaluationRequestClientBaseThread("Reasoner-Evaluation-SPARQL-Request-Client", "::Konclude::Test::ReasonerEvaluationSPARQLRequestClient") { mSPARQLOperationKeywordNameSet.insert("LOAD", "GraphLoad"); mSPARQLOperationKeywordNameSet.insert("SELECT", "SelectQuery"); mSPARQLOperationKeywordNameSet.insert("ASK", "AskQuery"); mSPARQLOperationKeywordNameSet.insert("PREPARE", "GraphLoad"); mConfSendPrepareBeforeQuerying = false; mConfSupportsPrepareCommand = false; mConfResponseSizeLimit = -1; mRemainingRestartRepeatingRequests = 0; } CReasonerEvaluationRequestClientSPARQLThread::~CReasonerEvaluationRequestClientSPARQLThread() { } bool CReasonerEvaluationRequestClientSPARQLThread::canHandleRequestFile(const QString& testFileString) { if (testFileString.endsWith(".sparql")) { return true; } return false; } bool CReasonerEvaluationRequestClientSPARQLThread::processTimer(qint64 timerID) { if (timerID == TIMERIDTOTALTIMEOUT) { ++mCurrentOperationNumber; closeAllRequestTimeout(); return true; } else if (timerID == TIMERIDREQUESTTIMEOUT) { ++mCurrentOperationNumber; CReasonerEvaluationRequestResponseSPARQL* sprqlReqRes = (CReasonerEvaluationRequestResponseSPARQL*)mNextReqRes; if (sprqlReqRes->getOperationMode() != "QUERY" || mNumberInitRequests > 0 || !mConfQueryTimeoutRestartRepeatPreparation || !hasNextRequest()) { closeAllRequestTimeout(); } else { closeRequestTimeoutContinueRestart(); } return true; } return false; } bool CReasonerEvaluationRequestClientSPARQLThread::closeRequestTimeoutContinueRestart() { cint64 elapsedTime = mNextRequestTiming.elapsed(); mNextReqRes->setResponseTime(elapsedTime); LOG(INFO, getLogDomain(), logTr("Request %1 of %2 timed out, trying restarting reasoner and continuing.").arg(1 + mCurrentRequestCount++).arg(mTotalRequestCount), this); mNextReqRes->setTimedOut(true); mResponse->setResponsedInTime(false); mResponse->addResponse(mNextReqRes, mNumberInitRequests-- > 0); mRestartRepeatingRequests = 0; cint64 requestNumber = 0; for (CReasonerEvaluationRequestResponse* request : mAllRequestList) { if (request == mNextReqRes) { break; } CReasonerEvaluationRequestResponseSPARQL* sprqlReqRes = (CReasonerEvaluationRequestResponseSPARQL*)request; if (sprqlReqRes->getOperationMode() != "QUERY" || requestNumber <= mTotalInitRequests) { mRemainingRequestList.prepend(request); ++mRestartRepeatingRequests; } else { break; } ++requestNumber; } mNumberInitRequests = mTotalInitRequests; mTotalRequestCount = mRemainingRequestList.count(); mCurrentRequestCount = 0; if (hasNextRequest()) { if (!mReasonerProvider->restartReasoner()) { closeAllRequestTimeout(); finishReasonerRequests(); return false; } else { LOG(INFO, getLogDomain(), logTr("Reasoner restarted, continuing evaluation."), this); } sendNextRequest(); return true; } else { finishReasonerRequests(); return false; } } bool CReasonerEvaluationRequestClientSPARQLThread::closeAllRequestTimeout() { cint64 elapsedTime = mNextRequestTiming.elapsed(); LOG(INFO, getLogDomain(), logTr("Request %1 of %2 timed out.").arg(1 + mCurrentRequestCount++).arg(mTotalRequestCount), this); if (mRestartRepeatingRequests <= 0) { mNextReqRes->setResponseTime(elapsedTime); mNextReqRes->setTimedOut(true); mResponse->setResponsedInTime(false); mResponse->addResponse(mNextReqRes, mNumberInitRequests-- > 0); } else { --mRestartRepeatingRequests; } while (!mRemainingRequestList.isEmpty()) { CReasonerEvaluationRequestResponse* nextReqResp = mRemainingRequestList.takeFirst(); if (mRestartRepeatingRequests <= 0) { nextReqResp->setResponseError(true); mResponse->addResponse(nextReqResp, mNumberInitRequests-- > 0); } else { --mRestartRepeatingRequests; } } finishReasonerRequests(); return true; } bool CReasonerEvaluationRequestClientSPARQLThread::closeNextRequest() { stopTimer(TIMERIDREQUESTTIMEOUT); cint64 elapsedTime = mNextRequestTiming.elapsed(); bool responseError = false; QIODevice* device = nullptr; if (mRestartRepeatingRequests <= 0) { LOG(INFO, getLogDomain(), logTr("Received response %1 of %2 in %3 ms.").arg(1 + mCurrentRequestCount++).arg(mTotalRequestCount).arg(elapsedTime), this); mNextReqRes->setResponseTime(elapsedTime); if (!mTransManager->hasFinishedSucecssfully(mWebResponse)) { CHttpTransactionManager::ABORT_REASON abortReason = CHttpTransactionManager::ABORT_NO_REASON; if (mTransManager->hasBeenAborted(mWebResponse, &abortReason)) { mNextReqRes->setEvaluationError(true); QString abortReasonString = "none given"; if (abortReason == CHttpTransactionManager::ABORT_DOWNLOAD_SIZE_LIMIT_REACHED) { abortReasonString = QString("Response size limit of %1 bytes has been reached").arg(mDownloadSizeLimit); } mNextReqRes->setEvaluationErrorString(QString("Request aborted with reason: %1.").arg(abortReasonString)); LOG(WARN, getLogDomain(), logTr("Request has been aborted with reason: %1.").arg(abortReasonString), this); } else { responseError = true; LOG(WARN, getLogDomain(), logTr("Response not successfully retrieved, interpreting it as error."), this); } } else { device = mTransManager->getResponseDataReadDevice(mWebResponse); if (device) { responseError = parseResponse(device); } else { responseError = true; } } mNextReqRes->setResponseError(responseError); if (!responseError) { mResponse->setSucessfullReasonerCommunication(true); } mResponse->addResponse(mNextReqRes, mNumberInitRequests-- > 0); } else { LOG(INFO, getLogDomain(), logTr("Received (restart repeated) response %1 of %2 in %3 ms.").arg(1 + mCurrentRequestCount++).arg(mTotalRequestCount).arg(elapsedTime), this); mRestartRepeatingRequests--; if (mTransManager->hasFinishedSucecssfully(mWebResponse)) { device = mTransManager->getResponseDataReadDevice(mWebResponse); device->readAll(); } } mNextReqRes = nullptr; mTransManager->releaseResponse(mWebResponse); return true; } CReasonerEvaluationRequestClientBaseThread* CReasonerEvaluationRequestClientSPARQLThread::readConfig(CConfiguration* config) { CReasonerEvaluationRequestClientBaseThread::readConfig(config); mConfSendPrepareBeforeQuerying = CConfigDataReader::readConfigBoolean(config, "Konclude.Evaluation.SPARQLSendPrepareBeforeQuerying"); mConfSupportsPrepareCommand = CConfigDataReader::readConfigBoolean(config, "Konclude.Evaluation.SPARQLSupportsPrepareOperation"); mConfResponseSizeLimit = CConfigDataReader::readConfigInteger(config, "Konclude.Evaluation.SPARQLResponseSizeLimit", -1); mConfQueryTimeoutRestartRepeatPreparation = CConfigDataReader::readConfigBoolean(config, "Konclude.Evaluation.SPARQLQueryTimeoutRestartRepeatPreparation"); mConfSeparateQueryingTimeout = CConfigDataReader::readConfigInteger(config, "Konclude.Evaluation.SPARQLQueryingTimeout", -1); return this; } bool CReasonerEvaluationRequestClientSPARQLThread::parseResponse(QIODevice* device) { bool sparqlQuery = ((CReasonerEvaluationRequestResponseSPARQL*)mNextReqRes)->getRequestOperationData()->getOperationType() == CSPARQLKnowledgeBaseSplittingOperationData::SPARQL_QUERY; bool responseError = false; QDomDocument document; QString errorMessage; int errorLine = 0; int errorColumn = 0; cint64 size = device->size(); if (mConfResponseSizeLimit != -1 && size > mConfResponseSizeLimit) { ((CReasonerEvaluationRequestResponseSPARQL*)mNextReqRes)->setEvaluationError(true); ((CReasonerEvaluationRequestResponseSPARQL*)mNextReqRes)->setEvaluationErrorString(QString("Response with %1 bytes exceeds limit of %2 bytes").arg(size).arg(mConfResponseSizeLimit)); LOG(WARN, getLogDomain(), logTr("Response with %1 bytes exceeds limit of %2 bytes, skipping response analysis.").arg(size).arg(mConfResponseSizeLimit), this); return false; } if (document.setContent(device, &errorMessage, &errorLine, &errorColumn)) { QDomElement rootNode = document.documentElement(); QDomElement responseNode = rootNode.firstChildElement(); while (!responseNode.isNull()) { ((CReasonerEvaluationRequestResponseSPARQL*)mNextReqRes)->addResponseNode(responseNode); responseNode = responseNode.nextSiblingElement(); } if (rootNode.isNull() && sparqlQuery) { responseError = true; } } else if (sparqlQuery) { responseError = true; if (!errorMessage.isEmpty()) { device->reset(); cint64 available = device->bytesAvailable(); QString responsePart = device->read(qMin((int)available, 10000)); ((CReasonerEvaluationRequestResponseSPARQL*)mNextReqRes)->setResponseErrorString(QString("Parsing error (line %2, column %3) '%1' of response '%4...' .").arg(errorMessage).arg(errorLine).arg(errorColumn).arg(responsePart)); LOG(INFO, getLogDomain(), logTr("Parsing of response failed due to error (line %2, column %3): %1.").arg(errorMessage).arg(errorLine).arg(errorColumn), this); } } return responseError; } bool CReasonerEvaluationRequestClientSPARQLThread::sendNextRequest() { if (mRemainingRequestList.isEmpty()) { return false; } mNextReqRes = mRemainingRequestList.takeFirst(); CReasonerEvaluationRequestResponseSPARQL* owllinkNextReqRes = (CReasonerEvaluationRequestResponseSPARQL*)mNextReqRes; QString requestString = getAdaptedRequestNode(owllinkNextReqRes->getRequestString(), owllinkNextReqRes->getRequestOperationData()); owllinkNextReqRes->setModifiedNode(requestString); QByteArray transReqByteArray(requestString.toUtf8()); QString operationName = owllinkNextReqRes->getOperationName(); if (mNumberInitRequests == 0) { mTestingTiming.start(); } mWebRequest = mTransManager->createRequest(QString("http://%1").arg(mAddressString), transReqByteArray); mNextRequestTiming.start(); CReasonerEvaluationRequestResponseSPARQL* sprqlReqRes = (CReasonerEvaluationRequestResponseSPARQL*)mNextReqRes; if (mConfSeparateQueryingTimeout != -1 && sprqlReqRes->getOperationMode() == "QUERY") { startTimerWithIntervalLimited(TIMERIDREQUESTTIMEOUT, mConfSeparateQueryingTimeout, 1); } else { startTimerWithIntervalLimited(TIMERIDREQUESTTIMEOUT, mRequestTimeout, 1); } mWebResponse = mTransManager->getResponse(mWebRequest); mTransManager->callbackFinishedRequest(mWebResponse,new CReasonerEvaluationNextEvent(this,mCurrentOperationNumber)); LOG(INFO,getLogDomain(),logTr("Sending SPARQL operation %1 of %2 with command '%3' to '%4'.").arg(mCurrentRequestCount+1).arg(mTotalRequestCount).arg(operationName).arg(mAddressString),this); return true; } QString CReasonerEvaluationRequestClientSPARQLThread::getAdaptedRequestNode(const QString& reqeustString, CSPARQLKnowledgeBaseSplittingOperationData* operationData) { QStringList partStringList(operationData->getOperationPartStringList()); if (operationData->getOperationType() == CSPARQLKnowledgeBaseSplittingOperationData::SPARQL_UPDATE_MANAGE) { QStringList modifiedPartStringList; while (!partStringList.isEmpty()) { const QString& part = partStringList.takeFirst(); if (!part.startsWith("#")) { modifiedPartStringList.append(part); QString keyword = part.toUpper(); if (keyword == "LOAD") { bool loadingIRIPartFound = false; while (!partStringList.isEmpty() && !loadingIRIPartFound) { const QString& loadingIRIPart = partStringList.takeFirst(); if (loadingIRIPart.toUpper() != "SILENT") { loadingIRIPartFound = true; QString iriString = getIRI(loadingIRIPart); if (iriString.startsWith("file:") && !iriString.startsWith("file://")) { QString relFileString = iriString; relFileString = relFileString.remove(0, 5); QDir currFilePath(QDir::currentPath()); QString absFileString = currFilePath.absoluteFilePath(relFileString); absFileString = QUrl::fromPercentEncoding(QUrl::fromLocalFile(absFileString).toString().toUtf8()); modifiedPartStringList.append(QString("<%1>").arg(absFileString)); LOG(INFO, getLogDomain(), logTr("Changed relative filepath '%1' to absolute filepath '%2'.").arg(iriString).arg(absFileString), this); } else { modifiedPartStringList.append(loadingIRIPart); } } else { modifiedPartStringList.append(loadingIRIPart); } } } else if (keyword == "PREFIX" || keyword == "@PREFIX") { if (partStringList.size() >= 2) { QString prefix = partStringList.takeFirst(); modifiedPartStringList.append(prefix); QString resolvedPrefix = prefix; while (resolvedPrefix.endsWith(":")) { resolvedPrefix = resolvedPrefix.mid(0, resolvedPrefix.length() - 1); } QString iriString = partStringList.takeFirst(); modifiedPartStringList.append(iriString); QString iri = getIRI(iriString); mPrefixHash.insert(resolvedPrefix, iri); } } } } return modifiedPartStringList.join(" "); } else { return reqeustString; } } bool CReasonerEvaluationRequestClientSPARQLThread::loadReasonerRequests(const QString& initFileString, const QString& testFileString) { mCurrentRequestCount = 0; mTotalRequestCount = 0; mRemainingRestartRepeatingRequests = 0; mRestartRepeatingRequests = 0; bool loaded = loadReasonerRequest(initFileString); mTotalInitRequests = mNumberInitRequests = mRemainingRequestList.count(); loaded &= loadReasonerRequest(testFileString); return loaded; } QString CReasonerEvaluationRequestClientSPARQLThread::getIRI(const QString& iriString) { if (iriString.startsWith("<") && iriString.endsWith(">")) { return iriString.mid(1, iriString.length() - 2); } else { int colonPos = iriString.indexOf(':'); if (colonPos != -1) { QString prefix = iriString.mid(0, colonPos); QString prefixResolvedString = mPrefixHash.value(prefix); if (!prefixResolvedString.isEmpty()) { QString remainingIRI = iriString.mid(colonPos + 1); if (!remainingIRI.isEmpty()) { return prefixResolvedString + remainingIRI; } } } return mBaseIRI + iriString; } } bool CReasonerEvaluationRequestClientSPARQLThread::loadReasonerRequest(const QString& requestFileString) { CSPARQLKnowledgeBaseSplittingOperationParser parser; parser.parseQueryFile(requestFileString); mBaseIRI = requestFileString; cint64 loadedRequestCount = 0; QList operationDataList = parser.getSPARQLOperationList(); bool hasUpdatingOperation = false; for (CSPARQLKnowledgeBaseSplittingOperationData* operationData : operationDataList) { CReasonerEvaluationRequestResponseSPARQL* requestRespone = new CReasonerEvaluationRequestResponseSPARQL(*operationData); QStringList partStringList(operationData->getOperationPartStringList()); requestRespone->setRequestNode(partStringList.join(" ")); bool queryingOperation = false; bool updatingOperation = false; QString operationMode; QString operationName; if (requestRespone->getRequestOperationData()->getOperationType() == CSPARQLKnowledgeBaseSplittingOperationData::SPARQL_QUERY) { operationMode = "QUERY"; queryingOperation = true; } else if (requestRespone->getRequestOperationData()->getOperationType() == CSPARQLKnowledgeBaseSplittingOperationData::SPARQL_UPDATE_MANAGE) { operationMode = "UPDATE"; updatingOperation = true; } else if (requestRespone->getRequestOperationData()->getOperationType() == CSPARQLKnowledgeBaseSplittingOperationData::SPARQL_UPDATE_MODIFY) { operationMode = "UPDATE"; updatingOperation = true; } requestRespone->setOperationMode(operationMode); for (const QString& partString : partStringList) { if (mSPARQLOperationKeywordNameSet.contains(partString.toUpper())) { operationName = mSPARQLOperationKeywordNameSet.value(partString.toUpper()); break; } } if (!operationName.isEmpty()) { requestRespone->setOperationName(operationName); } else { requestRespone->setOperationMode(operationMode); } if (mConfSendPrepareBeforeQuerying && mConfSupportsPrepareCommand) { if (hasUpdatingOperation && queryingOperation) { CSPARQLKnowledgeBaseSplittingOperationData prepareOperation(CSPARQLKnowledgeBaseSplittingOperationData::SPARQL_UPDATE_MANAGE, operationData->getKnowledgeBaseString(), QStringList() << "PREPARE" << "QUERYING" << operationData->getKnowledgeBaseString()); CReasonerEvaluationRequestResponseSPARQL* prepareRequestRespone = new CReasonerEvaluationRequestResponseSPARQL(prepareOperation); prepareRequestRespone->setOperationMode("PREPARE"); prepareRequestRespone->setOperationName(mSPARQLOperationKeywordNameSet.value("PREPARE")); mRemainingRequestList.append(prepareRequestRespone); mAllRequestList.append(prepareRequestRespone); ++mTotalRequestCount; ++loadedRequestCount; hasUpdatingOperation = false; } else { hasUpdatingOperation |= updatingOperation; } } mRemainingRequestList.append(requestRespone); mAllRequestList.append(requestRespone); ++mTotalRequestCount; ++loadedRequestCount; } LOG(INFO, getLogDomain(), logTr("Loaded %1 SPARQL operations from file '%2'.").arg(loadedRequestCount).arg(requestFileString), this); return true; } }; // end namespace Evaluation }; // end namespace Test }; // end namespace Konclude