/* * Copyright (C) 2013-2015, 2019 by the Konclude Developer Team. * * This file is part of the reasoning system Konclude. * For details and support, see . * * Konclude is free software: you can redistribute it and/or modify * it under the terms of version 3 of the GNU Lesser General Public * License (LGPLv3) as published by the Free Software Foundation. * * Konclude is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU (Lesser) General Public License for more details. * * You should have received a copy of the GNU (Lesser) General Public * License along with Konclude. If not, see . * */ #include "CSPARQLBatchFileLoader.h" namespace Konclude { namespace Control { namespace Loader { CSPARQLBatchFileLoader::CSPARQLBatchFileLoader() { mSPARQLInterpreter = 0; mLoaderConfig = 0; mConfLogProcessingTimes = false; mStreamSPARQLHeader = QString("\r\n").toLocal8Bit(); mStreamSPARQLFooter = QString("\r\n").toLocal8Bit(); } CSPARQLBatchFileLoader::~CSPARQLBatchFileLoader() { if (mWriteLimitSemaphore) { delete mWriteLimitSemaphore; } } CLoader *CSPARQLBatchFileLoader::init(CLoaderFactory *loaderFactory, CConfiguration *config) { mLoaderConfig = config; CConfigData *data = config->getLastConfigChange("Konclude.SPARQL.RequestFile"); if (data) { reqFileString = data->getString(); } data = config->getLastConfigChange("Konclude.SPARQL.ResponseFile"); if (data) { resFileString = data->getString(); } mConfDirectConsoleOutput = false; data = config->getLastConfigChange("Konclude.SPARQL.WriteResponseToStandardOutput"); if (data) { mConfDirectConsoleOutput = data->getBoolean(); } mConfDirectErrorOutput = false; data = config->getLastConfigChange("Konclude.SPARQL.WriteResponseToStandardError"); if (data) { mConfDirectErrorOutput = data->getBoolean(); } mConfCloseAfterOutput = false; data = config->getLastConfigChange("Konclude.SPARQL.CloseAfterProcessedRequest"); if (data) { mConfCloseAfterOutput = data->getBoolean(); } mBlockUntilProcessed = false; data = config->getLastConfigChange("Konclude.SPARQL.BlockUntilProcessedRequest"); if (data) { mBlockUntilProcessed = data->getBoolean(); } mConfLogProcessingTimes = false; data = config->getLastConfigChange("Konclude.SPARQL.LoggingExtendedProcessingTimes"); if (data) { mConfLogProcessingTimes = data->getBoolean(); } reasonerCommander = CConfigManagerReader::readCommanderManagerConfig(config); cint64 writeBufferBlockingLimit = CConfigDataReader::readConfigInteger(mLoaderConfig, "Konclude.SPARQL.File.WriteBufferBlockingLimit", 20); if (writeBufferBlockingLimit > 0) { mWriteLimitSemaphore = new QSemaphore(writeBufferBlockingLimit); } else { mWriteLimitSemaphore = nullptr; } mBlockingWarned = false; return this; } CLoader *CSPARQLBatchFileLoader::load() { startProcessing(); if (mBlockUntilProcessed) { mBlockingSem.acquire(); } return this; } CConfiguration *CSPARQLBatchFileLoader::getConfiguration() { if (!reasonerCommander) { return mLoaderConfig; } else { CGetDescriptionCommand *getDesComm = new CGetDescriptionCommand(); reasonerCommander->delegateCommand(getDesComm); CCommandExecutedBlocker commExeBlocker; commExeBlocker.waitExecutedCommand(getDesComm); CConfiguration *config = getDesComm->getConfiguration(); return config; } } bool CSPARQLBatchFileLoader::processCustomsEvents(QEvent::Type type, CCustomEvent *event) { if (COWLlinkProcessor::processCustomsEvents(type, event)) { return true; } else if (type == CResultStreamingWriteEvent::EVENTTYPE) { CResultStreamingWriteEvent* rswe = (CResultStreamingWriteEvent*)event; QList* bufferList = rswe->getBufferList(); bool last = rswe->isLast(); writeStreamDataToFile(bufferList, last); mWriteLimitSemaphore->release(1); return true; } return false; } COWLlinkProcessor *CSPARQLBatchFileLoader::initializeOWLlinkContent() { mResponseTime.start(); if (!reqFileString.isEmpty()) { QFile file(reqFileString); LOG(INFO,getLogDomain(),logTr("Processing SPARQL request from file '%1'.").arg(reqFileString),this); if (file.open(QIODevice::ReadOnly)) { QString fileContentString = file.readAll(); if (!resFileString.isEmpty()) { mSPARQLInterpreter = new CSPARQLRecordResultStreamingInterpreter(this, preSynchronizer, mLoaderConfig); } else { LOG(WARN, getLogDomain(), logTr("No output file given, requesting only answer counting."), this); mSPARQLInterpreter = new CSPARQLRecordResultStreamingInterpreter(nullptr, preSynchronizer, mLoaderConfig); } defaultCommandDelegater = mSPARQLInterpreter; preSynchronizer = new CPreconditionSynchronizer(mSPARQLInterpreter); CParseProcessSPARQLTextCommand* parseProcSPARQLCommand = new CParseProcessSPARQLTextCommand(fileContentString); CCommandProcessedCallbackEvent *proComm = new CCommandProcessedCallbackEvent(this, parseProcSPARQLCommand); parseProcSPARQLCommand->setRecorder(mSPARQLInterpreter); parseProcSPARQLCommand->addProcessedCallback(proComm); defaultCommandDelegater->delegateCommand(parseProcSPARQLCommand); mWritingStarted = false; mChunkPart = 0; mResponseFile = nullptr; mWritingFailed = false; mWritingAppendingMode = false; if (mConfLogProcessingTimes) { cint64 readMilliSeconds = mResponseTime.elapsed(); LOG(INFO,getLogDomain(),logTr("Read XML content in %1 ms.").arg(readMilliSeconds),this); } } else { LOG(ERROR,getLogDomain(),logTr("SPARQL request file '%1' not found.").arg(reqFileString),this); // seems to be required to preventing crashes // TODO: fix this->usleep(200); concludeOWLlinkContent(); } } return this; } bool CSPARQLBatchFileLoader::writeStreamData(const QList& bufferList, bool last) { if (mWriteLimitSemaphore) { if (!mBlockingWarned && mWriteLimitSemaphore->available() <= 2) { mBlockingWarned = true; LOG(WARN, getLogDomain(), logTr("Result writing buffer limit almost reached, result generation may be blocked due to slow file system write access."), this); } mWriteLimitSemaphore->acquire(1); } postEvent(new CResultStreamingWriteEvent(bufferList, last)); return !mWritingFailed; } CSPARQLStreamingWriter* CSPARQLBatchFileLoader::writeStreamDataToFile(QList* bufferList, bool last) { ++mChunkPart; // write with chunked encoding bool firstChunk = false; if (!mWritingStarted) { mWritingStarted = true; if (!resFileString.isEmpty()) { mResponseFile = new QFile(resFileString); bool fileOpen = false; if (!mWritingAppendingMode) { fileOpen = mResponseFile->open(QIODevice::WriteOnly); } else { fileOpen = mResponseFile->open(QIODevice::Append); if (fileOpen) { mResponseFile->write(QString("\r\n\r\n").toLocal8Bit()); } } if (!fileOpen) { LOG(ERROR, getLogDomain(), logTr("Opening file '%1' for writing failed.").arg(resFileString), this); delete mResponseFile; mResponseFile = nullptr; } else { mResponseFile->write(mStreamSPARQLHeader); } } firstChunk = true; } if (mResponseFile) { for (CSPARQLResultBufferWriteData bufferData : *bufferList) { for (cint64 i = bufferData.getWriteCount(); i > 0; --i) { mResponseFile->write(*bufferData.getBuffer()); } } } else { if (!mConfDirectConsoleOutput && !mConfDirectErrorOutput) { // answers are not further used, no need to continue computation mWritingFailed = true; } } if (mConfDirectConsoleOutput) { for (CSPARQLResultBufferWriteData bufferData : *bufferList) { for (cint64 i = bufferData.getWriteCount(); i > 0; --i) { std::cout << bufferData.getBuffer(); } } } if (mConfDirectErrorOutput) { for (CSPARQLResultBufferWriteData bufferData : *bufferList) { for (cint64 i = bufferData.getWriteCount(); i > 0; --i) { std::cout << bufferData.getBuffer(); } } } cint64 bufferSize = 0; for (CSPARQLResultBufferWriteData bufferData : *bufferList) { bufferSize += bufferData.getBuffer()->size() * bufferData.getWriteCount(); delete bufferData.getBuffer(); } if (last) { if (mResponseFile) { mResponseFile->write(mStreamSPARQLFooter); LOG(INFO, getLogDomain(), logTr("Last part of %1 with %2 bytes written.").arg(mChunkPart).arg(bufferSize), this); mResponseFile->close(); mWritingAppendingMode = true; mWritingStarted = false; delete mResponseFile; } } else { if (mResponseFile) { LOG(INFO, getLogDomain(), logTr("Writing part %1 with %2 bytes to file.").arg(mChunkPart).arg(bufferSize), this); mResponseFile->flush(); } } return this; } COWLlinkProcessor *CSPARQLBatchFileLoader::concludeOWLlinkContent() { if (mConfLogProcessingTimes) { cint64 responseMilliSeconds = mResponseTime.elapsed(); LOG(INFO,getLogDomain(),logTr("SPARQL request processed in %1 ms.").arg(responseMilliSeconds),this); } if (mBlockUntilProcessed) { mBlockingSem.release(); } if (mConfCloseAfterOutput) { CLogger::getInstance()->waitSynchronization(); //qSleep(10); //qWait(); QCoreApplication::exit(); } return this; } CLoader *CSPARQLBatchFileLoader::exit() { stopThread(); return this; } }; // end namespace Loader }; // end namespace Control }; // end namespace Konclude