Debian Codegraph @

Source/Control/Loader/CSPARQLBatchFileLoader.cpp

raw
1 /*
2 * Copyright (C) 2013-2015, 2019 by the Konclude Developer Team.
3 *
4 * This file is part of the reasoning system Konclude.
5 * For details and support, see <http://konclude.com/>.
6 *
7 * Konclude is free software: you can redistribute it and/or modify
8 * it under the terms of version 3 of the GNU Lesser General Public
9 * License (LGPLv3) as published by the Free Software Foundation.
10 *
11 * Konclude is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU (Lesser) General Public License for more details.
15 *
16 * You should have received a copy of the GNU (Lesser) General Public
17 * License along with Konclude. If not, see <http://www.gnu.org/licenses/>.
18 *
19 */
20
21 #include "CSPARQLBatchFileLoader.h"
22
23
24 namespace Konclude {
25
26 namespace Control {
27
28 namespace Loader {
29
30
31 CSPARQLBatchFileLoader::CSPARQLBatchFileLoader() {
32 mSPARQLInterpreter = 0;
33 mLoaderConfig = 0;
34 mConfLogProcessingTimes = false;
35
36 mStreamSPARQLHeader = QString("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n<sparql xmlns=\"http://www.w3.org/2005/sparql-results#\" xml:base=\"http://www.w3.org/2005/sparql-results#\" xmlns:owl=\"http://www.w3.org/2002/07/owl#\" xmlns:xsd=\"http://www.w3.org/2001/XMLSchema#\">").toLocal8Bit();
37 mStreamSPARQLFooter = QString("\r\n</sparql>").toLocal8Bit();
38 }
39
40
41
42 CSPARQLBatchFileLoader::~CSPARQLBatchFileLoader() {
43 if (mWriteLimitSemaphore) {
44 delete mWriteLimitSemaphore;
45 }
46 }
47
48
49 CLoader *CSPARQLBatchFileLoader::init(CLoaderFactory *loaderFactory, CConfiguration *config) {
50 mLoaderConfig = config;
51 CConfigData *data = config->getLastConfigChange("Konclude.SPARQL.RequestFile");
52 if (data) {
53 reqFileString = data->getString();
54 }
55
56 data = config->getLastConfigChange("Konclude.SPARQL.ResponseFile");
57 if (data) {
58 resFileString = data->getString();
59 }
60
61
62 mConfDirectConsoleOutput = false;
63 data = config->getLastConfigChange("Konclude.SPARQL.WriteResponseToStandardOutput");
64 if (data) {
65 mConfDirectConsoleOutput = data->getBoolean();
66 }
67 mConfDirectErrorOutput = false;
68 data = config->getLastConfigChange("Konclude.SPARQL.WriteResponseToStandardError");
69 if (data) {
70 mConfDirectErrorOutput = data->getBoolean();
71 }
72 mConfCloseAfterOutput = false;
73 data = config->getLastConfigChange("Konclude.SPARQL.CloseAfterProcessedRequest");
74 if (data) {
75 mConfCloseAfterOutput = data->getBoolean();
76 }
77 mBlockUntilProcessed = false;
78 data = config->getLastConfigChange("Konclude.SPARQL.BlockUntilProcessedRequest");
79 if (data) {
80 mBlockUntilProcessed = data->getBoolean();
81 }
82 mConfLogProcessingTimes = false;
83 data = config->getLastConfigChange("Konclude.SPARQL.LoggingExtendedProcessingTimes");
84 if (data) {
85 mConfLogProcessingTimes = data->getBoolean();
86 }
87
88 reasonerCommander = CConfigManagerReader::readCommanderManagerConfig(config);
89
90
91 cint64 writeBufferBlockingLimit = CConfigDataReader::readConfigInteger(mLoaderConfig, "Konclude.SPARQL.File.WriteBufferBlockingLimit", 20);
92 if (writeBufferBlockingLimit > 0) {
93 mWriteLimitSemaphore = new QSemaphore(writeBufferBlockingLimit);
94 } else {
95 mWriteLimitSemaphore = nullptr;
96 }
97 mBlockingWarned = false;
98
99
100 return this;
101 }
102
103
104
105 CLoader *CSPARQLBatchFileLoader::load() {
106 startProcessing();
107 if (mBlockUntilProcessed) {
108 mBlockingSem.acquire();
109 }
110 return this;
111 }
112
113
114
115 CConfiguration *CSPARQLBatchFileLoader::getConfiguration() {
116 if (!reasonerCommander) {
117 return mLoaderConfig;
118 } else {
119 CGetDescriptionCommand *getDesComm = new CGetDescriptionCommand();
120 reasonerCommander->delegateCommand(getDesComm);
121 CCommandExecutedBlocker commExeBlocker;
122 commExeBlocker.waitExecutedCommand(getDesComm);
123 CConfiguration *config = getDesComm->getConfiguration();
124 return config;
125 }
126 }
127
128
129
130 bool CSPARQLBatchFileLoader::processCustomsEvents(QEvent::Type type, CCustomEvent *event) {
131 if (COWLlinkProcessor::processCustomsEvents(type, event)) {
132 return true;
133 } else if (type == CResultStreamingWriteEvent::EVENTTYPE) {
134 CResultStreamingWriteEvent* rswe = (CResultStreamingWriteEvent*)event;
135 QList<CSPARQLResultBufferWriteData>* bufferList = rswe->getBufferList();
136 bool last = rswe->isLast();
137 writeStreamDataToFile(bufferList, last);
138 mWriteLimitSemaphore->release(1);
139 return true;
140 }
141 return false;
142 }
143
144
145 COWLlinkProcessor *CSPARQLBatchFileLoader::initializeOWLlinkContent() {
146 mResponseTime.start();
147 if (!reqFileString.isEmpty()) {
148 QFile file(reqFileString);
149 LOG(INFO,getLogDomain(),logTr("Processing SPARQL request from file '%1'.").arg(reqFileString),this);
150 if (file.open(QIODevice::ReadOnly)) {
151 QString fileContentString = file.readAll();
152
153 if (!resFileString.isEmpty()) {
154 mSPARQLInterpreter = new CSPARQLRecordResultStreamingInterpreter(this, preSynchronizer, mLoaderConfig);
155 } else {
156 LOG(WARN, getLogDomain(), logTr("No output file given, requesting only answer counting."), this);
157 mSPARQLInterpreter = new CSPARQLRecordResultStreamingInterpreter(nullptr, preSynchronizer, mLoaderConfig);
158 }
159 defaultCommandDelegater = mSPARQLInterpreter;
160 preSynchronizer = new CPreconditionSynchronizer(mSPARQLInterpreter);
161
162 CParseProcessSPARQLTextCommand* parseProcSPARQLCommand = new CParseProcessSPARQLTextCommand(fileContentString);
163 CCommandProcessedCallbackEvent *proComm = new CCommandProcessedCallbackEvent(this, parseProcSPARQLCommand);
164 parseProcSPARQLCommand->setRecorder(mSPARQLInterpreter);
165 parseProcSPARQLCommand->addProcessedCallback(proComm);
166 defaultCommandDelegater->delegateCommand(parseProcSPARQLCommand);
167
168 mWritingStarted = false;
169 mChunkPart = 0;
170 mResponseFile = nullptr;
171 mWritingFailed = false;
172 mWritingAppendingMode = false;
173
174
175 if (mConfLogProcessingTimes) {
176 cint64 readMilliSeconds = mResponseTime.elapsed();
177 LOG(INFO,getLogDomain(),logTr("Read XML content in %1 ms.").arg(readMilliSeconds),this);
178 }
179
180 } else {
181 LOG(ERROR,getLogDomain(),logTr("SPARQL request file '%1' not found.").arg(reqFileString),this);
182 // seems to be required to preventing crashes
183 // TODO: fix
184 this->usleep(200);
185 concludeOWLlinkContent();
186 }
187 }
188 return this;
189 }
190
191
192
193 bool CSPARQLBatchFileLoader::writeStreamData(const QList<CSPARQLResultBufferWriteData>& bufferList, bool last) {
194 if (mWriteLimitSemaphore) {
195 if (!mBlockingWarned && mWriteLimitSemaphore->available() <= 2) {
196 mBlockingWarned = true;
197 LOG(WARN, getLogDomain(), logTr("Result writing buffer limit almost reached, result generation may be blocked due to slow file system write access."), this);
198 }
199 mWriteLimitSemaphore->acquire(1);
200 }
201 postEvent(new CResultStreamingWriteEvent(bufferList, last));
202 return !mWritingFailed;
203 }
204
205 CSPARQLStreamingWriter* CSPARQLBatchFileLoader::writeStreamDataToFile(QList<CSPARQLResultBufferWriteData>* bufferList, bool last) {
206 ++mChunkPart;
207 // write with chunked encoding
208 bool firstChunk = false;
209 if (!mWritingStarted) {
210 mWritingStarted = true;
211 if (!resFileString.isEmpty()) {
212 mResponseFile = new QFile(resFileString);
213 bool fileOpen = false;
214 if (!mWritingAppendingMode) {
215 fileOpen = mResponseFile->open(QIODevice::WriteOnly);
216 } else {
217 fileOpen = mResponseFile->open(QIODevice::Append);
218 if (fileOpen) {
219 mResponseFile->write(QString("\r\n\r\n").toLocal8Bit());
220 }
221 }
222 if (!fileOpen) {
223 LOG(ERROR, getLogDomain(), logTr("Opening file '%1' for writing failed.").arg(resFileString), this);
224 delete mResponseFile;
225 mResponseFile = nullptr;
226 } else {
227 mResponseFile->write(mStreamSPARQLHeader);
228 }
229 }
230 firstChunk = true;
231 }
232 if (mResponseFile) {
233 for (CSPARQLResultBufferWriteData bufferData : *bufferList) {
234 for (cint64 i = bufferData.getWriteCount(); i > 0; --i) {
235 mResponseFile->write(*bufferData.getBuffer());
236 }
237 }
238 } else {
239 if (!mConfDirectConsoleOutput && !mConfDirectErrorOutput) {
240 // answers are not further used, no need to continue computation
241 mWritingFailed = true;
242 }
243 }
244 if (mConfDirectConsoleOutput) {
245 for (CSPARQLResultBufferWriteData bufferData : *bufferList) {
246 for (cint64 i = bufferData.getWriteCount(); i > 0; --i) {
247 std::cout << bufferData.getBuffer();
248 }
249 }
250 }
251 if (mConfDirectErrorOutput) {
252 for (CSPARQLResultBufferWriteData bufferData : *bufferList) {
253 for (cint64 i = bufferData.getWriteCount(); i > 0; --i) {
254 std::cout << bufferData.getBuffer();
255 }
256 }
257 }
258 cint64 bufferSize = 0;
259 for (CSPARQLResultBufferWriteData bufferData : *bufferList) {
260 bufferSize += bufferData.getBuffer()->size() * bufferData.getWriteCount();
261 delete bufferData.getBuffer();
262 }
263 if (last) {
264 if (mResponseFile) {
265 mResponseFile->write(mStreamSPARQLFooter);
266 LOG(INFO, getLogDomain(), logTr("Last part of %1 with %2 bytes written.").arg(mChunkPart).arg(bufferSize), this);
267 mResponseFile->close();
268 mWritingAppendingMode = true;
269 mWritingStarted = false;
270 delete mResponseFile;
271 }
272 } else {
273 if (mResponseFile) {
274 LOG(INFO, getLogDomain(), logTr("Writing part %1 with %2 bytes to file.").arg(mChunkPart).arg(bufferSize), this);
275 mResponseFile->flush();
276 }
277 }
278 return this;
279 }
280
281
282
283 COWLlinkProcessor *CSPARQLBatchFileLoader::concludeOWLlinkContent() {
284 if (mConfLogProcessingTimes) {
285 cint64 responseMilliSeconds = mResponseTime.elapsed();
286 LOG(INFO,getLogDomain(),logTr("SPARQL request processed in %1 ms.").arg(responseMilliSeconds),this);
287 }
288 if (mBlockUntilProcessed) {
289 mBlockingSem.release();
290 }
291 if (mConfCloseAfterOutput) {
292 CLogger::getInstance()->waitSynchronization();
293 //qSleep(10);
294 //qWait();
295 QCoreApplication::exit();
296 }
297 return this;
298 }
299
300
301 CLoader *CSPARQLBatchFileLoader::exit() {
302
303 stopThread();
304
305 return this;
306 }
307
308
309
310 }; // end namespace Loader
311
312 }; // end namespace Control
313
314 }; // end namespace Konclude