Debian Codegraph @

Source/Parser/CRDFRedlandRaptorSimpleConcurrentParser.cpp

raw
1 /*
2 * Copyright (C) 2013-2015, 2019 by the Konclude Developer Team.
3 *
4 * This file is part of the reasoning system Konclude.
5 * For details and support, see <http://konclude.com/>.
6 *
7 * Konclude is free software: you can redistribute it and/or modify
8 * it under the terms of version 3 of the GNU Lesser General Public
9 * License (LGPLv3) as published by the Free Software Foundation.
10 *
11 * Konclude is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU (Lesser) General Public License for more details.
15 *
16 * You should have received a copy of the GNU (Lesser) General Public
17 * License along with Konclude. If not, see <http://www.gnu.org/licenses/>.
18 *
19 */
20 #ifdef KONCLUDE_REDLAND_INTEGRATION
21
22 #include "CRDFRedlandRaptorSimpleConcurrentParser.h"
23
24
25 namespace Konclude {
26
27 namespace Parser {
28
29
30
31 CRDFRedlandRaptorSimpleConcurrentParser::CRDFRedlandRaptorSimpleConcurrentParser(COntologyBuilder* ontologyBuilder, CTRIPLES_DATA_UPDATE_TYPE updateType, QString redlandParsingFormat, CConfiguration* configuration) : CRDFRedlandRaptorParser(ontologyBuilder, updateType, redlandParsingFormat, configuration) {
32 mConcurrentOffsetSplitSize = CConfigDataReader::readConfigInteger(mConfiguration, "Konclude.Parser.RedlandRaptor.ConcurrentBytesBatchingSize", 50000000);
33 }
34
35
36 CRDFRedlandRaptorSimpleConcurrentParser::~CRDFRedlandRaptorSimpleConcurrentParser() {
37 }
38
39
40 class TParseOffsetData {
41 public:
42 TParseOffsetData(cint64 startOffset = 0, cint64 endOffset = 0) {
43 mStartOffset = startOffset;
44 mEndOffset = endOffset;
45 }
46 TParseOffsetData(const TParseOffsetData& data) {
47 mStartOffset = data.mStartOffset;
48 mEndOffset = data.mEndOffset;
49 mIndex = data.mIndex;
50 }
51 cint64 mIndex = 0;
52 cint64 mStartOffset = 0;
53 cint64 mLastPos = 0;
54 cint64 mEndOffset = 0;
55 QIODevice* mDevice;
56 bool mStartCharFound = false;
57 bool mEndCharFound = false;
58
59 QByteArray* mBasePrefixDataArray = nullptr;
60 cint64 mBasePrefixDataArrayReadPos = 0;
61 bool mBasePrefixDataArrayRead = false;
62
63 };
64
65 int raptor_iostream_read_bytes_from_qiodevice_with_offset(void *context, void *ptr, size_t size, size_t nmemb) {
66 TParseOffsetData* parseOffsetData = (TParseOffsetData*)context;
67
68 char* begCharPtr = (char*)ptr;
69 qint64 readElements = 0;
70 if (!parseOffsetData->mBasePrefixDataArrayRead && parseOffsetData->mBasePrefixDataArray) {
71 const char* data = parseOffsetData->mBasePrefixDataArray->constData();
72 int length = parseOffsetData->mBasePrefixDataArray->length();
73 while (readElements < nmemb && parseOffsetData->mBasePrefixDataArrayReadPos < length) {
74 char* charPointer = (char*)ptr;
75 int nextElByteCount = 0;
76 while ((size_t)nextElByteCount < size && parseOffsetData->mBasePrefixDataArrayReadPos < length) {
77 charPointer[nextElByteCount++] = data[parseOffsetData->mBasePrefixDataArrayReadPos++];
78 }
79 if ((size_t)nextElByteCount >= size) {
80 readElements++;
81 nextElByteCount = 0;
82 ptr = (void*)((char*)ptr + size);
83 }
84 }
85 if (parseOffsetData->mBasePrefixDataArrayReadPos >= length) {
86 parseOffsetData->mBasePrefixDataArrayRead = true;
87 }
88 }
89
90 QIODevice* ioDevice = parseOffsetData->mDevice;
91 parseOffsetData->mLastPos = ioDevice->pos();
92 cint64 maxSize = parseOffsetData->mEndOffset - parseOffsetData->mLastPos;
93 if (maxSize > 0 && readElements < nmemb) {
94 if (size == 1) {
95 size_t nmembCount = qMin((size_t)(nmemb - readElements), (size_t)maxSize);
96 readElements += ioDevice->read((char*)ptr, nmembCount);
97 ptr = (void*)((char*)ptr + nmembCount);
98 } else {
99 size_t nmembCount = qMin((size_t)(nmemb - readElements), (size_t)maxSize/size);
100 for (size_t i = 0; i < nmembCount && !ioDevice->atEnd(); ++i) {
101 qint64 read = ioDevice->read((char*)ptr, size);
102 if (read >= size) {
103 ptr = (void*)((char*)ptr + size);
104 readElements++;
105 } else {
106 break;
107 }
108 }
109 }
110 }
111 bool hasSpace = false;
112 bool hasPoint = false;
113 while (readElements < nmemb && !parseOffsetData->mEndCharFound && !ioDevice->atEnd()) {
114
115 char dataChar[2];
116 int nextElByteCount = 0;
117 while (!ioDevice->atEnd() && (size_t)nextElByteCount < size) {
118 cint64 read = ioDevice->read(dataChar, 1);
119 if (read == 1) {
120 if (dataChar[0] == ' ') {
121 hasSpace = true;
122 } else if (dataChar[0] == '.' && hasSpace && !hasPoint) {
123 hasPoint = true;
124 } else if (dataChar[0] == '\r' && hasPoint) {
125 } else if (dataChar[0] == '\n' && hasPoint) {
126 parseOffsetData->mEndCharFound = true;
127 } else {
128 hasSpace = false;
129 }
130 char* charPointer = (char*)ptr;
131 charPointer[nextElByteCount++] = dataChar[0];
132 }
133 }
134 if ((size_t)nextElByteCount >= size) {
135 readElements++;
136 nextElByteCount = 0;
137 ptr = (void*)((char*)ptr + size);
138 }
139
140
141 }
142 if (readElements > nmemb) {
143 cout << "elements buffer length overwritten" << endl;
144 }
145 if ((char*)ptr > begCharPtr + nmemb) {
146 cout << "pointer buffer length overwritten" << endl;
147 }
148 return readElements;
149 }
150
151
152 int raptor_iostream_read_eof_func_from_qiodevice_with_offset(void *context) {
153 TParseOffsetData* parseOffsetData = (TParseOffsetData*)context;
154 QIODevice* ioDevice = parseOffsetData->mDevice;
155 if (ioDevice->atEnd() || parseOffsetData->mEndCharFound) {
156 return 1;
157 }
158 return 0;
159 }
160
161
162 bool CRDFRedlandRaptorSimpleConcurrentParser::parseTriples(QIODevice* ioDevice, const QString& baseUriString) {
163
164 QString fileString = baseUriString;
165 if (fileString.startsWith("file:")) {
166 fileString = fileString.replace("file:", "");
167 }
168 QFile* file = new QFile(fileString);
169 while (!file->open(QIODevice::ReadOnly) && fileString.startsWith("/")) {
170 fileString = fileString.mid(1);
171 delete file;
172 file = new QFile(fileString);
173 }
174 delete file;
175 file = new QFile(fileString);
176 if (!file->open(QIODevice::ReadOnly) || mConfLoadTriplesIntoStore || (mRedlandParsingFormat != "turtle" && mRedlandParsingFormat != "ntriples")) {
177 delete file;
178 return CRDFRedlandRaptorParser::parseTriples(ioDevice, baseUriString);
179 }
180 cint64 bytesAvailable = ioDevice->bytesAvailable();
181 delete file;
182
183
184
185
186 class TParsedTripleData {
187 public:
188 CRedlandStoredTriplesData* mTriplesData = nullptr;
189 };
190
191
192 QVector<TParseOffsetData> parseOffsetDataVec((bytesAvailable / mConcurrentOffsetSplitSize)+1);
193
194 cint64 idx = 0;
195 for (cint64 i = 0; i < bytesAvailable; i += mConcurrentOffsetSplitSize) {
196 TParseOffsetData data = TParseOffsetData(i, i + mConcurrentOffsetSplitSize);
197 data.mIndex = idx;
198 parseOffsetDataVec[idx++] = data;
199 }
200 LOG(INFO, getLogDomain(), logTr("Starting concurrent triple parsing for file '%1' with %2 partitions.").arg(fileString).arg(parseOffsetDataVec.size()), this);
201
202
203 QString* raptorParsingErrorString = nullptr;
204 bool warningNoStore = false;
205
206 QByteArray basePrefixDataArray;
207
208 while (!ioDevice->atEnd()) {
209 QByteArray data = ioDevice->readLine().trimmed();
210 QString dataString = data;
211 dataString = dataString.toUpper();
212 bool isBasePrefixData = false;
213 if (dataString.startsWith("PREFIX") || dataString.startsWith("@PREFIX")) {
214 isBasePrefixData = true;
215 }
216 if (dataString.startsWith("BASE") || dataString.startsWith("@BASE")) {
217 isBasePrefixData = true;
218 }
219 if (isBasePrefixData) {
220 basePrefixDataArray.append(data);
221 basePrefixDataArray.append(QString("\r\n"));
222 } else if (!dataString.startsWith("#")) {
223 break;
224 }
225 }
226
227 function<CRedlandStoredTriplesData*(const TParseOffsetData& parseData)> parseTriplesMapFunc = [&](const TParseOffsetData& parseData) ->CRedlandStoredTriplesData* {
228 QFile file(fileString);
229 if (parseData.mStartOffset != parseData.mEndOffset && file.open(QIODevice::ReadOnly)) {
230
231 TParseOffsetData tmpParseData(parseData);
232 tmpParseData.mDevice = &file;
233
234 file.seek(parseData.mStartOffset);
235
236 cint64 startSkipCount = 0;
237 if (!tmpParseData.mStartCharFound && parseData.mStartOffset > 0) {
238 tmpParseData.mBasePrefixDataArray = &basePrefixDataArray;
239 char dataChar[2];
240 bool hasSpace = false;
241 bool hasPoint = false;
242 while (!tmpParseData.mStartCharFound && !tmpParseData.mDevice->atEnd()) {
243 cint64 read = tmpParseData.mDevice->read(dataChar, 1);
244 if (read == 1) {
245 if (dataChar[0] == ' ') {
246 hasSpace = true;
247 } else if (dataChar[0] == '.' && hasSpace && !hasPoint) {
248 hasPoint = true;
249 } else if (dataChar[0] == '\r' && hasPoint) {
250 } else if (dataChar[0] == '\n' && hasPoint) {
251 tmpParseData.mStartCharFound = true;
252 } else {
253 hasSpace = false;
254 }
255 startSkipCount++;
256 }
257 }
258 } else {
259 tmpParseData.mBasePrefixDataArrayRead = true;
260 tmpParseData.mStartCharFound = true;
261 }
262
263 raptor_iostream_handler qioDeviceIOStreamHandler;
264 qioDeviceIOStreamHandler.version = 2;
265 qioDeviceIOStreamHandler.read_bytes = &raptor_iostream_read_bytes_from_qiodevice_with_offset;
266 qioDeviceIOStreamHandler.read_eof = &raptor_iostream_read_eof_func_from_qiodevice_with_offset;
267 qioDeviceIOStreamHandler.init = nullptr;
268 qioDeviceIOStreamHandler.finish = nullptr;
269 qioDeviceIOStreamHandler.write_byte = nullptr;
270 qioDeviceIOStreamHandler.write_bytes = nullptr;
271 qioDeviceIOStreamHandler.write_end = nullptr;
272
273 CRedlandStoredTriplesData* tripleData = getUpdatingTripleData();
274 raptor_world* raptor_world_ptr = librdf_world_get_raptor(tripleData->getRedlandWorld());
275
276
277 raptor_world_set_log_handler(raptor_world_ptr, &raptorParsingErrorString, &raptor_log_handler_error_notifier);
278
279 raptor_iostream* raptor_stream = raptor_new_iostream_from_handler(raptor_world_ptr, &tmpParseData, &qioDeviceIOStreamHandler);
280
281
282 QByteArray baseUriStringByteArray(baseUriString.toUtf8());
283 librdf_uri *baseUri = librdf_new_uri(tripleData->getRedlandWorld(), (const unsigned char*)baseUriStringByteArray.constData());
284
285
286 QByteArray mRedlandParsingFormatByteArray = mRedlandParsingFormat.toUtf8();
287 const char* parser_name = mRedlandParsingFormatByteArray.constData();
288 librdf_parser* parser = librdf_new_parser(tripleData->getRedlandWorld(), parser_name, NULL, NULL);
289 if (!parser) {
290 mErrorString = QString("Failed to create parser");
291 mParsingError = true;
292 }
293
294
295 //if (parser && raptor_stream && librdf_parser_parse_iostream_into_model(parser, raptor_stream, baseUri, tripleData->getRedlandStreamedModel())) {
296 // mParsingError = true;
297 //}
298
299 librdf_stream* tripleStream = nullptr;
300 if (parser && raptor_stream) {
301 tripleStream = librdf_parser_parse_iostream_as_stream(parser, raptor_stream, baseUri);
302 if (!tripleStream) {
303 mErrorString = QString("Failed to load data stream");
304 mParsingError = true;
305 }
306
307 librdf_node* critPredicate1 = librdf_new_node_from_uri_string(tripleData->getRedlandWorld(), (const unsigned char*)PREFIX_RDFS_SUBCLASS_OF);
308 librdf_node* critPredicate2 = librdf_new_node_from_uri_string(tripleData->getRedlandWorld(), (const unsigned char*)PREFIX_OWL_COMPLEMENT_OF);
309
310 CXLinker<librdf_statement*>* statementLinker = nullptr;
311 CXLinker<librdf_statement*>* lastStatementLinker = nullptr;
312 while (!librdf_stream_end(tripleStream)) {
313 librdf_statement* statement = librdf_stream_get_object(tripleStream);
314
315 if (!librdf_model_contains_statement(tripleData->getRedlandIndexedModel(), statement)) {
316 librdf_statement* statementCopy = librdf_new_statement_from_statement(statement);
317 CXLinker<librdf_statement*>* newStatementLinker = new CXLinker<librdf_statement*>();
318 newStatementLinker->initLinker(statementCopy, nullptr);
319 if (statementLinker) {
320 lastStatementLinker->setNext(newStatementLinker);
321 lastStatementLinker = newStatementLinker;
322 } else {
323 statementLinker = newStatementLinker;
324 lastStatementLinker = newStatementLinker;
325 }
326 if (!warningNoStore && (librdf_node_equals(critPredicate1, librdf_statement_get_predicate(statementCopy)) || librdf_node_equals(critPredicate2, librdf_statement_get_predicate(statementCopy)))) {
327 warningNoStore = true;
328 LOG(WARN, getLogDomain(), logTr("(Nontrivial) OWL axioms and expressions cannot be extracted from triples since they are not loaded into the triple store."), this);
329 }
330 }
331 librdf_stream_next(tripleStream);
332 }
333 librdf_free_node(critPredicate1);
334 librdf_free_node(critPredicate2);
335 tripleData->setRedlandStatementLinker(statementLinker);
336 }
337
338 if (raptor_stream) {
339 raptor_free_iostream(raptor_stream);
340 }
341
342 if (parser) {
343 librdf_free_parser(parser);
344 }
345
346 return tripleData;
347
348
349 }
350 return nullptr;
351
352 };
353
354
355 QVector<CRedlandStoredTriplesData*> triplesDataVec = QtConcurrent::blockingMapped(parseOffsetDataVec, parseTriplesMapFunc);
356
357
358
359 if (raptorParsingErrorString) {
360 mParsingError = true;
361 mErrorString = *raptorParsingErrorString;
362 delete raptorParsingErrorString;
363 raptorParsingErrorString = nullptr;
364 }
365
366
367 if (mParsingError) {
368 qDeleteAll(triplesDataVec);
369 } else {
370 for (CRedlandStoredTriplesData* tripleData : triplesDataVec) {
371 mOntologyBuilder->addTriplesData(tripleData);
372 }
373 }
374
375 return !mParsingError;
376 }
377
378
379
380 }; // end namespace Parser
381
382 }; // end namespace Konclude
383
384 #endif // !KONCLUDE_REDLAND_INTEGRATION