Debian Codegraph @

Source/Reasoner/Revision/Persistence/COntologyRevisionRedlandExternalTriplesSPARQLServicePersistencer.cpp

raw
1 /*
2 * Copyright (C) 2013-2015, 2019 by the Konclude Developer Team.
3 *
4 * This file is part of the reasoning system Konclude.
5 * For details and support, see <http://konclude.com/>.
6 *
7 * Konclude is free software: you can redistribute it and/or modify
8 * it under the terms of version 3 of the GNU Lesser General Public
9 * License (LGPLv3) as published by the Free Software Foundation.
10 *
11 * Konclude is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU (Lesser) General Public License for more details.
15 *
16 * You should have received a copy of the GNU (Lesser) General Public
17 * License along with Konclude. If not, see <http://www.gnu.org/licenses/>.
18 *
19 */
20 #ifdef KONCLUDE_REDLAND_INTEGRATION
21
22 #include "COntologyRevisionRedlandExternalTriplesSPARQLServicePersistencer.h"
23
24
25 namespace Konclude {
26
27 namespace Reasoner {
28
29 namespace Revision {
30
31 namespace Persistence {
32
33 COntologyRevisionRedlandExternalTriplesSPARQLServicePersistencer::COntologyRevisionRedlandExternalTriplesSPARQLServicePersistencer(CConfiguration* config) : CLogIdentifier("::Konclude::Reasoner::Persistence::RedlandExternalTriplesSPARQLServicePersistencer", this) {
34 mConfig = config;
35
36
37 mDefaultDatatypeString = PREFIX_XML_STRING_DATATYPE;
38 mBlankNodeIdentifierPrefix = QString("kbni_%1_").arg(QUuid::createUuid().toString().replace("{", "").replace("}", ""));;
39
40
41 mConfInitialLoading = CConfigDataReader::readConfigBoolean(mConfig, "Konclude.Persistence.RedlandExternalTriplesSPARQLServicePersistencer.InitialLoading");
42
43 mRDFSerializingFormat = QString("turtle");
44
45 mLoadingLimit = CConfigDataReader::readConfigInteger(mConfig, "Konclude.Persistence.RedlandExternalTriplesSPARQLServicePersistencer.LoadingLimitSize");
46 mPersistingStepSize = CConfigDataReader::readConfigInteger(mConfig, "Konclude.Persistence.RedlandExternalTriplesSPARQLServicePersistencer.PersistingStepSize");
47
48 mServiceUriData = CConfigDataReader::readConfigString(mConfig, "Konclude.Persistence.RedlandExternalTriplesSPARQLServicePersistencer.ServiceAddress").toUtf8();
49 mCheckingQueryString = CConfigDataReader::readConfigString(mConfig, "Konclude.Persistence.RedlandExternalTriplesSPARQLServicePersistencer.CheckingQueryTemplate");
50 mLoadingQueryString = CConfigDataReader::readConfigString(mConfig, "Konclude.Persistence.RedlandExternalTriplesSPARQLServicePersistencer.LoadingQueryTemplate");
51 mPersistQueryString = CConfigDataReader::readConfigString(mConfig, "Konclude.Persistence.RedlandExternalTriplesSPARQLServicePersistencer.PersistingQueryTemplate");
52 mDeletingQueryString = CConfigDataReader::readConfigString(mConfig, "Konclude.Persistence.RedlandExternalTriplesSPARQLServicePersistencer.DeletingQueryTemplate");
53 mCreatingQueryString = CConfigDataReader::readConfigString(mConfig, "Konclude.Persistence.RedlandExternalTriplesSPARQLServicePersistencer.CreatingQueryTemplate");
54
55 mDatabaseNameString = CConfigDataReader::readConfigString(mConfig, "Konclude.Persistence.RedlandExternalTriplesSPARQLServicePersistencer.TriplesDatabaseName");
56
57
58 mConfPersistPostBatchFormEncoded = CConfigDataReader::readConfigBoolean(mConfig, "Konclude.Persistence.RedlandExternalTriplesSPARQLServicePersistencer.HTTPPostFormEncodedPersisting");
59
60 if (mConfPersistPostBatchFormEncoded) {
61 mHttpManager = new CQtHttpTransactionManager();
62 } else {
63 mHttpManager = nullptr;
64 }
65 }
66
67 COntologyRevisionRedlandExternalTriplesSPARQLServicePersistencer::~COntologyRevisionRedlandExternalTriplesSPARQLServicePersistencer() {
68 if (mHttpManager) {
69 mHttpManager->stopThread(true);
70 delete mHttpManager;
71 }
72 }
73
74
75
76 COntologyRevisionRedlandExternalStoredTriplesConnectionData* COntologyRevisionRedlandExternalTriplesSPARQLServicePersistencer::getOntologyPersistingTriplesData(const QString& ontName) {
77
78 COntologyRevisionRedlandExternalStoredTriplesConnectionData* triplesData = mOntologyTriplesDataHash.value(ontName);
79
80 if (!triplesData) {
81
82 triplesData = new COntologyRevisionRedlandExternalStoredTriplesConnectionData();
83 mOntologyTriplesDataHash.insert(ontName, triplesData);
84
85 }
86 return triplesData;
87 }
88
89
90
91
92
93
94 CRedlandStoredTriplesData* COntologyRevisionRedlandExternalTriplesSPARQLServicePersistencer::getUpdatingTripleData() {
95
96 bool localTriplesDataFlag = false;
97 CRedlandStoredTriplesData* tripleData = nullptr;
98
99 if (!tripleData) {
100 tripleData = new CRedlandStoredTriplesData();
101 tripleData->initTriplesData(TRIPLES_DATA_ADDITION, nullptr);
102 }
103
104 librdf_world* world = tripleData->getRedlandWorld();
105 if (!world) {
106 world = librdf_new_world();
107 librdf_world_open(world);
108 tripleData->setRedlandWorldData(world);
109 }
110
111 librdf_storage* indexedStorage = tripleData->getRedlandIndexedStorage();
112 if (!indexedStorage && world) {
113 indexedStorage = librdf_new_storage(world, "hashes", NULL, "hash-type='memory',index-predicates='yes'");
114 tripleData->setRedlandIndexedStorageData(indexedStorage);
115 }
116
117 librdf_model* indexedModel = tripleData->getRedlandIndexedModel();
118 if (!indexedModel && indexedStorage) {
119 indexedModel = librdf_new_model(world, indexedStorage, NULL);
120 tripleData->setRedlandIndexedModelData(indexedModel);
121 }
122 /*else {
123 model = librdf_new_model_from_model(prevModel);
124 tripleData->setRedlandModelData(model);
125 }*/
126
127 if (!indexedModel) {
128 delete tripleData;
129 tripleData = nullptr;
130 }
131
132 return tripleData;
133
134 }
135
136
137
138
139 bool COntologyRevisionRedlandExternalTriplesSPARQLServicePersistencer::persistOntologyRevision(COntologyRevision* ontoRev, COntologyRevision* prevOntoRev) {
140
141 if (mOntologyNameFileHash.contains(prevOntoRev) && ontoRev->getOntology()->getBuildData()->getChangeAxiomList()->size() == prevOntoRev->getOntology()->getBuildData()->getChangeAxiomList()->size()) {
142 COntologyTriplesData* triplesData = ontoRev->getOntology()->getOntologyTriplesData();
143 COntologyTriplesData* prevTriplesData = prevOntoRev->getOntology()->getOntologyTriplesData();
144 if (prevTriplesData && triplesData && triplesData->getLatestTriplesData(false) == prevTriplesData->getLatestTriplesData(false) || !prevTriplesData && !triplesData) {
145 mOntologyNameFileHash.insert(ontoRev, mOntologyNameFileHash.value(prevOntoRev));
146 return true;
147 }
148 }
149
150 CConcreteOntology* ontology = ontoRev->getOntology();
151 QString ontName = ontology->getOntologyName();
152 cint64 ontoVers = ontoRev->getOntologyRevisionDepth();
153
154 COntologyRevisionRedlandExternalStoredTriplesConnectionData* revTriplesData = getOntologyPersistingTriplesData(ontName);
155 if (revTriplesData) {
156 COntologyAxiomExpressionsRenderVisitor renderer;
157 COntologyRedlandRDFRenderer rdfRenderer(nullptr);
158
159 CONTOLOGYAXIOMSET<CAxiomExpression*>* currAxiomSet = ontology->getBuildData()->getTellAxiomSet();
160 CQtList<CAxiomExpression*> axiomList;
161 for (CAxiomExpression* axiomExp : *currAxiomSet) {
162 if (!revTriplesData->getPersistedAxiomExpressionSet()->contains(axiomExp)) {
163 axiomList.append(axiomExp);
164 }
165 }
166 QList<CTriplesData*> triplesDataList;
167 COntologyTriplesData* ontTriplesData = ontoRev->getOntology()->getOntologyTriplesData();
168 if (ontTriplesData) {
169 for (CTriplesData* tripleData : *ontTriplesData->getAllTriplesData()) {
170 if (!revTriplesData->getPersistedTriplesDataSet()->contains(tripleData)) {
171 triplesDataList.append(tripleData);
172 }
173 }
174 }
175
176
177 LOG(INFO, getLogDomain(), logTr("Preparing new revision %2 of ontology '%1' for storing in external triple store").arg(ontName).arg(ontoVers), this);
178 renderer.visitAndRender(axiomList, ontology, &rdfRenderer);
179 renderer.visitAndRender(&triplesDataList, ontology, &rdfRenderer);
180
181 if (!revTriplesData->isInitiallyLoaded()) {
182 if (hasOntologyPersisted(ontName)) {
183 if (mConfInitialLoading) {
184 loadLatestOntologyRevision(ontName, ontoRev);
185 }
186 } else {
187 createPersistingOntology(ontName);
188 }
189 }
190 revTriplesData->setInitiallyLoaded(true);
191
192 LOG(INFO, getLogDomain(), logTr("Trying saving new revision %2 of ontology '%1' in external triple store").arg(ontName).arg(ontoVers), this);
193 CRedlandStoredTriplesData* triplesData = (COntologyRevisionRedlandExternalStoredTriplesConnectionData*)rdfRenderer.getRenderedTripleData(nullptr);
194
195
196 QByteArray serFormByteArray = mRDFSerializingFormat.toUtf8();
197
198 unsigned char* modelString = librdf_model_to_string(triplesData->getRedlandIndexedModel(), nullptr, serFormByteArray.constData(), nullptr, nullptr);
199 QStringList tripleLines = QString::fromUtf8((char*)modelString).split("\n");
200
201
202 QStringList prefixLines;
203
204
205
206
207 QString firstTripleLine = tripleLines.first().trimmed();
208 while (!tripleLines.isEmpty() && firstTripleLine.toUpper().startsWith("PREFIX") || firstTripleLine.toUpper().startsWith("@PREFIX")) {
209 if (firstTripleLine.toUpper().startsWith("PREFIX")) {
210 QString line = firstTripleLine.mid(6);
211 if (line.endsWith(".")) {
212 line = line.mid(0, line.length() - 1);
213 }
214 QString prefixLine = QString("PREFIX %1").arg(line);
215 prefixLines.append(prefixLine);
216 } else if (firstTripleLine.toUpper().startsWith("@PREFIX")) {
217 QString line = firstTripleLine.mid(7);
218 if (line.endsWith(".")) {
219 line = line.mid(0, line.length() - 1);
220 }
221 QString prefixLine = QString("PREFIX %1").arg(line);
222 prefixLines.append(prefixLine);
223 }
224 tripleLines.takeFirst();
225 if (!tripleLines.isEmpty()) {
226 firstTripleLine = tripleLines.first().trimmed();
227 }
228 }
229
230
231 cint64 triplesSentCount = 0;
232 cint64 persistingStep = 0;
233 while (!tripleLines.isEmpty()) {
234 persistingStep++;
235
236 cint64 persistingCount = 0;
237 QStringList persistingTripleLines;
238 bool breakPoint = false;
239 while (!tripleLines.isEmpty() && (persistingCount < mPersistingStepSize || !breakPoint || mPersistingStepSize <= 0)) {
240 QString line = tripleLines.takeFirst().trimmed();
241 if (!line.isEmpty()) {
242 persistingTripleLines.append(line);
243 if (line.endsWith(".")) {
244 breakPoint = true;
245 } else if (!line.isEmpty()) {
246 breakPoint = false;
247 }
248 triplesSentCount++;
249 }
250 persistingCount++;
251 }
252
253 if (!persistingTripleLines.isEmpty()) {
254 QString persistQueryString = mPersistQueryString;
255 QString encodedOntologyName = ontName;
256 persistQueryString = replaceQueryStringVariables(persistQueryString, encodedOntologyName, mConfPersistPostBatchFormEncoded);
257 persistQueryString = persistQueryString.replace("$PrefixesString", prefixLines.join("\r\n"));
258 persistQueryString = persistQueryString.replace("$TriplesString", persistingTripleLines.join("\r\n"));
259
260 if (mConfPersistPostBatchFormEncoded) {
261 sendPersistingHTTPPostFormEncodedRequest(persistQueryString, ontName);
262 } else {
263 sendPersistingRedlandClientRequest(persistQueryString, ontName);
264 }
265
266 }
267 }
268
269 LOG(INFO, getLogDomain(), logTr("Saved new revision %2 of ontology '%1' in external triple store by sending %3 triples").arg(ontName).arg(ontoVers).arg(triplesSentCount), this);
270
271 for (CAxiomExpression* axiomExp : *currAxiomSet) {
272 revTriplesData->getPersistedAxiomExpressionSet()->insert(axiomExp);
273 }
274 if (ontTriplesData) {
275 for (CTriplesData* tripleData : *ontTriplesData->getAllTriplesData()) {
276 revTriplesData->getPersistedTriplesDataSet()->insert(tripleData);
277 }
278 }
279
280 CXLinker<librdf_statement*>* statementLinker = triplesData->getRedlandStatementLinker();
281 for (CXLinker<librdf_statement*>* statementLinkerIt = statementLinker; statementLinkerIt; ) {
282 CXLinker<librdf_statement*>* tmpStatementLinker = statementLinkerIt;
283 statementLinkerIt = statementLinkerIt->getNext();
284
285 librdf_statement* statement = tmpStatementLinker->getData();
286 librdf_free_statement(statement);
287
288 delete tmpStatementLinker;
289 }
290 triplesData->setRedlandStatementLinker(nullptr);
291 delete triplesData;
292 }
293
294 return true;
295 }
296
297
298
299 bool COntologyRevisionRedlandExternalTriplesSPARQLServicePersistencer::sendPersistingHTTPPostFormEncodedRequest(const QString &persistQueryString, const QString& ontName) {
300
301
302 // may require some additional grants for user SPARQL for virtuoso, e.g., (can be executed in the SQL command line)
303 //GRANT EXECUTE ON DB.DBA.SPARQL_INSERT_CTOR_ACC TO "SPARQL";
304 //GRANT EXECUTE ON DB.DBA.SPARQL_INS_OR_DEL_OR_MODIFY_CTOR_FIN TO "SPARQL";
305
306
307 QString databaseNameString = replaceQueryStringVariables(mDatabaseNameString, ontName);
308
309 QByteArray postData;
310
311 QUrlQuery query;
312 query.addQueryItem("default-graph-uri", databaseNameString.replace("%","%25"));
313 query.addQueryItem("query", persistQueryString);
314
315 postData = query.toString(QUrl::FullyEncoded).toUtf8();
316
317 CHttpRequest* request = mHttpManager->createRequest(mServiceUriData, postData);
318 mHttpManager->setRequestHeader(request, "Content-Type", "application/x-www-form-urlencoded");
319
320 CHttpResponse* response = mHttpManager->getResponse(request);
321 mHttpManager->waitFinished(response);
322
323 QString* resText = mHttpManager->getExtractedText(response);
324 QString* errorString = mHttpManager->getErrorString(response);
325 delete resText;
326 delete errorString;
327
328
329 mHttpManager->releaseResponse(response, true);
330 return true;
331 }
332
333
334
335 bool COntologyRevisionRedlandExternalTriplesSPARQLServicePersistencer::sendPersistingRedlandClientRequest(const QString &persistQueryString, const QString& ontName) {
336 QByteArray persitQueryData = persistQueryString.toUtf8();
337
338
339 CRedlandStoredTriplesData* newTriplesData = getUpdatingTripleData();
340 librdf_world* world = newTriplesData->getRedlandWorld();
341
342
343 rasqal_service* rasqalService = getRasqalService(world, ontName, persitQueryData);
344
345
346 rasqal_query_results* results = rasqal_service_execute(rasqalService);
347
348 // may require some additional grants for user SPARQL for virtuoso, e.g., (can be executed in the SQL command line)
349 //GRANT EXECUTE ON DB.DBA.SPARQL_INSERT_DICT_CONTENT TO "SPARQL";
350 //GRANT EXECUTE ON DB.DBA.L_O_LOOK TO "SPARQL";
351
352 //if (!rasqal_query_results_finished(results)) {
353 // bool error = true;
354 //}
355
356 if (results) {
357 rasqal_free_query_results(results);
358 }
359
360 rasqal_free_service(rasqalService);
361 delete newTriplesData;
362 return true;
363 }
364
365 QString COntologyRevisionRedlandExternalTriplesSPARQLServicePersistencer::replaceQueryStringVariables(QString string, const QString& ontologyName, bool doublePercentageEncoding) {
366 if (string.contains("$KnowledgeBaseName")) {
367 string = string.replace("$KnowledgeBaseName", ontologyName);
368 }
369 if (string.contains("$PercentEncodedKnowledgeBaseName")) {
370 QString encodedOntologyName = QUrl::toPercentEncoding(ontologyName);
371 if (doublePercentageEncoding) {
372 // qt somehow does not encode the % symbol correctly
373 encodedOntologyName = encodedOntologyName.replace("%", "%25");
374 }
375 string = string.replace("$PercentEncodedKnowledgeBaseName", encodedOntologyName);
376 }
377 return string;
378 }
379
380
381
382 bool COntologyRevisionRedlandExternalTriplesSPARQLServicePersistencer::hasOntologyPersisted(const QString& ontologyName) {
383
384 bool hasOntologyPersisted = false;
385
386
387 QString checkingQueryString = mCheckingQueryString;
388 checkingQueryString = replaceQueryStringVariables(checkingQueryString, ontologyName);
389 QByteArray checkingQueryData = checkingQueryString.toUtf8();
390
391 CRedlandStoredTriplesData* newTriplesData = getUpdatingTripleData();
392 librdf_world* world = newTriplesData->getRedlandWorld();
393
394 rasqal_service* rasqalService = getRasqalService(world, ontologyName, checkingQueryData);
395
396
397 rasqal_query_results* results = rasqal_service_execute(rasqalService);
398
399
400
401 CXLinker<librdf_statement*>* statmentLinker = nullptr;
402 CXLinker<librdf_statement*>* lastStatementLinker = nullptr;
403
404
405 if (results && rasqal_query_results_is_bindings(results)) {
406
407 if (!rasqal_query_results_finished(results)) {
408 hasOntologyPersisted = true;
409 }
410 }
411 if (results) {
412 rasqal_free_query_results(results);
413 }
414
415 rasqal_free_service(rasqalService);
416 delete newTriplesData;
417
418
419 return hasOntologyPersisted;
420 }
421
422
423
424 bool COntologyRevisionRedlandExternalTriplesSPARQLServicePersistencer::hasNewOntologyRevision(const QString& ontologyName, COntologyRevision* ontoRev) {
425
426 if (hasOntologyPersisted(ontologyName)) {
427 COntologyRevisionRedlandExternalStoredTriplesConnectionData* triplesData = getOntologyPersistingTriplesData(ontologyName);
428 if (triplesData && !triplesData->isInitiallyLoaded()) {
429 return true;
430 }
431 }
432
433 return false;
434 }
435
436
437 bool COntologyRevisionRedlandExternalTriplesSPARQLServicePersistencer::deletePersistedOntology(const QString& ontologyName) {
438
439 QByteArray deleteQueryData = mDeletingQueryString.toUtf8();
440 deleteQueryData = replaceQueryStringVariables(deleteQueryData, ontologyName).toUtf8();
441
442
443 CRedlandStoredTriplesData* newTriplesData = getUpdatingTripleData();
444 librdf_world* world = newTriplesData->getRedlandWorld();
445
446
447 rasqal_service* rasqalService = getRasqalService(world, ontologyName, deleteQueryData);
448
449
450 rasqal_query_results* results = rasqal_service_execute(rasqalService);
451
452 if (results) {
453 rasqal_free_query_results(results);
454 }
455
456 rasqal_free_service(rasqalService);
457 delete newTriplesData;
458
459
460 LOG(INFO, getLogDomain(), logTr("Deleted persisted version of ontology '%1' in external triple store").arg(ontologyName), this);
461
462 return true;
463 }
464
465
466 bool COntologyRevisionRedlandExternalTriplesSPARQLServicePersistencer::createPersistingOntology(const QString& ontologyName) {
467
468 QByteArray creatingQueryData = mCreatingQueryString.toUtf8();
469 creatingQueryData = replaceQueryStringVariables(creatingQueryData, ontologyName).toUtf8();
470
471 CRedlandStoredTriplesData* newTriplesData = getUpdatingTripleData();
472 librdf_world* world = newTriplesData->getRedlandWorld();
473
474 rasqal_service* rasqalService = getRasqalService(world, ontologyName, creatingQueryData);
475
476 rasqal_query_results* results = rasqal_service_execute(rasqalService);
477
478 if (results) {
479 rasqal_free_query_results(results);
480 }
481
482 rasqal_free_service(rasqalService);
483 delete newTriplesData;
484 LOG(INFO, getLogDomain(), logTr("Created database for persisting ontology '%1' in external triple store").arg(ontologyName), this);
485
486 return true;
487 }
488
489
490 librdf_node* COntologyRevisionRedlandExternalTriplesSPARQLServicePersistencer::createIRIResourceNode(const QString& iri, librdf_world* world) {
491 QByteArray bytes = iri.toUtf8();
492 raptor_uri* uri = (raptor_uri*)librdf_new_uri(world, (const unsigned char*)bytes.constData());
493 librdf_node* node = librdf_new_node_from_uri(world, uri);
494 return node;
495 }
496
497
498 librdf_node* COntologyRevisionRedlandExternalTriplesSPARQLServicePersistencer::createBlankNode(const QString& identifier, librdf_world* world) {
499 QByteArray bytes = identifier.toUtf8();
500 librdf_node* node = librdf_new_node_from_blank_identifier(world, (const unsigned char*)bytes.constData());
501 return node;
502 }
503
504 librdf_node* COntologyRevisionRedlandExternalTriplesSPARQLServicePersistencer::createBlankNode(cint64 id, librdf_world* world) {
505 QString identifier = QString("%1%2").arg(mBlankNodeIdentifierPrefix).arg(id);
506 QByteArray bytes = identifier.toUtf8();
507 librdf_node* node = librdf_new_node_from_blank_identifier(world, (const unsigned char*)bytes.constData());
508 return node;
509 }
510
511 librdf_node* COntologyRevisionRedlandExternalTriplesSPARQLServicePersistencer::createLiteralNode(const QString& literal, const QString& datatype, librdf_world* world) {
512 QByteArray datatypeBytes = datatype.toUtf8();
513 librdf_uri* datatypeUri = nullptr;
514 if (datatype != mDefaultDatatypeString) {
515 librdf_new_uri(world, (const unsigned char*)datatypeBytes.constData());
516 }
517 QByteArray literalBytes = literal.toUtf8();
518 librdf_node* node = librdf_new_node_from_typed_literal(world, (const unsigned char*)literalBytes.constData(), NULL, datatypeUri);
519 return node;
520 };
521
522
523
524
525 bool COntologyRevisionRedlandExternalTriplesSPARQLServicePersistencer::loadLatestOntologyRevision(const QString& ontologyName, COntologyRevision* ontoRev) {
526 COntologyRevisionRedlandExternalStoredTriplesConnectionData* triplesData = getOntologyPersistingTriplesData(ontologyName);
527 if (triplesData && !triplesData->isInitiallyLoaded() && mConfInitialLoading && hasOntologyPersisted(ontologyName)) {
528
529 LOG(INFO, getLogDomain(), logTr("Trying loading of initial revision of ontology '%1' from external SPARQL service").arg(ontologyName), this);
530
531 CConcreteOntologyUpdateCollectorBuilder *builder = new CConcreteOntologyUpdateCollectorBuilder(ontoRev->getOntology());
532 builder->initializeBuilding();
533
534 CRedlandStoredTriplesData* newTriplesData = nullptr;
535 cint64 statementCount = 0;
536 newTriplesData = getUpdatingTripleData();
537
538 cint64 offset = 0;
539 bool continueLoading = true;
540 while (continueLoading) {
541 continueLoading = false;
542 bool hasLoadingLimit = false;
543
544
545 QString loadingQueryString = mLoadingQueryString;
546 loadingQueryString = replaceQueryStringVariables(loadingQueryString, ontologyName);
547 if (loadingQueryString.contains("$LoadingLimit") && loadingQueryString.contains("$LoadingOffset")) {
548 hasLoadingLimit = true;
549 loadingQueryString = loadingQueryString.replace("$LoadingLimit", QString::number(mLoadingLimit));
550 loadingQueryString = loadingQueryString.replace("$LoadingOffset", QString::number(offset));
551 offset += mLoadingLimit;
552 }
553 QByteArray loadingQueryData = loadingQueryString.toUtf8();
554
555 librdf_world* world = newTriplesData->getRedlandWorld();
556 rasqal_service* rasqalService = getRasqalService(world, ontologyName, loadingQueryData);
557
558
559 rasqal_query_results* results = rasqal_service_execute(rasqalService);
560
561
562
563 CXLinker<librdf_statement*>* statmentLinker = nullptr;
564 CXLinker<librdf_statement*>* lastStatementLinker = nullptr;
565
566
567 if (results && rasqal_query_results_is_bindings(results)) {
568
569 int bindCount = rasqal_query_results_get_bindings_count(results);
570
571
572
573 const unsigned char **names = nullptr;
574 rasqal_literal** values = nullptr;
575
576
577
578 QByteArray plainLiteralBytes = QString(PREFIX_XML_PLAINLITERAL_DATATYPE).toUtf8();
579 librdf_uri *plainLiteralUri = librdf_new_uri(world, (const unsigned char*)plainLiteralBytes.constData());
580
581
582 while (!rasqal_query_results_finished(results)) {
583
584 if (rasqal_query_results_get_bindings(results, &names, &values))
585 break;
586
587 if (names) {
588 cint64 nodeSet = 0;
589 librdf_node* node[3] = { nullptr, nullptr, nullptr };
590 if (statementCount > 10) {
591 bool debug = true;
592 }
593 for (int i = 0; names[i]; i++) {
594 rasqal_literal* valueNode = values[i];
595 if (valueNode) {
596 rasqal_literal_type type = rasqal_literal_get_type(valueNode);
597 if (type == RASQAL_LITERAL_URI) {
598 const unsigned char* chrString = rasqal_literal_as_string(valueNode);
599 QString valueString = QString::fromUtf8((char*)chrString);
600 if (valueString.startsWith("_:")) {
601 node[i] = createBlankNode(valueString.mid(2), world);
602 } else {
603 node[i] = createIRIResourceNode(valueString, world);
604 }
605 nodeSet++;
606 } else if (type == RASQAL_LITERAL_BLANK) {
607 const unsigned char* chrString = rasqal_literal_as_string(valueNode);
608 node[i] = createBlankNode(QString::fromUtf8((char*)chrString), world);
609 nodeSet++;
610 } else if (type == RASQAL_LITERAL_STRING) {
611 const unsigned char* chrString = rasqal_literal_as_string(valueNode);
612 raptor_uri* datatypeUri = rasqal_literal_datatype(valueNode);
613 if (datatypeUri) {
614 const unsigned char* datatypeString = raptor_uri_as_string(datatypeUri);
615 node[i] = createLiteralNode(QString::fromUtf8((char*)chrString), QString::fromUtf8((char*)datatypeString), world);
616 } else {
617 node[i] = createLiteralNode(QString::fromUtf8((char*)chrString), "", world);
618 }
619 nodeSet++;
620 } else {
621 const unsigned char* chrString = rasqal_literal_as_string(valueNode);
622 raptor_uri* datatypeUri = rasqal_literal_datatype(valueNode);
623 if (datatypeUri) {
624 const unsigned char* datatypeString = raptor_uri_as_string(datatypeUri);
625 node[i] = createLiteralNode(QString::fromUtf8((char*)chrString), QString::fromUtf8((char*)datatypeString), world);
626 }
627 }
628 }
629 if (nodeSet >= 3) {
630 if (hasLoadingLimit) {
631 continueLoading = true;
632 }
633 librdf_statement* newStatement = librdf_new_statement_from_nodes(world, node[0], node[1], node[2]);
634 librdf_model_add_statement(newTriplesData->getRedlandIndexedModel(), newStatement);
635
636 CXLinker<librdf_statement*>* tmpRenderedLinker = new CXLinker<librdf_statement *>();
637 tmpRenderedLinker->initLinker(newStatement);
638 statementCount++;
639
640 if (tmpRenderedLinker) {
641 if (!statmentLinker) {
642 statmentLinker = tmpRenderedLinker;
643 lastStatementLinker = statmentLinker->getLastListLink();
644 } else {
645 lastStatementLinker->append(tmpRenderedLinker);
646 lastStatementLinker = tmpRenderedLinker->getLastListLink();
647 }
648 }
649
650 }
651
652 }
653 }
654 rasqal_query_results_next(results);
655 }
656
657 }
658 newTriplesData->setRedlandStatementLinker(statmentLinker);
659
660
661 if (results) {
662 rasqal_free_query_results(results);
663 }
664
665 rasqal_free_service(rasqalService);
666 }
667
668
669 builder->addTriplesData(newTriplesData);
670 triplesData->getPersistedTriplesDataSet()->insert(newTriplesData);
671
672 CConcreteOntologyRedlandTriplesDataExpressionMapper* triplesMapper = new CConcreteOntologyRedlandTriplesDataExpressionMapper(builder);
673 triplesMapper->mapTriples(ontoRev->getOntology(), ontoRev->getOntology()->getOntologyTriplesData());
674
675
676 QSet<CAxiomExpression*>* axioms = builder->getAddedAxiomSet();
677 for (CAxiomExpression* axiom : *axioms) {
678 triplesData->getPersistedAxiomExpressionSet()->insert(axiom);
679 }
680
681 builder->completeBuilding();
682 delete builder;
683
684 LOG(INFO, getLogDomain(), logTr("Loaded initial revision of ontology '%1' from external SPARQL service").arg(ontologyName), this);
685
686 triplesData->setInitiallyLoaded(true);
687 return true;
688 }
689 return false;
690 }
691
692 rasqal_service* COntologyRevisionRedlandExternalTriplesSPARQLServicePersistencer::getRasqalService(librdf_world* world, const QString& ontologyName, QByteArray &queryData) {
693 rasqal_world* rasqalWorld = librdf_world_get_rasqal(world);
694 raptor_uri* serviceUri = (raptor_uri*)librdf_new_uri(world, (const unsigned char*)mServiceUriData.constData());
695
696 QString databaseNameString = replaceQueryStringVariables(mDatabaseNameString, ontologyName);
697 QByteArray databaseNameData = databaseNameString.toUtf8();
698 raptor_uri* databaseNameUri = (raptor_uri*)librdf_new_uri(world, (const unsigned char*)databaseNameData.constData());
699
700
701 rasqal_data_graph* dataGraph = rasqal_new_data_graph_from_uri(rasqalWorld, databaseNameUri, databaseNameUri, RASQAL_DATA_GRAPH_BACKGROUND, NULL, NULL, NULL);
702 raptor_sequence* dataGraphs = raptor_new_sequence(nullptr, nullptr);
703 raptor_sequence_set_at(dataGraphs, 0, dataGraph);
704 rasqal_service* rasqalService = rasqal_new_service(rasqalWorld, serviceUri, (const unsigned char*)queryData.constData(), dataGraphs);
705 return rasqalService;
706 }
707
708
709 }; // end namespace Persistence
710
711 }; // end namespace Revision
712
713 }; // end namespace Reasoner
714
715 }; // end namespace Konclude
716
717 #endif // !KONCLUDE_REDLAND_INTEGRATION