Debian Codegraph @

Source/Reasoner/Revision/Persistence/COntologyRevisionRedlandExternalTriplesDSNOnlyWritingPersistencer.cpp

raw
1 /*
2 * Copyright (C) 2013-2015, 2019 by the Konclude Developer Team.
3 *
4 * This file is part of the reasoning system Konclude.
5 * For details and support, see <http://konclude.com/>.
6 *
7 * Konclude is free software: you can redistribute it and/or modify
8 * it under the terms of version 3 of the GNU Lesser General Public
9 * License (LGPLv3) as published by the Free Software Foundation.
10 *
11 * Konclude is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU (Lesser) General Public License for more details.
15 *
16 * You should have received a copy of the GNU (Lesser) General Public
17 * License along with Konclude. If not, see <http://www.gnu.org/licenses/>.
18 *
19 */
20 #ifdef KONCLUDE_REDLAND_INTEGRATION
21
22 #include "COntologyRevisionRedlandExternalTriplesDSNOnlyWritingPersistencer.h"
23
24
25 namespace Konclude {
26
27 namespace Reasoner {
28
29 namespace Revision {
30
31 namespace Persistence {
32
33 COntologyRevisionRedlandExternalTriplesDSNOnlyWritingPersistencer::COntologyRevisionRedlandExternalTriplesDSNOnlyWritingPersistencer(CConfiguration* config) : CLogIdentifier("::Konclude::Reasoner::Persistence::RedlandExternalTriplesDSNOnlyWritingPersistencer", this) {
34 mConfig = config;
35
36
37
38 mDsnConfig = CConfigDataReader::readConfigString(mConfig, "Konclude.Persistence.RedlandExternalTriplesDSNOnlyWritingPersistencer.DSNConfig");
39 mTriplesDBName = CConfigDataReader::readConfigString(mConfig, "Konclude.Persistence.RedlandExternalTriplesDSNOnlyWritingPersistencer.TriplesDatabaseName");
40
41 mConfInitialLoading = CConfigDataReader::readConfigBoolean(mConfig, "Konclude.Persistence.RedlandExternalTriplesDSNOnlyWritingPersistencer.InitialLoading");
42 mConfDirectFullLoading = CConfigDataReader::readConfigBoolean(mConfig, "Konclude.Persistence.RedlandExternalTriplesDSNOnlyWritingPersistencer.DirectFullTriplesLoading");
43 }
44
45 COntologyRevisionRedlandExternalTriplesDSNOnlyWritingPersistencer::~COntologyRevisionRedlandExternalTriplesDSNOnlyWritingPersistencer() {
46 }
47
48
49
50 COntologyRevisionRedlandExternalStoredTriplesConnectionData* COntologyRevisionRedlandExternalTriplesDSNOnlyWritingPersistencer::getOntologyPersistingTriplesData(const QString& ontName) {
51
52 COntologyRevisionRedlandExternalStoredTriplesConnectionData* triplesData = mOntologyTriplesDataHash.value(ontName);
53
54 if (!triplesData) {
55
56 triplesData = new COntologyRevisionRedlandExternalStoredTriplesConnectionData();
57 initTriplesData(triplesData, ontName);
58 mOntologyTriplesDataHash.insert(ontName, triplesData);
59
60 }
61 return triplesData;
62 }
63
64
65
66
67
68
69 CRedlandStoredTriplesData* COntologyRevisionRedlandExternalTriplesDSNOnlyWritingPersistencer::getUpdatingTripleData() {
70
71 bool localTriplesDataFlag = false;
72 CRedlandStoredTriplesData* tripleData = nullptr;
73
74 if (!tripleData) {
75 tripleData = new CRedlandStoredTriplesData();
76 tripleData->initTriplesData(TRIPLES_DATA_ADDITION, nullptr);
77 }
78
79 librdf_world* world = tripleData->getRedlandWorld();
80 if (!world) {
81 world = librdf_new_world();
82 librdf_world_open(world);
83 tripleData->setRedlandWorldData(world);
84 }
85
86 librdf_storage* indexedStorage = tripleData->getRedlandIndexedStorage();
87 if (!indexedStorage && world) {
88 indexedStorage = librdf_new_storage(world, "hashes", NULL, "hash-type='memory',index-predicates='yes'");
89 tripleData->setRedlandIndexedStorageData(indexedStorage);
90 }
91
92 librdf_model* indexedModel = tripleData->getRedlandIndexedModel();
93 if (!indexedModel && indexedStorage) {
94 indexedModel = librdf_new_model(world, indexedStorage, NULL);
95 tripleData->setRedlandIndexedModelData(indexedModel);
96 }
97 /*else {
98 model = librdf_new_model_from_model(prevModel);
99 tripleData->setRedlandModelData(model);
100 }*/
101
102
103 if (!indexedModel) {
104 delete tripleData;
105 tripleData = nullptr;
106 }
107
108 return tripleData;
109
110 }
111
112
113
114 COntologyRevisionRedlandExternalTriplesDSNOnlyWritingPersistencer* COntologyRevisionRedlandExternalTriplesDSNOnlyWritingPersistencer::initTriplesData(CRedlandStoredTriplesData* triplesData, const QString& ontName) {
115 librdf_world* world = triplesData->getRedlandWorld();
116 if (!world) {
117 world = librdf_new_world();
118 librdf_world_open(world);
119 triplesData->setRedlandWorldData(world);
120 }
121
122
123 librdf_storage* indexedStorage = triplesData->getRedlandIndexedStorage();
124 if (!indexedStorage && world) {
125 //indexedStorage = librdf_new_storage(world, "virtuoso", "http://LUBM-test", "dsn='VirtuosoTest',user='dba',password='dba'");
126
127 QString dsnConfigString = mDsnConfig;
128 QString dbNameString = mTriplesDBName;
129
130 if (dbNameString.contains("$KnowledgeBaseName")) {
131 dbNameString = dbNameString.replace("$KnowledgeBaseName", ontName);
132 }
133 if (dbNameString.contains("$PercentEncodedKnowledgeBaseName")) {
134 dbNameString = dbNameString.replace("$PercentEncodedKnowledgeBaseName", QUrl::toPercentEncoding(ontName));
135 }
136
137 QByteArray triplesDBNameArray = dbNameString.toUtf8();
138 QByteArray dsnConfigArray = dsnConfigString.toUtf8();
139
140 indexedStorage = librdf_new_storage(world, "virtuoso", triplesDBNameArray.constData(), dsnConfigArray.constData());
141 triplesData->setRedlandIndexedStorageData(indexedStorage);
142 }
143
144 librdf_model* indexedModel = triplesData->getRedlandIndexedModel();
145 if (!indexedModel && indexedStorage) {
146 indexedModel = librdf_new_model(world, indexedStorage, NULL);
147 triplesData->setRedlandIndexedModelData(indexedModel);
148 }
149 return this;
150 }
151
152
153
154
155 bool COntologyRevisionRedlandExternalTriplesDSNOnlyWritingPersistencer::persistOntologyRevision(COntologyRevision* ontoRev, COntologyRevision* prevOntoRev) {
156
157 if (mOntologyNameFileHash.contains(prevOntoRev) && ontoRev->getOntology()->getBuildData()->getChangeAxiomList()->size() == prevOntoRev->getOntology()->getBuildData()->getChangeAxiomList()->size()) {
158 COntologyTriplesData* triplesData = ontoRev->getOntology()->getOntologyTriplesData();
159 COntologyTriplesData* prevTriplesData = prevOntoRev->getOntology()->getOntologyTriplesData();
160 if (prevTriplesData && triplesData && triplesData->getLatestTriplesData(false) == prevTriplesData->getLatestTriplesData(false) || !prevTriplesData && !triplesData) {
161 mOntologyNameFileHash.insert(ontoRev, mOntologyNameFileHash.value(prevOntoRev));
162 return true;
163 }
164 }
165
166 CConcreteOntology* ontology = ontoRev->getOntology();
167 QString ontName = ontology->getOntologyName();
168 cint64 ontoVers = ontoRev->getOntologyRevisionDepth();
169
170 COntologyRevisionRedlandExternalStoredTriplesConnectionData* triplesData = getOntologyPersistingTriplesData(ontName);
171 if (triplesData) {
172 COntologyAxiomExpressionsRenderVisitor renderer;
173 COntologyRedlandRDFRenderer rdfRenderer(triplesData->getRedlandWorld());
174
175 CONTOLOGYAXIOMSET<CAxiomExpression*>* currAxiomSet = ontology->getBuildData()->getTellAxiomSet();
176 CQtList<CAxiomExpression*> axiomList;
177 for (CAxiomExpression* axiomExp : *currAxiomSet) {
178 if (!triplesData->getPersistedAxiomExpressionSet()->contains(axiomExp)) {
179 axiomList.append(axiomExp);
180 }
181 }
182 QList<CTriplesData*> triplesDataList;
183 COntologyTriplesData* ontTriplesData = ontoRev->getOntology()->getOntologyTriplesData();
184 if (ontTriplesData) {
185 for (CTriplesData* tripleData : *ontTriplesData->getAllTriplesData()) {
186 if (!triplesData->getPersistedTriplesDataSet()->contains(tripleData)) {
187 triplesDataList.append(tripleData);
188 }
189 }
190 }
191
192
193 LOG(INFO, getLogDomain(), logTr("Preparing new revision %2 of ontology '%1' for storing in external triple store").arg(ontName).arg(ontoVers), this);
194 renderer.visitAndRender(axiomList, ontology, &rdfRenderer);
195 renderer.visitAndRender(&triplesDataList, ontology, &rdfRenderer);
196
197 if (!triplesData->isInitiallyLoaded() && mConfInitialLoading) {
198 loadLatestOntologyRevision(ontName, ontoRev);
199 }
200 triplesData->setInitiallyLoaded(true);
201
202 LOG(INFO, getLogDomain(), logTr("Trying saving new revision %2 of ontology '%1' in external triple store").arg(ontName).arg(ontoVers), this);
203 triplesData = (COntologyRevisionRedlandExternalStoredTriplesConnectionData*)rdfRenderer.getRenderedTripleData(triplesData);
204 LOG(INFO, getLogDomain(), logTr("Saved new revision %2 of ontology '%1' in external triple store").arg(ontName).arg(ontoVers), this);
205
206
207 CXLinker<librdf_statement*>* statementLinker = triplesData->getRedlandStatementLinker();
208 for (CAxiomExpression* axiomExp : *currAxiomSet) {
209 triplesData->getPersistedAxiomExpressionSet()->insert(axiomExp);
210 }
211 if (ontTriplesData) {
212 for (CTriplesData* tripleData : *ontTriplesData->getAllTriplesData()) {
213 triplesData->getPersistedTriplesDataSet()->insert(tripleData);
214 }
215 }
216
217 for (CXLinker<librdf_statement*>* statementLinkerIt = statementLinker; statementLinkerIt; ) {
218 CXLinker<librdf_statement*>* tmpStatementLinker = statementLinkerIt;
219 statementLinkerIt = statementLinkerIt->getNext();
220
221 librdf_statement* statement = tmpStatementLinker->getData();
222 librdf_free_statement(statement);
223
224 delete tmpStatementLinker;
225 }
226 triplesData->setRedlandStatementLinker(nullptr);
227 }
228
229 return true;
230 }
231
232
233
234 bool COntologyRevisionRedlandExternalTriplesDSNOnlyWritingPersistencer::hasOntologyPersisted(const QString& ontologyName) {
235
236 COntologyRevisionRedlandExternalStoredTriplesConnectionData* triplesData = getOntologyPersistingTriplesData(ontologyName);
237 bool hasOntologyPersisted = false;
238 if (triplesData) {
239
240 librdf_stream* stream = librdf_model_as_stream(triplesData->getRedlandIndexedModel());
241 if (stream) {
242 if (!librdf_stream_end(stream)) {
243 hasOntologyPersisted = true;
244 }
245 librdf_free_stream(stream);
246 }
247 }
248
249 return hasOntologyPersisted;
250 }
251
252
253
254 bool COntologyRevisionRedlandExternalTriplesDSNOnlyWritingPersistencer::hasNewOntologyRevision(const QString& ontologyName, COntologyRevision* ontoRev) {
255
256 if (hasOntologyPersisted(ontologyName)) {
257 COntologyRevisionRedlandExternalStoredTriplesConnectionData* triplesData = getOntologyPersistingTriplesData(ontologyName);
258 if (triplesData && !triplesData->isInitiallyLoaded()) {
259 return true;
260 }
261 }
262
263 return false;
264 }
265
266
267 bool COntologyRevisionRedlandExternalTriplesDSNOnlyWritingPersistencer::deletePersistedOntology(const QString& ontologyName) {
268 CRedlandStoredTriplesData* triplesData = new CRedlandStoredTriplesData();
269 initTriplesData(triplesData, ontologyName);
270
271 librdf_stream* stream = librdf_model_as_stream(triplesData->getRedlandIndexedModel());
272 if (stream) {
273 while (!librdf_stream_end(stream)) {
274 librdf_statement* statement = librdf_stream_get_object(stream);
275 librdf_model_remove_statement(triplesData->getRedlandIndexedModel(), statement);
276 librdf_stream_next(stream);
277 }
278 librdf_free_stream(stream);
279 }
280 delete triplesData;
281 return true;
282 }
283
284 librdf_node* COntologyRevisionRedlandExternalTriplesDSNOnlyWritingPersistencer::getLocalizedNode(librdf_node* node, CRedlandStoredTriplesData* newTriplesData) {
285
286
287 librdf_world* newWorld = newTriplesData->getRedlandWorld();
288
289 if (librdf_node_is_resource(node)) {
290
291 raptor_uri* prevUri = librdf_node_get_uri(node);
292 raptor_uri* newUri = (raptor_uri*)librdf_new_uri(newWorld, librdf_uri_as_string(prevUri));
293 librdf_node* newNode = librdf_new_node_from_uri(newWorld, newUri);
294 return newNode;
295
296 } else if (librdf_node_is_blank(node)) {
297 librdf_node* newNode = librdf_new_node_from_blank_identifier(newWorld, librdf_node_get_blank_identifier(node));
298 return newNode;
299
300 } else if (librdf_node_is_literal(node)) {
301 unsigned char* literalValue = librdf_node_get_literal_value(node);
302 raptor_uri* datatypeUri = librdf_node_get_literal_value_datatype_uri(node);
303 raptor_uri* newDatatypeUri = nullptr;
304
305 if (datatypeUri) {
306 newDatatypeUri = (raptor_uri*)librdf_new_uri(newWorld, librdf_uri_as_string(datatypeUri));
307 } else {
308 newDatatypeUri = (raptor_uri*)librdf_new_uri(newWorld, (unsigned char*)PREFIX_XML_STRING_DATATYPE);
309 }
310
311 librdf_node* newNode = librdf_new_node_from_typed_literal(newWorld, literalValue, NULL, newDatatypeUri);
312
313 return newNode;
314
315 }
316 return nullptr;
317 }
318
319
320 bool COntologyRevisionRedlandExternalTriplesDSNOnlyWritingPersistencer::loadLatestOntologyRevision(const QString& ontologyName, COntologyRevision* ontoRev) {
321 COntologyRevisionRedlandExternalStoredTriplesConnectionData* triplesData = getOntologyPersistingTriplesData(ontologyName);
322 if (triplesData && !triplesData->isInitiallyLoaded() && mConfInitialLoading && hasOntologyPersisted(ontologyName)) {
323
324 LOG(INFO, getLogDomain(), logTr("Trying loading of initial revision of ontology '%1' from external triple store").arg(ontologyName), this);
325
326 CConcreteOntologyUpdateCollectorBuilder *builder = new CConcreteOntologyUpdateCollectorBuilder(ontoRev->getOntology());
327 builder->initializeBuilding();
328
329 CRedlandStoredTriplesData* newTriplesData = nullptr;
330 cint64 statementCount = 0;
331 if (mConfDirectFullLoading) {
332 newTriplesData = getUpdatingTripleData();
333
334 CRedlandStoredTriplesData* refTriplesData = new CRedlandStoredTriplesData();
335 refTriplesData->initTriplesData(TRIPLES_DATA_ADDITION);
336 initTriplesData(refTriplesData, ontologyName);
337
338 CXLinker<librdf_statement*>* statmentLinker = nullptr;
339 CXLinker<librdf_statement*>* lastStatementLinker = nullptr;
340
341 librdf_stream* stream = librdf_model_as_stream(refTriplesData->getRedlandIndexedModel());
342 if (stream) {
343 while (!librdf_stream_end(stream)) {
344 librdf_statement* statement = librdf_stream_get_object(stream);
345
346 librdf_node* subjectNode = librdf_statement_get_subject(statement);
347 librdf_node* locSubjectNode = getLocalizedNode(subjectNode, newTriplesData);
348
349 librdf_node* predicateNode = librdf_statement_get_predicate(statement);
350 librdf_node* locPredicateNode = getLocalizedNode(predicateNode, newTriplesData);
351
352 librdf_node* objectNode = librdf_statement_get_object(statement);
353 librdf_node* locObjectNode = getLocalizedNode(objectNode, newTriplesData);
354
355 librdf_statement* newStatement = librdf_new_statement_from_nodes(newTriplesData->getRedlandWorld(), locSubjectNode, locPredicateNode, locObjectNode);
356 librdf_model_add_statement(newTriplesData->getRedlandIndexedModel(), newStatement);
357
358 CXLinker<librdf_statement*>* tmpRenderedLinker = new CXLinker<librdf_statement *>();
359 tmpRenderedLinker->initLinker(newStatement);
360 statementCount++;
361
362 if (tmpRenderedLinker) {
363 if (!statmentLinker) {
364 statmentLinker = tmpRenderedLinker;
365 lastStatementLinker = statmentLinker->getLastListLink();
366 } else {
367 lastStatementLinker->append(tmpRenderedLinker);
368 lastStatementLinker = tmpRenderedLinker->getLastListLink();
369 }
370 }
371
372
373 librdf_stream_next(stream);
374 }
375 librdf_free_stream(stream);
376 newTriplesData->setRedlandStatementLinker(statmentLinker);
377 }
378 delete refTriplesData;
379
380
381 } else {
382 newTriplesData = new CRedlandStoredTriplesData();
383 newTriplesData->initTriplesData(TRIPLES_DATA_ADDITION);
384 initTriplesData(newTriplesData, ontologyName);
385 }
386
387
388 builder->addTriplesData(newTriplesData);
389 triplesData->getPersistedTriplesDataSet()->insert(newTriplesData);
390
391 CConcreteOntologyRedlandTriplesDataExpressionMapper* triplesMapper = new CConcreteOntologyRedlandTriplesDataExpressionMapper(builder);
392 triplesMapper->mapTriples(ontoRev->getOntology(), ontoRev->getOntology()->getOntologyTriplesData());
393
394
395 QSet<CAxiomExpression*>* axioms = builder->getAddedAxiomSet();
396 for (CAxiomExpression* axiom : *axioms) {
397 triplesData->getPersistedAxiomExpressionSet()->insert(axiom);
398 }
399
400 builder->completeBuilding();
401 delete builder;
402
403 LOG(INFO, getLogDomain(), logTr("Loaded initial revision of ontology '%1' from external triple store").arg(ontologyName), this);
404
405 triplesData->setInitiallyLoaded(true);
406 return true;
407 }
408 return false;
409 }
410
411
412
413 }; // end namespace Persistence
414
415 }; // end namespace Revision
416
417 }; // end namespace Reasoner
418
419 }; // end namespace Konclude
420
421 #endif // !KONCLUDE_REDLAND_INTEGRATION