Debian Codegraph @

Source/Reasoner/Triples/CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer.cpp

raw
1 /*
2 * Copyright (C) 2013-2015, 2019 by the Konclude Developer Team.
3 *
4 * This file is part of the reasoning system Konclude.
5 * For details and support, see <http://konclude.com/>.
6 *
7 * Konclude is free software: you can redistribute it and/or modify
8 * it under the terms of version 3 of the GNU Lesser General Public
9 * License (LGPLv3) as published by the Free Software Foundation.
10 *
11 * Konclude is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU (Lesser) General Public License for more details.
15 *
16 * You should have received a copy of the GNU (Lesser) General Public
17 * License along with Konclude. If not, see <http://www.gnu.org/licenses/>.
18 *
19 */
20 #ifdef KONCLUDE_REDLAND_INTEGRATION
21
22 #include "CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer.h"
23
24
25 namespace Konclude {
26
27 namespace Reasoner {
28
29 namespace Triples {
30
31
32
33 CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer::CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer() {
34 mIndiResolvingKeyCount = 211;
35 mIndiResolvingHashVec.resize(mIndiResolvingKeyCount);
36
37 }
38
39 CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer::~CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer() {
40 }
41
42 cint64 CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer::getCStringHashValue(const char* p) {
43 cint64 result = 0;
44 const cint64 prime = 31;
45 for (cint64 i = 0; p[i] != 0; ++i) {
46 result = p[i] + (result * prime);
47 }
48 return result;
49 }
50
51 cint64 CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer::getNodeHashValue(librdf_node* redlandNode) {
52 cint64 hashValue = 0;
53 if (librdf_node_is_literal(redlandNode)) {
54 hashValue = getCStringHashValue((const char*)librdf_node_get_literal_value(redlandNode));
55 } else if (!librdf_node_is_blank(redlandNode)) {
56 librdf_uri* uri = librdf_node_get_uri(redlandNode);
57 hashValue = getCStringHashValue((const char*)librdf_uri_as_string(uri));
58 } else {
59 hashValue = getCStringHashValue((const char*)librdf_node_get_blank_identifier(redlandNode));
60 }
61 return hashValue;
62 }
63
64
65 CIndividual* CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer::getIndividual(librdf_node* indiNode, CRedlandStoredTriplesIndividualResolvingHashMemoryManaged* indiResolveHash, CRedlandStoredTriplesIndexingBaseBatchLinkerVector<CRedlandStoredTriplesIndexingBaseResolvingLinker>* keyMappedVec) {
66 CIndividual*& indi = indiResolveHash->getIndividual(indiNode);
67 if (!indi) {
68 CRedlandStoredTriplesIndividualPreProcessMemoryManagingContext* indiDataMemAllocManager = indiResolveHash->getIndividualDataMemoryManager();
69 indi = CObjectAllocator<CIndividual>::allocateAndConstruct(indiDataMemAllocManager->getMemoryAllocationManager());
70 indi->initIndividual(0);
71
72 const char* indiNameCharPointer = nullptr;
73 bool blankNode = false;
74 if (!librdf_node_is_blank(indiNode)) {
75 librdf_uri* uri = librdf_node_get_uri(indiNode);
76 indiNameCharPointer = (const char*)librdf_uri_as_string(uri);
77 } else {
78 indiNameCharPointer = (const char*)librdf_node_get_blank_identifier(indiNode);
79 blankNode = true;
80 }
81
82 CIRIName* newName = CObjectAllocator<CIRIName>::allocateAndConstruct(indiDataMemAllocManager->getMemoryAllocationManager());
83 QString indiName = QString::fromUtf8(indiNameCharPointer);
84 if (blankNode) {
85 if (!indiName.startsWith("_:")) {
86 indiName = mAnonymousOntologyIdentifier + indiName;
87 } else {
88 indiName = mAnonymousOntologyIdentifier + indiName.mid(2);
89 }
90 }
91 newName->init(indiName);
92 CLinker<CName*>* newNameLinker = CObjectAllocator< CLinker<CName*> >::allocateAndConstruct(indiDataMemAllocManager->getMemoryAllocationManager());
93 newNameLinker->init(newName);
94 indi->addNameLinker(newNameLinker);
95
96 keyMappedVec->addCreationIndvidual(indi);
97 }
98 return indi;
99 }
100
101
102
103
104
105
106
107
108
109
110
111
112 bool CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer::indexABoxIndividuals(CConcreteOntology* updateConcreteOntology, COntologyTriplesData* ontologyTripleData) {
113
114 mIndexingTime.start();
115
116 mUupdatingOntology = updateConcreteOntology;
117 mPreprocessContext = new CPreProcessContextBase(updateConcreteOntology, updateConcreteOntology->getConfiguration());
118
119 mAnonymousOntologyIdentifier = updateConcreteOntology->getOntologyName() + ":";
120 if (!mAnonymousOntologyIdentifier.startsWith("_:")) {
121 mAnonymousOntologyIdentifier = "_:" + mAnonymousOntologyIdentifier;
122 }
123 mNextIndiId = 0;
124 mDatatypeHashInitialized = false;
125
126 bool newTriplesMapped = false;
127
128
129
130
131
132 mTripleDeleteMapFunc = [&](CRedlandStoredTriplesData*& redlandTriplesData) {
133 if (redlandTriplesData) {
134
135 librdf_world* world = redlandTriplesData->getRedlandWorld();
136 mMemMan = updateConcreteOntology->getDataBoxes()->getBoxContext()->getMemoryAllocationManager();
137
138
139 CXLinker<librdf_statement*>* statementLinker = redlandTriplesData->getRedlandStatementLinker();
140 if (statementLinker) {
141
142 while (statementLinker) {
143 librdf_statement* statement = statementLinker->getData();
144 librdf_free_statement(statement);
145 CXLinker<librdf_statement*>* tmpStatementLinker = statementLinker;
146 statementLinker = statementLinker->getNext();
147 delete tmpStatementLinker;
148 }
149 redlandTriplesData->setRedlandStatementLinker(nullptr);
150 } else {
151 librdf_stream* stream = librdf_model_as_stream(redlandTriplesData->getRedlandIndexedModel());
152 if (stream) {
153 while (!librdf_stream_end(stream)) {
154 librdf_statement* statement = librdf_stream_get_object(stream);
155 librdf_model_remove_statement(redlandTriplesData->getRedlandIndexedModel(), statement);
156 librdf_free_statement(statement);
157 librdf_stream_next(stream);
158 }
159 librdf_free_stream(stream);
160 }
161
162 }
163
164 }
165 };
166
167 mIndiKeyReduceFunc = [&](CRedlandStoredTriplesIndexingBaseBatchLinkerVector<CRedlandStoredTriplesIndexingBaseResolvingLinker>& vec, const CRedlandStoredTriplesIndexingBaseBatchLinkerVector<CRedlandStoredTriplesIndexingBaseResolvingLinker>& reduceVec) -> void {
168
169 CBoxContext* boxContext = updateConcreteOntology->getDataBoxes()->getBoxContext();
170 if (vec.size() < reduceVec.size()) {
171 vec.resize(reduceVec.size());
172 }
173 for (cint64 i = 0; i < vec.count(); ++i) {
174 CRedlandStoredTriplesIndexingBaseBatchLinkerVectorData<CRedlandStoredTriplesIndexingBaseResolvingLinker>*& vecData = vec[i];
175 CRedlandStoredTriplesIndexingBaseBatchLinkerVectorData<CRedlandStoredTriplesIndexingBaseResolvingLinker>* const & reduceVecData = reduceVec[i];
176 if (reduceVecData) {
177 if (!vecData) {
178 vecData = reduceVecData;
179 } else {
180 vecData->mergeBatchLinkers(reduceVecData);
181 }
182 }
183
184 }
185 for (CXLinker<CIndividual*>* indiLinkerIt = reduceVec.getIndvidualCreationLinker(); indiLinkerIt; indiLinkerIt = indiLinkerIt->getNext()) {
186 CIndividual* indi = indiLinkerIt->getData();
187 indi->setIndividualID(mNextIndiId++);
188
189 updateConcreteOntology->getABox()->getIndividualVector(true)->setData(indi->getIndividualID(), indi);
190 updateConcreteOntology->getABox()->getActiveIndividualSet(true)->insert(indi);
191
192 updateConcreteOntology->getStringMapping()->getIndividualNameIndividualMappingHash()->insert(CIRIName::getRecentIRIName(indi->getNameLinker()), indi);
193 }
194 vec.appendMemoryPool(reduceVec.getMemoryPools());
195 };
196
197
198
199
200 cint64 procCount = QThreadPool::globalInstance()->maxThreadCount();
201 QList<CTriplesData*>* triplesDataList = ontologyTripleData->getAllTriplesData();
202
203 if (!triplesDataList->isEmpty()) {
204 CTriplesData* tripleData = triplesDataList->first();
205
206 CRedlandStoredTriplesData* redlandTriplesData = dynamic_cast<CRedlandStoredTriplesData*>(tripleData);
207
208 librdf_world* world = redlandTriplesData->getRedlandWorld();
209 mMemMan = updateConcreteOntology->getDataBoxes()->getBoxContext()->getMemoryAllocationManager();
210
211
212 librdf_node* rdfTypePredicate = librdf_new_node_from_uri_string(world, (const unsigned char*)PREFIX_RDF_TYPE);
213 librdf_node* namedIndividualObject = librdf_new_node_from_uri_string(world, (const unsigned char*)PREFIX_OWL_NAMED_INDIVIDUAL);
214
215
216 if (!mExistingEntitiesInitialized) {
217 mExistingEntitiesInitialized = true;
218 mRoleNodeDataHash.clear();
219 QSet<CRole*>* activeRoleSet = updateConcreteOntology->getRBox()->getActivePropertyRoleSet(false);
220 if (activeRoleSet) {
221 for (CRole* role : *activeRoleSet) {
222 QByteArray roleCharData = CIRIName::getRecentIRIName(role->getPropertyNameLinker()).toUtf8();
223 librdf_node* roleNode = librdf_new_node_from_uri_string(world, (const unsigned char*)roleCharData.constData());
224 mRoleNodeDataHash.insert(CRedlandNodeHasher(roleNode), new CRoleNodeData(roleNode, role, role->isDataRole()));
225 }
226 }
227
228
229 mConceptNodeDataHash.clear();
230 CConceptNodeData* topConceptData = nullptr;
231 QSet<CConcept*>* activeConceptSet = updateConcreteOntology->getTBox()->getActiveClassConceptSet(false);
232 if (activeConceptSet) {
233 for (CConcept* concept : *activeConceptSet) {
234 QByteArray conceptCharData = CIRIName::getRecentIRIName(concept->getClassNameLinker()).toUtf8();
235 librdf_node* conceptNode = librdf_new_node_from_uri_string(world, (const unsigned char*)conceptCharData.constData());
236 CConceptNodeData* conceptData = new CConceptNodeData(conceptNode, concept);
237 mConceptNodeDataHash.insert(CRedlandNodeHasher(conceptNode), conceptData);
238 if (concept->getConceptTag() == 1) {
239 topConceptData = conceptData;
240 }
241 }
242 }
243
244
245 mIndividualNodeDataHash.clear();
246 QSet<CIndividual*>* activeIndiSet = updateConcreteOntology->getABox()->getActiveIndividualSet(false);
247 if (activeIndiSet && !activeIndiSet->isEmpty()) {
248 cint64 maxIndividualId = updateConcreteOntology->getABox()->getNextIndividualId(false);
249 mNextIndiId = qMax(maxIndividualId, mNextIndiId);
250 for (CIndividual* indi : *activeIndiSet) {
251 maxIndividualId = qMax(indi->getIndividualID(), maxIndividualId);
252 QByteArray indiCharData = CIRIName::getRecentIRIName(indi->getIndividualNameLinker()).toUtf8();
253 librdf_node* indiNode = librdf_new_node_from_uri_string(world, (const unsigned char*)indiCharData.constData());
254
255 CIndividualNodeData* indiData = new CIndividualNodeData(indiNode, indi);
256 mIndividualNodeDataHash.insert(CRedlandNodeHasher(indiNode), indiData);
257 mNextIndiId = qMax(indi->getIndividualID(), mNextIndiId);
258 }
259 mNextIndiId++;
260 }
261 }
262
263 }
264
265 if (triplesDataList->count() < procCount * 2) {
266 for (CTriplesData* tripleData : *ontologyTripleData->getAllTriplesData()) {
267 CRedlandStoredTriplesData* redlandTriplesData = dynamic_cast<CRedlandStoredTriplesData*>(tripleData);
268 librdf_world* world = redlandTriplesData->getRedlandWorld();
269 librdf_node* rdfTypePredicate = librdf_new_node_from_uri_string(world, (const unsigned char*)PREFIX_RDF_TYPE);
270 librdf_node* namedIndividualObject = librdf_new_node_from_uri_string(world, (const unsigned char*)PREFIX_OWL_NAMED_INDIVIDUAL);
271
272 indexTriples(redlandTriplesData, rdfTypePredicate, namedIndividualObject, updateConcreteOntology);
273
274 }
275
276 bool deleteIndexedTriplesData = CConfigDataReader::readConfigBoolean(updateConcreteOntology->getConfiguration(), "Konclude.Calculation.Preprocessing.TripleEncodedAssertionsIndexing.DeleteTriplesDataAfterIndexing", false);
277 if (deleteIndexedTriplesData) {
278 QVector<CRedlandStoredTriplesData*> triplesDataVec(triplesDataList->count());
279 cint64 tripleDataId = 0;
280 for (CTriplesData* tripleData : *triplesDataList) {
281 CRedlandStoredTriplesData* redlandTriplesData = (CRedlandStoredTriplesData*)tripleData;
282 ensureDatatypeHashInitialized(updateConcreteOntology, redlandTriplesData);
283 triplesDataVec[tripleDataId++] = redlandTriplesData;
284 }
285 QtConcurrent::blockingMap(triplesDataVec, mTripleDeleteMapFunc);
286 }
287
288 } else {
289 indexTripleData(triplesDataList, updateConcreteOntology);
290 }
291
292 for (cint64 i = 0; i < mIndiResolvingKeyCount; ++i) {
293 CRedlandStoredTriplesIndividualResolvingHashMemoryManaged*& indiResolveHash = mIndiResolvingHashVec[i];
294 if (indiResolveHash) {
295 mConceptAssertionCount += indiResolveHash->getConceptAssertionCount();
296 mObjectPropertyAssertionCount += indiResolveHash->getObjectPropertyAssertionCount();
297 mDataPropertyAssertionCount += indiResolveHash->getDataPropertyAssertionCount();
298 delete indiResolveHash;
299 indiResolveHash = nullptr;
300 }
301 }
302
303
304 LOG(INFO, "::Konclude::Reasoner::TripleEncodedAssertionsIndexer", QString("Identified %1 concept, %2 object property, and %3 data property assertions for overall %4 individuals in %5 ms.").arg(mConceptAssertionCount).arg(mObjectPropertyAssertionCount).arg(mDataPropertyAssertionCount).arg(mNextIndiId).arg(mIndexingTime.elapsed()), this);
305
306 delete mPreprocessContext;
307
308 return newTriplesMapped;
309 }
310
311
312
313
314
315
316
317
318 void CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer::procssIndividualIndexing(CRedlandStoredTriplesIndexingBaseBatchLinkerVector<CRedlandStoredTriplesIndexingBaseResolvingLinker>& mappedIndi1HashKeyVec, CRedlandStoredTriplesData* redlandTriplesData, CConcreteOntology* updateConcreteOntology) {
319
320
321 function<CRedlandStoredTriplesIndexingBaseBatchLinkerVector<CRedlandStoredTriplesIndexingBaseResolvingLinker>(const CRedlandStoredTriplesIndexingBaseBatchLinkerVectorData<CRedlandStoredTriplesIndexingBaseResolvingLinker>* const & batchData)> individual2ResolveKeyMapFunc =
322 [&](const CRedlandStoredTriplesIndexingBaseBatchLinkerVectorData<CRedlandStoredTriplesIndexingBaseResolvingLinker>* const & batchData) ->CRedlandStoredTriplesIndexingBaseBatchLinkerVector<CRedlandStoredTriplesIndexingBaseResolvingLinker> {
323
324 CRedlandStoredTriplesIndexingBaseBatchLinkerVector<CRedlandStoredTriplesIndexingBaseResolvingLinker> keyMappedVec(mIndiResolvingKeyCount);
325 if (batchData) {
326
327 CRedlandStoredTriplesIndexingBaseResolvingLinker* resolveLinker = batchData->getNextBatchLinker();
328 while (resolveLinker) {
329
330 cint64 index = qAbs(resolveLinker->mIndi1HashValue % mIndiResolvingKeyCount);
331 CRedlandStoredTriplesIndividualResolvingHashMemoryManaged*& indiResolveHash = mIndiResolvingHashVec[index];
332 if (!indiResolveHash) {
333 indiResolveHash = new CRedlandStoredTriplesIndividualResolvingHashMemoryManaged(mUupdatingOntology, mUupdatingOntology->getConfiguration(), mAnonymousOntologyIdentifier);
334 }
335
336 CRedlandStoredTriplesIndexingBaseResolvingLinker::ASSERTION_TYPE assertionType = (CRedlandStoredTriplesIndexingBaseResolvingLinker::ASSERTION_TYPE)resolveLinker->getData();
337
338 librdf_statement* statement = resolveLinker->mStatement;
339 librdf_node* predicateNode = librdf_statement_get_predicate(statement);
340 librdf_node* objectNode = librdf_statement_get_object(statement);
341 librdf_node* subjectNode = librdf_statement_get_subject(statement);
342
343 resolveLinker->mIndi1 = getIndividual(subjectNode, indiResolveHash, &keyMappedVec);
344
345
346 if (assertionType == CRedlandStoredTriplesIndexingBaseResolvingLinker::ASSERTION_CONCEPT) {
347 CConceptAssertionLinker* conAssLinker = indiResolveHash->createAssertionConceptLinker();
348 conAssLinker->initNegLinker(resolveLinker->mConcept, false);
349
350 resolveLinker->mIndi1->addAssertionConceptLinker(conAssLinker);
351
352 resolveLinker = resolveLinker->getNext();
353 } else if (assertionType == CRedlandStoredTriplesIndexingBaseResolvingLinker::ASSERTION_DATA) {
354 CRedlandStoredTriplesIndividualPreProcessMemoryManagingContext* indiMemManContext = indiResolveHash->getIndividualDataMemoryManager();
355 CDataAssertionLinker* dataAssLinker = indiResolveHash->createAssertionDataLinker();
356
357 const char* literalValue = (const char*)librdf_node_get_literal_value(objectNode);
358 librdf_uri* datatypeUri = librdf_node_get_literal_value_datatype_uri(objectNode);
359 CDatatype* datatype = getDatatypeFromDatatypeUri(datatypeUri, mUupdatingOntology, redlandTriplesData);
360
361 CDataLiteral* dataLiteral = CObjectParameterizingAllocator<CDataLiteral, CContext*>::allocateAndConstructAndParameterize(indiMemManContext->getMemoryAllocationManager(), indiMemManContext);
362 CDataLiteralNormalizerPreProcess* dataLiteralNormalizer = indiResolveHash->getIndividualDataLiteralNormalizer();
363
364 dataLiteral->initDataLiteral(QString::fromUtf8(literalValue), datatype);
365 dataLiteralNormalizer->createPreprocessedDataLiteralValue(dataLiteral, indiMemManContext, indiMemManContext);
366 dataAssLinker->initDataAssertionLinker(resolveLinker->mRole, dataLiteral);
367
368 resolveLinker->mIndi1->addAssertionDataLinker(dataAssLinker);
369 resolveLinker = resolveLinker->getNext();
370 } else if (assertionType == CRedlandStoredTriplesIndexingBaseResolvingLinker::ASSERTION_ROLE) {
371 cint64 nextIndex = qAbs(resolveLinker->mIndi2HashValue % mIndiResolvingKeyCount);
372 CRoleAssertionLinker* roleAssLinker = indiResolveHash->createAssertionRoleLinker();
373 resolveLinker->mRoleAssLinker = roleAssLinker;
374 resolveLinker->mIndi1->addAssertionRoleLinker(roleAssLinker);
375 CRedlandStoredTriplesIndexingBaseResolvingLinker* tmpResolveLinker = resolveLinker;
376 resolveLinker = resolveLinker->getNext();
377 tmpResolveLinker->clearNext();
378 keyMappedVec.addBatchLinker(nextIndex, tmpResolveLinker);
379 } else {
380 resolveLinker = resolveLinker->getNext();
381 }
382
383 }
384 }
385 return keyMappedVec;
386 };
387
388
389 function<CRedlandStoredTriplesIndexingBaseBatchLinkerVector<CRedlandStoredTriplesIndexingBaseResolvingLinker>(const CRedlandStoredTriplesIndexingBaseBatchLinkerVectorData<CRedlandStoredTriplesIndexingBaseResolvingLinker>* const & batchData)> individual3ResolveKeyMapFunc =
390 [&](const CRedlandStoredTriplesIndexingBaseBatchLinkerVectorData<CRedlandStoredTriplesIndexingBaseResolvingLinker>* const & batchData) ->CRedlandStoredTriplesIndexingBaseBatchLinkerVector<CRedlandStoredTriplesIndexingBaseResolvingLinker> {
391
392 CRedlandStoredTriplesIndexingBaseBatchLinkerVector<CRedlandStoredTriplesIndexingBaseResolvingLinker> keyMappedVec(mIndiResolvingKeyCount);
393 if (batchData) {
394
395 CRedlandStoredTriplesIndexingBaseResolvingLinker* resolveLinker = batchData->getNextBatchLinker();
396 while (resolveLinker) {
397
398 cint64 index = qAbs(resolveLinker->mIndi2HashValue % mIndiResolvingKeyCount);
399 CRedlandStoredTriplesIndividualResolvingHashMemoryManaged*& indiResolveHash = mIndiResolvingHashVec[index];
400 if (!indiResolveHash) {
401 indiResolveHash = new CRedlandStoredTriplesIndividualResolvingHashMemoryManaged(mUupdatingOntology, mUupdatingOntology->getConfiguration(), mAnonymousOntologyIdentifier);
402 }
403
404 librdf_statement* statement = resolveLinker->mStatement;
405 librdf_node* objectNode = librdf_statement_get_object(statement);
406 resolveLinker->mIndi2 = getIndividual(objectNode, indiResolveHash, &keyMappedVec);
407
408 CRedlandStoredTriplesIndexingBaseResolvingLinker::ASSERTION_TYPE assertionType = (CRedlandStoredTriplesIndexingBaseResolvingLinker::ASSERTION_TYPE)resolveLinker->getData();
409 if (assertionType == CRedlandStoredTriplesIndexingBaseResolvingLinker::ASSERTION_ROLE) {
410 CRoleAssertionLinker* roleAssLinker = resolveLinker->mRoleAssLinker;
411 roleAssLinker->initRoleAssertionLinker(resolveLinker->mRole, resolveLinker->mIndi2);
412 CReverseRoleAssertionLinker* reverseRoleAssLinker = indiResolveHash->createReverseAssertionRoleLinker();
413 reverseRoleAssLinker->initReverseRoleAssertionLinker(roleAssLinker, resolveLinker->mIndi1);
414 resolveLinker->mIndi2->addReverseAssertionRoleLinker(reverseRoleAssLinker);
415 }
416
417 resolveLinker = resolveLinker->getNext();
418 }
419 }
420 return keyMappedVec;
421 };
422
423
424
425
426 CRedlandStoredTriplesIndexingBaseBatchLinkerVector<CRedlandStoredTriplesIndexingBaseResolvingLinker> mappedIndi2HashKeyVec = QtConcurrent::blockingMappedReduced<CRedlandStoredTriplesIndexingBaseBatchLinkerVector<CRedlandStoredTriplesIndexingBaseResolvingLinker>>(mappedIndi1HashKeyVec, individual2ResolveKeyMapFunc, mIndiKeyReduceFunc);
427 CRedlandStoredTriplesIndexingBaseBatchLinkerVector<CRedlandStoredTriplesIndexingBaseResolvingLinker> mappedIndi3HashKeyVec = QtConcurrent::blockingMappedReduced<CRedlandStoredTriplesIndexingBaseBatchLinkerVector<CRedlandStoredTriplesIndexingBaseResolvingLinker>>(mappedIndi2HashKeyVec, individual3ResolveKeyMapFunc, mIndiKeyReduceFunc);
428
429 CMemoryPool* mappedIndi1HashKeyVecMemoryPools = mappedIndi1HashKeyVec.takeMemoryPools();
430 CMemoryPool* mappedIndi2HashKeyVecMemoryPools = mappedIndi2HashKeyVec.takeMemoryPools();
431 CMemoryPool* mappedIndi3HashKeyVecMemoryPools = mappedIndi3HashKeyVec.takeMemoryPools();
432 //QtConcurrent::run(QThreadPool::globalInstance(), [=] {
433 clearMemoryPools(mappedIndi1HashKeyVecMemoryPools);
434 //});
435 //QtConcurrent::run(QThreadPool::globalInstance(), [=] {
436 clearMemoryPools(mappedIndi2HashKeyVecMemoryPools);
437 //});
438 //QtConcurrent::run(QThreadPool::globalInstance(), [=] {
439 clearMemoryPools(mappedIndi3HashKeyVecMemoryPools);
440 //});
441 }
442
443
444
445
446 void CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer::clearMemoryPools(CMemoryPool* memPools) {
447 while (memPools) {
448 CMemoryPool* tmpMemPool = memPools;
449 memPools = memPools->getNextMemoryPool();
450 delete[] tmpMemPool->getMemoryBlockData();
451 delete tmpMemPool;
452 }
453 }
454
455
456
457 void CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer::indexTripleData(QList<CTriplesData*>* triplesDataList, CConcreteOntology* updateConcreteOntology) {
458 QVector<CRedlandStoredTriplesData*> triplesDataVec(triplesDataList->count());
459 cint64 tripleDataId = 0;
460 for (CTriplesData* tripleData : *triplesDataList) {
461 CRedlandStoredTriplesData* redlandTriplesData = (CRedlandStoredTriplesData*)tripleData;
462 ensureDatatypeHashInitialized(updateConcreteOntology, redlandTriplesData);
463 triplesDataVec[tripleDataId++] = redlandTriplesData;
464 }
465
466 indexHashExistingIndividuals(updateConcreteOntology);
467
468
469 function<CRedlandStoredTriplesIndexingBaseBatchLinkerVector<CRedlandStoredTriplesIndexingBaseResolvingLinker>(CRedlandStoredTriplesData* redlandTriplesData)> individual1ResolveKeyMapFunc =
470 [&](CRedlandStoredTriplesData* redlandTriplesData) ->CRedlandStoredTriplesIndexingBaseBatchLinkerVector<CRedlandStoredTriplesIndexingBaseResolvingLinker> {
471
472 CRedlandStoredTriplesIndexingBaseBatchLinkerVector<CRedlandStoredTriplesIndexingBaseResolvingLinker> keyMappedVec(mIndiResolvingKeyCount);
473 if (redlandTriplesData) {
474
475
476 librdf_world* world = redlandTriplesData->getRedlandWorld();
477 mMemMan = updateConcreteOntology->getDataBoxes()->getBoxContext()->getMemoryAllocationManager();
478
479
480 librdf_node* rdfTypePredicate = librdf_new_node_from_uri_string(world, (const unsigned char*)PREFIX_RDF_TYPE);
481 librdf_node* namedIndividualObject = librdf_new_node_from_uri_string(world, (const unsigned char*)PREFIX_OWL_NAMED_INDIVIDUAL);
482
483 CXLinker<librdf_statement*>* statementLinker = redlandTriplesData->getRedlandStatementLinker();
484 if (statementLinker) {
485
486 while (statementLinker) {
487 librdf_statement* statement = statementLinker->getData();
488 handleStatementMapping(statement, keyMappedVec, rdfTypePredicate, namedIndividualObject);
489 statementLinker = statementLinker->getNext();
490 }
491 } else {
492 librdf_stream* stream = librdf_model_as_stream(redlandTriplesData->getRedlandIndexedModel());
493 if (stream) {
494 while (!librdf_stream_end(stream)) {
495 librdf_statement* statement = librdf_stream_get_object(stream);
496 handleStatementMapping(statement, keyMappedVec, rdfTypePredicate, namedIndividualObject);
497 librdf_stream_next(stream);
498 }
499 librdf_free_stream(stream);
500 }
501
502 }
503
504 }
505 return keyMappedVec;
506 };
507
508
509
510 CRedlandStoredTriplesIndexingBaseBatchLinkerVector<CRedlandStoredTriplesIndexingBaseResolvingLinker> mappedIndi1HashKeyVec = QtConcurrent::blockingMappedReduced<CRedlandStoredTriplesIndexingBaseBatchLinkerVector<CRedlandStoredTriplesIndexingBaseResolvingLinker>>(triplesDataVec, individual1ResolveKeyMapFunc, mIndiKeyReduceFunc);
511
512 procssIndividualIndexing(mappedIndi1HashKeyVec, nullptr, updateConcreteOntology);
513
514
515
516
517
518
519 bool deleteIndexedTriplesData = CConfigDataReader::readConfigBoolean(updateConcreteOntology->getConfiguration(), "Konclude.Calculation.Preprocessing.TripleEncodedAssertionsIndexing.DeleteTriplesDataAfterIndexing", false);
520 if (deleteIndexedTriplesData) {
521 QtConcurrent::blockingMap(triplesDataVec, mTripleDeleteMapFunc);
522 }
523
524
525
526
527
528
529
530 updateConcreteOntology->getABox()->setNextIndividualId(mNextIndiId);
531 }
532
533
534
535
536
537
538 void CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer::indexTriples(CRedlandStoredTriplesData* redlandTriplesData, librdf_node* rdfTypePredicate, librdf_node* namedIndividualObject, CConcreteOntology* updateConcreteOntology) {
539 CIndividualNodeData* lastIndiData = nullptr;
540 librdf_node* lastSubjectNode = nullptr;
541
542 cint64 statementBatchLimit = 1000;
543 cint64 statementBatchCountLimit = 1000;
544
545 CXLinker<librdf_statement*>* statementLinker = redlandTriplesData->getRedlandStatementLinker();
546 if (statementLinker) {
547 ensureDatatypeHashInitialized(updateConcreteOntology, redlandTriplesData);
548
549
550 indexHashExistingIndividuals(updateConcreteOntology);
551
552
553
554 CXLinker<librdf_statement*>* statementLinkerIt = statementLinker;
555 while (statementLinkerIt) {
556
557 CRedlandStoredTriplesIndexingBaseBatchLinkerVector<CRedlandStoredTriplesIndexingBaseBatchLinker> statementBatchingLinkerVec(mIndiResolvingKeyCount);
558 cint64 statementBatchSize = 0;
559 cint64 statementBatchCount = 0;
560 while (statementLinkerIt && statementBatchCount < statementBatchCountLimit) {
561 CXLinker<librdf_statement*>* statementBeginLinker = statementLinkerIt;
562 CXLinker<librdf_statement*>* statementEndLinker = statementLinkerIt;
563 statementBatchSize = 0;
564 while (statementEndLinker && statementBatchSize < statementBatchLimit) {
565 statementEndLinker = statementEndLinker->getNext();
566 ++statementBatchSize;
567 }
568 statementLinkerIt = statementEndLinker;
569 CRedlandStoredTriplesIndexingBaseBatchLinker* linker = statementBatchingLinkerVec.createBatchLinker();
570 linker->initBatchLinker(statementBeginLinker, statementEndLinker, statementBatchSize);
571 statementBatchingLinkerVec.addBatchLinker(statementBatchingLinkerVec.getNextIndex(true), linker);
572 statementBatchCount++;
573 }
574
575
576
577
578
579
580
581
582 function<CRedlandStoredTriplesIndexingBaseBatchLinkerVector<CRedlandStoredTriplesIndexingBaseResolvingLinker>(const CRedlandStoredTriplesIndexingBaseBatchLinkerVectorData<CRedlandStoredTriplesIndexingBaseBatchLinker>* const & batchData)> individual1ResolveKeyMapFunc =
583 [&](const CRedlandStoredTriplesIndexingBaseBatchLinkerVectorData<CRedlandStoredTriplesIndexingBaseBatchLinker>* const & batchData) ->CRedlandStoredTriplesIndexingBaseBatchLinkerVector<CRedlandStoredTriplesIndexingBaseResolvingLinker> {
584
585 CRedlandStoredTriplesIndexingBaseBatchLinkerVector<CRedlandStoredTriplesIndexingBaseResolvingLinker> keyMappedVec(mIndiResolvingKeyCount);
586 if (batchData) {
587 CRedlandStoredTriplesIndexingBaseBatchLinker* batchLinker = batchData->getNextBatchLinker();
588 while (batchLinker) {
589
590 CXLinker<librdf_statement*>* statementLinker = batchLinker->getNextStatementLinker();
591 while (statementLinker) {
592 librdf_statement* statement = statementLinker->getData();
593
594 handleStatementMapping(statement, keyMappedVec, rdfTypePredicate, namedIndividualObject);
595
596 statementLinker = batchLinker->getNextStatementLinker();
597 }
598
599 batchLinker = batchLinker->getNext();
600 }
601 }
602 return keyMappedVec;
603 };
604
605
606 CRedlandStoredTriplesIndexingBaseBatchLinkerVector<CRedlandStoredTriplesIndexingBaseResolvingLinker> mappedIndi1HashKeyVec = QtConcurrent::blockingMappedReduced<CRedlandStoredTriplesIndexingBaseBatchLinkerVector<CRedlandStoredTriplesIndexingBaseResolvingLinker>>(statementBatchingLinkerVec, individual1ResolveKeyMapFunc, mIndiKeyReduceFunc);
607
608 procssIndividualIndexing(mappedIndi1HashKeyVec, redlandTriplesData, updateConcreteOntology);
609
610 CMemoryPool* statementBatchingLinkerVecMemoryPools = statementBatchingLinkerVec.takeMemoryPools();
611 //QtConcurrent::run(QThreadPool::globalInstance(), [=] {
612 clearMemoryPools(statementBatchingLinkerVecMemoryPools);
613 //});
614
615 }
616
617
618 //CBoxContext* boxContext = updateConcreteOntology->getDataBoxes()->getBoxContext();
619 //for (cint64 i = 0; i < mIndiResolvingKeyCount; ++i) {
620 // CRedlandStoredTriplesIndividualResolvingHashMemoryManaged* indiResolveHash = mIndiResolvingHashVec[i];
621 // if (indiResolveHash) {
622 // CMemoryPool* memPools = indiResolveHash->takeIndividualDataMemoryPools();
623 // boxContext->addUsedMemoryPools(memPools);
624 // for (CQtManagedRestrictedModificationHash<CRedlandNodeHasher, CIndividual*>::const_iterator it = indiResolveHash->constBegin(), itEnd = indiResolveHash->constEnd(); it != itEnd; ++it) {
625 // CIndividual* indi = it.value();
626 // indi->setIndividualID(mNextIndiId++);
627 // updateConcreteOntology->getABox()->getIndividualVector(true)->setData(indi->getIndividualID(), indi);
628 // updateConcreteOntology->getABox()->getActiveIndividualSet(true)->insert(indi);
629 // updateConcreteOntology->getStringMapping()->getIndividualNameIndividualMappingHash()->insert(CIRIName::getRecentIRIName(indi->getNameLinker()), indi);
630 // }
631 // }
632 //}
633
634
635 //while (statementLinker) {
636
637 // librdf_statement* statement = statementLinker->getData();
638
639
640 // identifyIndividuals(statement, rdfTypePredicate, namedIndividualObject, lastIndiData, lastSubjectNode, updateConcreteOntology, conceptAssertionCount, redlandTriplesData, dataPropertyAssertionCount, objectPropertyAssertionCount);
641
642
643 // statementLinker = statementLinker->getNext();
644 //}
645 } else {
646 librdf_stream* stream = librdf_model_as_stream(redlandTriplesData->getRedlandIndexedModel());
647 if (stream) {
648 while (!librdf_stream_end(stream)) {
649 librdf_statement* statement = librdf_stream_get_object(stream);
650 identifyIndividuals(statement, rdfTypePredicate, namedIndividualObject, lastIndiData, lastSubjectNode, updateConcreteOntology, redlandTriplesData);
651 librdf_stream_next(stream);
652 }
653 librdf_free_stream(stream);
654 }
655
656 }
657
658
659 updateConcreteOntology->getABox()->setNextIndividualId(mNextIndiId);
660
661 }
662
663 void CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer::handleStatementMapping(librdf_statement* statement, CRedlandStoredTriplesIndexingBaseBatchLinkerVector<CRedlandStoredTriplesIndexingBaseResolvingLinker> &keyMappedVec, librdf_node* rdfTypePredicate, librdf_node* namedIndividualObject) {
664 librdf_node* predicateNode = librdf_statement_get_predicate(statement);
665 librdf_node* objectNode = librdf_statement_get_object(statement);
666 librdf_node* subjectNode = librdf_statement_get_subject(statement);
667
668 CRedlandStoredTriplesIndexingBaseResolvingLinker* resolvingLinker = keyMappedVec.createBatchLinker();
669 resolvingLinker->setData(CRedlandStoredTriplesIndexingBaseResolvingLinker::ASSERTION_NONE);
670 resolvingLinker->mStatement = statement;
671
672 resolvingLinker->mIndi1HashValue = getNodeHashValue(subjectNode);
673 cint64 index = qAbs(resolvingLinker->mIndi1HashValue % mIndiResolvingKeyCount);
674
675 if (librdf_node_equals(predicateNode, rdfTypePredicate)) {
676 if (librdf_node_equals(objectNode, namedIndividualObject)) {
677 resolvingLinker->setData(CRedlandStoredTriplesIndexingBaseResolvingLinker::ASSERTION_INDIVIDUAL);
678 keyMappedVec.addBatchLinker(index, resolvingLinker);
679 } else {
680 resolvingLinker->setData(CRedlandStoredTriplesIndexingBaseResolvingLinker::ASSERTION_CONCEPT);
681 CConceptNodeData* conceptData = mConceptNodeDataHash.value(CRedlandNodeHasher(objectNode));
682 if (conceptData) {
683 resolvingLinker->mConcept = conceptData->mConcept;
684 keyMappedVec.addBatchLinker(index, resolvingLinker);
685 }
686 }
687 } else {
688 CRoleNodeData* roleData = mRoleNodeDataHash.value(CRedlandNodeHasher(predicateNode));
689 if (roleData) {
690 resolvingLinker->mRole = roleData->mRole;
691 if (roleData->mDataRole) {
692 if (librdf_node_is_literal(objectNode)) {
693 resolvingLinker->setData(CRedlandStoredTriplesIndexingBaseResolvingLinker::ASSERTION_DATA);
694 keyMappedVec.addBatchLinker(index, resolvingLinker);
695 }
696 } else {
697 if (!librdf_node_is_literal(objectNode)) {
698 resolvingLinker->setData(CRedlandStoredTriplesIndexingBaseResolvingLinker::ASSERTION_ROLE);
699 keyMappedVec.addBatchLinker(index, resolvingLinker);
700 resolvingLinker->mIndi2HashValue = getNodeHashValue(objectNode);
701 }
702 }
703 }
704 }
705 }
706
707 void CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer::indexHashExistingIndividuals(CConcreteOntology* updateConcreteOntology) {
708 QSet<CIndividual*>* activeIndiSet = updateConcreteOntology->getABox()->getActiveIndividualSet(false);
709 if (!mExistingIndividualsHashed) {
710 mExistingIndividualsHashed = true;
711 if (activeIndiSet && !activeIndiSet->isEmpty()) {
712
713 for (QHash<CRedlandNodeHasher, CIndividualNodeData*>::const_iterator it = mIndividualNodeDataHash.constBegin(), itEnd = mIndividualNodeDataHash.constEnd(); it != itEnd; ++it) {
714 cint64 hashValue = it.key().getNodeHashValue();
715 cint64 index = qAbs(hashValue % mIndiResolvingKeyCount);
716 CRedlandStoredTriplesIndividualResolvingHashMemoryManaged*& indiResolveHash = mIndiResolvingHashVec[index];
717 if (!indiResolveHash) {
718 indiResolveHash = new CRedlandStoredTriplesIndividualResolvingHashMemoryManaged(mUupdatingOntology, mUupdatingOntology->getConfiguration(), mAnonymousOntologyIdentifier);
719 }
720 indiResolveHash->insertIndividual(it.key(), it.value()->mIndividual);
721 }
722 }
723 }
724 }
725
726 }; // end namespace Triples
727
728 }; // end namespace Reasoner
729
730 }; // end namespace Konclude
731
732 #endif // !KONCLUDE_REDLAND_INTEGRATION