/* * Copyright (C) 2013-2015, 2019 by the Konclude Developer Team. * * This file is part of the reasoning system Konclude. * For details and support, see . * * Konclude is free software: you can redistribute it and/or modify * it under the terms of version 3 of the GNU Lesser General Public * License (LGPLv3) as published by the Free Software Foundation. * * Konclude is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU (Lesser) General Public License for more details. * * You should have received a copy of the GNU (Lesser) General Public * License along with Konclude. If not, see . * */ #ifdef KONCLUDE_REDLAND_INTEGRATION #include "CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer.h" namespace Konclude { namespace Reasoner { namespace Triples { CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer::CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer() { mIndiResolvingKeyCount = 211; mIndiResolvingHashVec.resize(mIndiResolvingKeyCount); } CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer::~CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer() { } cint64 CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer::getCStringHashValue(const char* p) { cint64 result = 0; const cint64 prime = 31; for (cint64 i = 0; p[i] != 0; ++i) { result = p[i] + (result * prime); } return result; } cint64 CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer::getNodeHashValue(librdf_node* redlandNode) { cint64 hashValue = 0; if (librdf_node_is_literal(redlandNode)) { hashValue = getCStringHashValue((const char*)librdf_node_get_literal_value(redlandNode)); } else if (!librdf_node_is_blank(redlandNode)) { librdf_uri* uri = librdf_node_get_uri(redlandNode); hashValue = getCStringHashValue((const char*)librdf_uri_as_string(uri)); } else { hashValue = getCStringHashValue((const char*)librdf_node_get_blank_identifier(redlandNode)); } return hashValue; } CIndividual* CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer::getIndividual(librdf_node* indiNode, CRedlandStoredTriplesIndividualResolvingHashMemoryManaged* indiResolveHash, CRedlandStoredTriplesIndexingBaseBatchLinkerVector* keyMappedVec) { CIndividual*& indi = indiResolveHash->getIndividual(indiNode); if (!indi) { CRedlandStoredTriplesIndividualPreProcessMemoryManagingContext* indiDataMemAllocManager = indiResolveHash->getIndividualDataMemoryManager(); indi = CObjectAllocator::allocateAndConstruct(indiDataMemAllocManager->getMemoryAllocationManager()); indi->initIndividual(0); const char* indiNameCharPointer = nullptr; bool blankNode = false; if (!librdf_node_is_blank(indiNode)) { librdf_uri* uri = librdf_node_get_uri(indiNode); indiNameCharPointer = (const char*)librdf_uri_as_string(uri); } else { indiNameCharPointer = (const char*)librdf_node_get_blank_identifier(indiNode); blankNode = true; } CIRIName* newName = CObjectAllocator::allocateAndConstruct(indiDataMemAllocManager->getMemoryAllocationManager()); QString indiName = QString::fromUtf8(indiNameCharPointer); if (blankNode) { if (!indiName.startsWith("_:")) { indiName = mAnonymousOntologyIdentifier + indiName; } else { indiName = mAnonymousOntologyIdentifier + indiName.mid(2); } } newName->init(indiName); CLinker* newNameLinker = CObjectAllocator< CLinker >::allocateAndConstruct(indiDataMemAllocManager->getMemoryAllocationManager()); newNameLinker->init(newName); indi->addNameLinker(newNameLinker); keyMappedVec->addCreationIndvidual(indi); } return indi; } bool CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer::indexABoxIndividuals(CConcreteOntology* updateConcreteOntology, COntologyTriplesData* ontologyTripleData) { mIndexingTime.start(); mUupdatingOntology = updateConcreteOntology; mPreprocessContext = new CPreProcessContextBase(updateConcreteOntology, updateConcreteOntology->getConfiguration()); mAnonymousOntologyIdentifier = updateConcreteOntology->getOntologyName() + ":"; if (!mAnonymousOntologyIdentifier.startsWith("_:")) { mAnonymousOntologyIdentifier = "_:" + mAnonymousOntologyIdentifier; } mNextIndiId = 0; mDatatypeHashInitialized = false; bool newTriplesMapped = false; mTripleDeleteMapFunc = [&](CRedlandStoredTriplesData*& redlandTriplesData) { if (redlandTriplesData) { librdf_world* world = redlandTriplesData->getRedlandWorld(); mMemMan = updateConcreteOntology->getDataBoxes()->getBoxContext()->getMemoryAllocationManager(); CXLinker* statementLinker = redlandTriplesData->getRedlandStatementLinker(); if (statementLinker) { while (statementLinker) { librdf_statement* statement = statementLinker->getData(); librdf_free_statement(statement); CXLinker* tmpStatementLinker = statementLinker; statementLinker = statementLinker->getNext(); delete tmpStatementLinker; } redlandTriplesData->setRedlandStatementLinker(nullptr); } else { librdf_stream* stream = librdf_model_as_stream(redlandTriplesData->getRedlandIndexedModel()); if (stream) { while (!librdf_stream_end(stream)) { librdf_statement* statement = librdf_stream_get_object(stream); librdf_model_remove_statement(redlandTriplesData->getRedlandIndexedModel(), statement); librdf_free_statement(statement); librdf_stream_next(stream); } librdf_free_stream(stream); } } } }; mIndiKeyReduceFunc = [&](CRedlandStoredTriplesIndexingBaseBatchLinkerVector& vec, const CRedlandStoredTriplesIndexingBaseBatchLinkerVector& reduceVec) -> void { CBoxContext* boxContext = updateConcreteOntology->getDataBoxes()->getBoxContext(); if (vec.size() < reduceVec.size()) { vec.resize(reduceVec.size()); } for (cint64 i = 0; i < vec.count(); ++i) { CRedlandStoredTriplesIndexingBaseBatchLinkerVectorData*& vecData = vec[i]; CRedlandStoredTriplesIndexingBaseBatchLinkerVectorData* const & reduceVecData = reduceVec[i]; if (reduceVecData) { if (!vecData) { vecData = reduceVecData; } else { vecData->mergeBatchLinkers(reduceVecData); } } } for (CXLinker* indiLinkerIt = reduceVec.getIndvidualCreationLinker(); indiLinkerIt; indiLinkerIt = indiLinkerIt->getNext()) { CIndividual* indi = indiLinkerIt->getData(); indi->setIndividualID(mNextIndiId++); updateConcreteOntology->getABox()->getIndividualVector(true)->setData(indi->getIndividualID(), indi); updateConcreteOntology->getABox()->getActiveIndividualSet(true)->insert(indi); updateConcreteOntology->getStringMapping()->getIndividualNameIndividualMappingHash()->insert(CIRIName::getRecentIRIName(indi->getNameLinker()), indi); } vec.appendMemoryPool(reduceVec.getMemoryPools()); }; cint64 procCount = QThreadPool::globalInstance()->maxThreadCount(); QList* triplesDataList = ontologyTripleData->getAllTriplesData(); if (!triplesDataList->isEmpty()) { CTriplesData* tripleData = triplesDataList->first(); CRedlandStoredTriplesData* redlandTriplesData = dynamic_cast(tripleData); librdf_world* world = redlandTriplesData->getRedlandWorld(); mMemMan = updateConcreteOntology->getDataBoxes()->getBoxContext()->getMemoryAllocationManager(); librdf_node* rdfTypePredicate = librdf_new_node_from_uri_string(world, (const unsigned char*)PREFIX_RDF_TYPE); librdf_node* namedIndividualObject = librdf_new_node_from_uri_string(world, (const unsigned char*)PREFIX_OWL_NAMED_INDIVIDUAL); if (!mExistingEntitiesInitialized) { mExistingEntitiesInitialized = true; mRoleNodeDataHash.clear(); QSet* activeRoleSet = updateConcreteOntology->getRBox()->getActivePropertyRoleSet(false); if (activeRoleSet) { for (CRole* role : *activeRoleSet) { QByteArray roleCharData = CIRIName::getRecentIRIName(role->getPropertyNameLinker()).toUtf8(); librdf_node* roleNode = librdf_new_node_from_uri_string(world, (const unsigned char*)roleCharData.constData()); mRoleNodeDataHash.insert(CRedlandNodeHasher(roleNode), new CRoleNodeData(roleNode, role, role->isDataRole())); } } mConceptNodeDataHash.clear(); CConceptNodeData* topConceptData = nullptr; QSet* activeConceptSet = updateConcreteOntology->getTBox()->getActiveClassConceptSet(false); if (activeConceptSet) { for (CConcept* concept : *activeConceptSet) { QByteArray conceptCharData = CIRIName::getRecentIRIName(concept->getClassNameLinker()).toUtf8(); librdf_node* conceptNode = librdf_new_node_from_uri_string(world, (const unsigned char*)conceptCharData.constData()); CConceptNodeData* conceptData = new CConceptNodeData(conceptNode, concept); mConceptNodeDataHash.insert(CRedlandNodeHasher(conceptNode), conceptData); if (concept->getConceptTag() == 1) { topConceptData = conceptData; } } } mIndividualNodeDataHash.clear(); QSet* activeIndiSet = updateConcreteOntology->getABox()->getActiveIndividualSet(false); if (activeIndiSet && !activeIndiSet->isEmpty()) { cint64 maxIndividualId = updateConcreteOntology->getABox()->getNextIndividualId(false); mNextIndiId = qMax(maxIndividualId, mNextIndiId); for (CIndividual* indi : *activeIndiSet) { maxIndividualId = qMax(indi->getIndividualID(), maxIndividualId); QByteArray indiCharData = CIRIName::getRecentIRIName(indi->getIndividualNameLinker()).toUtf8(); librdf_node* indiNode = librdf_new_node_from_uri_string(world, (const unsigned char*)indiCharData.constData()); CIndividualNodeData* indiData = new CIndividualNodeData(indiNode, indi); mIndividualNodeDataHash.insert(CRedlandNodeHasher(indiNode), indiData); mNextIndiId = qMax(indi->getIndividualID(), mNextIndiId); } mNextIndiId++; } } } if (triplesDataList->count() < procCount * 2) { for (CTriplesData* tripleData : *ontologyTripleData->getAllTriplesData()) { CRedlandStoredTriplesData* redlandTriplesData = dynamic_cast(tripleData); librdf_world* world = redlandTriplesData->getRedlandWorld(); librdf_node* rdfTypePredicate = librdf_new_node_from_uri_string(world, (const unsigned char*)PREFIX_RDF_TYPE); librdf_node* namedIndividualObject = librdf_new_node_from_uri_string(world, (const unsigned char*)PREFIX_OWL_NAMED_INDIVIDUAL); indexTriples(redlandTriplesData, rdfTypePredicate, namedIndividualObject, updateConcreteOntology); } bool deleteIndexedTriplesData = CConfigDataReader::readConfigBoolean(updateConcreteOntology->getConfiguration(), "Konclude.Calculation.Preprocessing.TripleEncodedAssertionsIndexing.DeleteTriplesDataAfterIndexing", false); if (deleteIndexedTriplesData) { QVector triplesDataVec(triplesDataList->count()); cint64 tripleDataId = 0; for (CTriplesData* tripleData : *triplesDataList) { CRedlandStoredTriplesData* redlandTriplesData = (CRedlandStoredTriplesData*)tripleData; ensureDatatypeHashInitialized(updateConcreteOntology, redlandTriplesData); triplesDataVec[tripleDataId++] = redlandTriplesData; } QtConcurrent::blockingMap(triplesDataVec, mTripleDeleteMapFunc); } } else { indexTripleData(triplesDataList, updateConcreteOntology); } for (cint64 i = 0; i < mIndiResolvingKeyCount; ++i) { CRedlandStoredTriplesIndividualResolvingHashMemoryManaged*& indiResolveHash = mIndiResolvingHashVec[i]; if (indiResolveHash) { mConceptAssertionCount += indiResolveHash->getConceptAssertionCount(); mObjectPropertyAssertionCount += indiResolveHash->getObjectPropertyAssertionCount(); mDataPropertyAssertionCount += indiResolveHash->getDataPropertyAssertionCount(); delete indiResolveHash; indiResolveHash = nullptr; } } LOG(INFO, "::Konclude::Reasoner::TripleEncodedAssertionsIndexer", QString("Identified %1 concept, %2 object property, and %3 data property assertions for overall %4 individuals in %5 ms.").arg(mConceptAssertionCount).arg(mObjectPropertyAssertionCount).arg(mDataPropertyAssertionCount).arg(mNextIndiId).arg(mIndexingTime.elapsed()), this); delete mPreprocessContext; return newTriplesMapped; } void CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer::procssIndividualIndexing(CRedlandStoredTriplesIndexingBaseBatchLinkerVector& mappedIndi1HashKeyVec, CRedlandStoredTriplesData* redlandTriplesData, CConcreteOntology* updateConcreteOntology) { function(const CRedlandStoredTriplesIndexingBaseBatchLinkerVectorData* const & batchData)> individual2ResolveKeyMapFunc = [&](const CRedlandStoredTriplesIndexingBaseBatchLinkerVectorData* const & batchData) ->CRedlandStoredTriplesIndexingBaseBatchLinkerVector { CRedlandStoredTriplesIndexingBaseBatchLinkerVector keyMappedVec(mIndiResolvingKeyCount); if (batchData) { CRedlandStoredTriplesIndexingBaseResolvingLinker* resolveLinker = batchData->getNextBatchLinker(); while (resolveLinker) { cint64 index = qAbs(resolveLinker->mIndi1HashValue % mIndiResolvingKeyCount); CRedlandStoredTriplesIndividualResolvingHashMemoryManaged*& indiResolveHash = mIndiResolvingHashVec[index]; if (!indiResolveHash) { indiResolveHash = new CRedlandStoredTriplesIndividualResolvingHashMemoryManaged(mUupdatingOntology, mUupdatingOntology->getConfiguration(), mAnonymousOntologyIdentifier); } CRedlandStoredTriplesIndexingBaseResolvingLinker::ASSERTION_TYPE assertionType = (CRedlandStoredTriplesIndexingBaseResolvingLinker::ASSERTION_TYPE)resolveLinker->getData(); librdf_statement* statement = resolveLinker->mStatement; librdf_node* predicateNode = librdf_statement_get_predicate(statement); librdf_node* objectNode = librdf_statement_get_object(statement); librdf_node* subjectNode = librdf_statement_get_subject(statement); resolveLinker->mIndi1 = getIndividual(subjectNode, indiResolveHash, &keyMappedVec); if (assertionType == CRedlandStoredTriplesIndexingBaseResolvingLinker::ASSERTION_CONCEPT) { CConceptAssertionLinker* conAssLinker = indiResolveHash->createAssertionConceptLinker(); conAssLinker->initNegLinker(resolveLinker->mConcept, false); resolveLinker->mIndi1->addAssertionConceptLinker(conAssLinker); resolveLinker = resolveLinker->getNext(); } else if (assertionType == CRedlandStoredTriplesIndexingBaseResolvingLinker::ASSERTION_DATA) { CRedlandStoredTriplesIndividualPreProcessMemoryManagingContext* indiMemManContext = indiResolveHash->getIndividualDataMemoryManager(); CDataAssertionLinker* dataAssLinker = indiResolveHash->createAssertionDataLinker(); const char* literalValue = (const char*)librdf_node_get_literal_value(objectNode); librdf_uri* datatypeUri = librdf_node_get_literal_value_datatype_uri(objectNode); CDatatype* datatype = getDatatypeFromDatatypeUri(datatypeUri, mUupdatingOntology, redlandTriplesData); CDataLiteral* dataLiteral = CObjectParameterizingAllocator::allocateAndConstructAndParameterize(indiMemManContext->getMemoryAllocationManager(), indiMemManContext); CDataLiteralNormalizerPreProcess* dataLiteralNormalizer = indiResolveHash->getIndividualDataLiteralNormalizer(); dataLiteral->initDataLiteral(QString::fromUtf8(literalValue), datatype); dataLiteralNormalizer->createPreprocessedDataLiteralValue(dataLiteral, indiMemManContext, indiMemManContext); dataAssLinker->initDataAssertionLinker(resolveLinker->mRole, dataLiteral); resolveLinker->mIndi1->addAssertionDataLinker(dataAssLinker); resolveLinker = resolveLinker->getNext(); } else if (assertionType == CRedlandStoredTriplesIndexingBaseResolvingLinker::ASSERTION_ROLE) { cint64 nextIndex = qAbs(resolveLinker->mIndi2HashValue % mIndiResolvingKeyCount); CRoleAssertionLinker* roleAssLinker = indiResolveHash->createAssertionRoleLinker(); resolveLinker->mRoleAssLinker = roleAssLinker; resolveLinker->mIndi1->addAssertionRoleLinker(roleAssLinker); CRedlandStoredTriplesIndexingBaseResolvingLinker* tmpResolveLinker = resolveLinker; resolveLinker = resolveLinker->getNext(); tmpResolveLinker->clearNext(); keyMappedVec.addBatchLinker(nextIndex, tmpResolveLinker); } else { resolveLinker = resolveLinker->getNext(); } } } return keyMappedVec; }; function(const CRedlandStoredTriplesIndexingBaseBatchLinkerVectorData* const & batchData)> individual3ResolveKeyMapFunc = [&](const CRedlandStoredTriplesIndexingBaseBatchLinkerVectorData* const & batchData) ->CRedlandStoredTriplesIndexingBaseBatchLinkerVector { CRedlandStoredTriplesIndexingBaseBatchLinkerVector keyMappedVec(mIndiResolvingKeyCount); if (batchData) { CRedlandStoredTriplesIndexingBaseResolvingLinker* resolveLinker = batchData->getNextBatchLinker(); while (resolveLinker) { cint64 index = qAbs(resolveLinker->mIndi2HashValue % mIndiResolvingKeyCount); CRedlandStoredTriplesIndividualResolvingHashMemoryManaged*& indiResolveHash = mIndiResolvingHashVec[index]; if (!indiResolveHash) { indiResolveHash = new CRedlandStoredTriplesIndividualResolvingHashMemoryManaged(mUupdatingOntology, mUupdatingOntology->getConfiguration(), mAnonymousOntologyIdentifier); } librdf_statement* statement = resolveLinker->mStatement; librdf_node* objectNode = librdf_statement_get_object(statement); resolveLinker->mIndi2 = getIndividual(objectNode, indiResolveHash, &keyMappedVec); CRedlandStoredTriplesIndexingBaseResolvingLinker::ASSERTION_TYPE assertionType = (CRedlandStoredTriplesIndexingBaseResolvingLinker::ASSERTION_TYPE)resolveLinker->getData(); if (assertionType == CRedlandStoredTriplesIndexingBaseResolvingLinker::ASSERTION_ROLE) { CRoleAssertionLinker* roleAssLinker = resolveLinker->mRoleAssLinker; roleAssLinker->initRoleAssertionLinker(resolveLinker->mRole, resolveLinker->mIndi2); CReverseRoleAssertionLinker* reverseRoleAssLinker = indiResolveHash->createReverseAssertionRoleLinker(); reverseRoleAssLinker->initReverseRoleAssertionLinker(roleAssLinker, resolveLinker->mIndi1); resolveLinker->mIndi2->addReverseAssertionRoleLinker(reverseRoleAssLinker); } resolveLinker = resolveLinker->getNext(); } } return keyMappedVec; }; CRedlandStoredTriplesIndexingBaseBatchLinkerVector mappedIndi2HashKeyVec = QtConcurrent::blockingMappedReduced>(mappedIndi1HashKeyVec, individual2ResolveKeyMapFunc, mIndiKeyReduceFunc); CRedlandStoredTriplesIndexingBaseBatchLinkerVector mappedIndi3HashKeyVec = QtConcurrent::blockingMappedReduced>(mappedIndi2HashKeyVec, individual3ResolveKeyMapFunc, mIndiKeyReduceFunc); CMemoryPool* mappedIndi1HashKeyVecMemoryPools = mappedIndi1HashKeyVec.takeMemoryPools(); CMemoryPool* mappedIndi2HashKeyVecMemoryPools = mappedIndi2HashKeyVec.takeMemoryPools(); CMemoryPool* mappedIndi3HashKeyVecMemoryPools = mappedIndi3HashKeyVec.takeMemoryPools(); //QtConcurrent::run(QThreadPool::globalInstance(), [=] { clearMemoryPools(mappedIndi1HashKeyVecMemoryPools); //}); //QtConcurrent::run(QThreadPool::globalInstance(), [=] { clearMemoryPools(mappedIndi2HashKeyVecMemoryPools); //}); //QtConcurrent::run(QThreadPool::globalInstance(), [=] { clearMemoryPools(mappedIndi3HashKeyVecMemoryPools); //}); } void CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer::clearMemoryPools(CMemoryPool* memPools) { while (memPools) { CMemoryPool* tmpMemPool = memPools; memPools = memPools->getNextMemoryPool(); delete[] tmpMemPool->getMemoryBlockData(); delete tmpMemPool; } } void CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer::indexTripleData(QList* triplesDataList, CConcreteOntology* updateConcreteOntology) { QVector triplesDataVec(triplesDataList->count()); cint64 tripleDataId = 0; for (CTriplesData* tripleData : *triplesDataList) { CRedlandStoredTriplesData* redlandTriplesData = (CRedlandStoredTriplesData*)tripleData; ensureDatatypeHashInitialized(updateConcreteOntology, redlandTriplesData); triplesDataVec[tripleDataId++] = redlandTriplesData; } indexHashExistingIndividuals(updateConcreteOntology); function(CRedlandStoredTriplesData* redlandTriplesData)> individual1ResolveKeyMapFunc = [&](CRedlandStoredTriplesData* redlandTriplesData) ->CRedlandStoredTriplesIndexingBaseBatchLinkerVector { CRedlandStoredTriplesIndexingBaseBatchLinkerVector keyMappedVec(mIndiResolvingKeyCount); if (redlandTriplesData) { librdf_world* world = redlandTriplesData->getRedlandWorld(); mMemMan = updateConcreteOntology->getDataBoxes()->getBoxContext()->getMemoryAllocationManager(); librdf_node* rdfTypePredicate = librdf_new_node_from_uri_string(world, (const unsigned char*)PREFIX_RDF_TYPE); librdf_node* namedIndividualObject = librdf_new_node_from_uri_string(world, (const unsigned char*)PREFIX_OWL_NAMED_INDIVIDUAL); CXLinker* statementLinker = redlandTriplesData->getRedlandStatementLinker(); if (statementLinker) { while (statementLinker) { librdf_statement* statement = statementLinker->getData(); handleStatementMapping(statement, keyMappedVec, rdfTypePredicate, namedIndividualObject); statementLinker = statementLinker->getNext(); } } else { librdf_stream* stream = librdf_model_as_stream(redlandTriplesData->getRedlandIndexedModel()); if (stream) { while (!librdf_stream_end(stream)) { librdf_statement* statement = librdf_stream_get_object(stream); handleStatementMapping(statement, keyMappedVec, rdfTypePredicate, namedIndividualObject); librdf_stream_next(stream); } librdf_free_stream(stream); } } } return keyMappedVec; }; CRedlandStoredTriplesIndexingBaseBatchLinkerVector mappedIndi1HashKeyVec = QtConcurrent::blockingMappedReduced>(triplesDataVec, individual1ResolveKeyMapFunc, mIndiKeyReduceFunc); procssIndividualIndexing(mappedIndi1HashKeyVec, nullptr, updateConcreteOntology); bool deleteIndexedTriplesData = CConfigDataReader::readConfigBoolean(updateConcreteOntology->getConfiguration(), "Konclude.Calculation.Preprocessing.TripleEncodedAssertionsIndexing.DeleteTriplesDataAfterIndexing", false); if (deleteIndexedTriplesData) { QtConcurrent::blockingMap(triplesDataVec, mTripleDeleteMapFunc); } updateConcreteOntology->getABox()->setNextIndividualId(mNextIndiId); } void CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer::indexTriples(CRedlandStoredTriplesData* redlandTriplesData, librdf_node* rdfTypePredicate, librdf_node* namedIndividualObject, CConcreteOntology* updateConcreteOntology) { CIndividualNodeData* lastIndiData = nullptr; librdf_node* lastSubjectNode = nullptr; cint64 statementBatchLimit = 1000; cint64 statementBatchCountLimit = 1000; CXLinker* statementLinker = redlandTriplesData->getRedlandStatementLinker(); if (statementLinker) { ensureDatatypeHashInitialized(updateConcreteOntology, redlandTriplesData); indexHashExistingIndividuals(updateConcreteOntology); CXLinker* statementLinkerIt = statementLinker; while (statementLinkerIt) { CRedlandStoredTriplesIndexingBaseBatchLinkerVector statementBatchingLinkerVec(mIndiResolvingKeyCount); cint64 statementBatchSize = 0; cint64 statementBatchCount = 0; while (statementLinkerIt && statementBatchCount < statementBatchCountLimit) { CXLinker* statementBeginLinker = statementLinkerIt; CXLinker* statementEndLinker = statementLinkerIt; statementBatchSize = 0; while (statementEndLinker && statementBatchSize < statementBatchLimit) { statementEndLinker = statementEndLinker->getNext(); ++statementBatchSize; } statementLinkerIt = statementEndLinker; CRedlandStoredTriplesIndexingBaseBatchLinker* linker = statementBatchingLinkerVec.createBatchLinker(); linker->initBatchLinker(statementBeginLinker, statementEndLinker, statementBatchSize); statementBatchingLinkerVec.addBatchLinker(statementBatchingLinkerVec.getNextIndex(true), linker); statementBatchCount++; } function(const CRedlandStoredTriplesIndexingBaseBatchLinkerVectorData* const & batchData)> individual1ResolveKeyMapFunc = [&](const CRedlandStoredTriplesIndexingBaseBatchLinkerVectorData* const & batchData) ->CRedlandStoredTriplesIndexingBaseBatchLinkerVector { CRedlandStoredTriplesIndexingBaseBatchLinkerVector keyMappedVec(mIndiResolvingKeyCount); if (batchData) { CRedlandStoredTriplesIndexingBaseBatchLinker* batchLinker = batchData->getNextBatchLinker(); while (batchLinker) { CXLinker* statementLinker = batchLinker->getNextStatementLinker(); while (statementLinker) { librdf_statement* statement = statementLinker->getData(); handleStatementMapping(statement, keyMappedVec, rdfTypePredicate, namedIndividualObject); statementLinker = batchLinker->getNextStatementLinker(); } batchLinker = batchLinker->getNext(); } } return keyMappedVec; }; CRedlandStoredTriplesIndexingBaseBatchLinkerVector mappedIndi1HashKeyVec = QtConcurrent::blockingMappedReduced>(statementBatchingLinkerVec, individual1ResolveKeyMapFunc, mIndiKeyReduceFunc); procssIndividualIndexing(mappedIndi1HashKeyVec, redlandTriplesData, updateConcreteOntology); CMemoryPool* statementBatchingLinkerVecMemoryPools = statementBatchingLinkerVec.takeMemoryPools(); //QtConcurrent::run(QThreadPool::globalInstance(), [=] { clearMemoryPools(statementBatchingLinkerVecMemoryPools); //}); } //CBoxContext* boxContext = updateConcreteOntology->getDataBoxes()->getBoxContext(); //for (cint64 i = 0; i < mIndiResolvingKeyCount; ++i) { // CRedlandStoredTriplesIndividualResolvingHashMemoryManaged* indiResolveHash = mIndiResolvingHashVec[i]; // if (indiResolveHash) { // CMemoryPool* memPools = indiResolveHash->takeIndividualDataMemoryPools(); // boxContext->addUsedMemoryPools(memPools); // for (CQtManagedRestrictedModificationHash::const_iterator it = indiResolveHash->constBegin(), itEnd = indiResolveHash->constEnd(); it != itEnd; ++it) { // CIndividual* indi = it.value(); // indi->setIndividualID(mNextIndiId++); // updateConcreteOntology->getABox()->getIndividualVector(true)->setData(indi->getIndividualID(), indi); // updateConcreteOntology->getABox()->getActiveIndividualSet(true)->insert(indi); // updateConcreteOntology->getStringMapping()->getIndividualNameIndividualMappingHash()->insert(CIRIName::getRecentIRIName(indi->getNameLinker()), indi); // } // } //} //while (statementLinker) { // librdf_statement* statement = statementLinker->getData(); // identifyIndividuals(statement, rdfTypePredicate, namedIndividualObject, lastIndiData, lastSubjectNode, updateConcreteOntology, conceptAssertionCount, redlandTriplesData, dataPropertyAssertionCount, objectPropertyAssertionCount); // statementLinker = statementLinker->getNext(); //} } else { librdf_stream* stream = librdf_model_as_stream(redlandTriplesData->getRedlandIndexedModel()); if (stream) { while (!librdf_stream_end(stream)) { librdf_statement* statement = librdf_stream_get_object(stream); identifyIndividuals(statement, rdfTypePredicate, namedIndividualObject, lastIndiData, lastSubjectNode, updateConcreteOntology, redlandTriplesData); librdf_stream_next(stream); } librdf_free_stream(stream); } } updateConcreteOntology->getABox()->setNextIndividualId(mNextIndiId); } void CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer::handleStatementMapping(librdf_statement* statement, CRedlandStoredTriplesIndexingBaseBatchLinkerVector &keyMappedVec, librdf_node* rdfTypePredicate, librdf_node* namedIndividualObject) { librdf_node* predicateNode = librdf_statement_get_predicate(statement); librdf_node* objectNode = librdf_statement_get_object(statement); librdf_node* subjectNode = librdf_statement_get_subject(statement); CRedlandStoredTriplesIndexingBaseResolvingLinker* resolvingLinker = keyMappedVec.createBatchLinker(); resolvingLinker->setData(CRedlandStoredTriplesIndexingBaseResolvingLinker::ASSERTION_NONE); resolvingLinker->mStatement = statement; resolvingLinker->mIndi1HashValue = getNodeHashValue(subjectNode); cint64 index = qAbs(resolvingLinker->mIndi1HashValue % mIndiResolvingKeyCount); if (librdf_node_equals(predicateNode, rdfTypePredicate)) { if (librdf_node_equals(objectNode, namedIndividualObject)) { resolvingLinker->setData(CRedlandStoredTriplesIndexingBaseResolvingLinker::ASSERTION_INDIVIDUAL); keyMappedVec.addBatchLinker(index, resolvingLinker); } else { resolvingLinker->setData(CRedlandStoredTriplesIndexingBaseResolvingLinker::ASSERTION_CONCEPT); CConceptNodeData* conceptData = mConceptNodeDataHash.value(CRedlandNodeHasher(objectNode)); if (conceptData) { resolvingLinker->mConcept = conceptData->mConcept; keyMappedVec.addBatchLinker(index, resolvingLinker); } } } else { CRoleNodeData* roleData = mRoleNodeDataHash.value(CRedlandNodeHasher(predicateNode)); if (roleData) { resolvingLinker->mRole = roleData->mRole; if (roleData->mDataRole) { if (librdf_node_is_literal(objectNode)) { resolvingLinker->setData(CRedlandStoredTriplesIndexingBaseResolvingLinker::ASSERTION_DATA); keyMappedVec.addBatchLinker(index, resolvingLinker); } } else { if (!librdf_node_is_literal(objectNode)) { resolvingLinker->setData(CRedlandStoredTriplesIndexingBaseResolvingLinker::ASSERTION_ROLE); keyMappedVec.addBatchLinker(index, resolvingLinker); resolvingLinker->mIndi2HashValue = getNodeHashValue(objectNode); } } } } } void CRedlandStoredTriplesIndividualAssertionConvertionQtConcurrentIndexer::indexHashExistingIndividuals(CConcreteOntology* updateConcreteOntology) { QSet* activeIndiSet = updateConcreteOntology->getABox()->getActiveIndividualSet(false); if (!mExistingIndividualsHashed) { mExistingIndividualsHashed = true; if (activeIndiSet && !activeIndiSet->isEmpty()) { for (QHash::const_iterator it = mIndividualNodeDataHash.constBegin(), itEnd = mIndividualNodeDataHash.constEnd(); it != itEnd; ++it) { cint64 hashValue = it.key().getNodeHashValue(); cint64 index = qAbs(hashValue % mIndiResolvingKeyCount); CRedlandStoredTriplesIndividualResolvingHashMemoryManaged*& indiResolveHash = mIndiResolvingHashVec[index]; if (!indiResolveHash) { indiResolveHash = new CRedlandStoredTriplesIndividualResolvingHashMemoryManaged(mUupdatingOntology, mUupdatingOntology->getConfiguration(), mAnonymousOntologyIdentifier); } indiResolveHash->insertIndividual(it.key(), it.value()->mIndividual); } } } } }; // end namespace Triples }; // end namespace Reasoner }; // end namespace Konclude #endif // !KONCLUDE_REDLAND_INTEGRATION