/* * Copyright (C) 2013-2015, 2019 by the Konclude Developer Team. * * This file is part of the reasoning system Konclude. * For details and support, see . * * Konclude is free software: you can redistribute it and/or modify * it under the terms of version 3 of the GNU Lesser General Public * License (LGPLv3) as published by the Free Software Foundation. * * Konclude is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU (Lesser) General Public License for more details. * * You should have received a copy of the GNU (Lesser) General Public * License along with Konclude. If not, see . * */ #ifdef KONCLUDE_REDLAND_INTEGRATION #include "CRDFRedlandRaptorSimpleConcurrentParser.h" namespace Konclude { namespace Parser { CRDFRedlandRaptorSimpleConcurrentParser::CRDFRedlandRaptorSimpleConcurrentParser(COntologyBuilder* ontologyBuilder, CTRIPLES_DATA_UPDATE_TYPE updateType, QString redlandParsingFormat, CConfiguration* configuration) : CRDFRedlandRaptorParser(ontologyBuilder, updateType, redlandParsingFormat, configuration) { mConcurrentOffsetSplitSize = CConfigDataReader::readConfigInteger(mConfiguration, "Konclude.Parser.RedlandRaptor.ConcurrentBytesBatchingSize", 50000000); } CRDFRedlandRaptorSimpleConcurrentParser::~CRDFRedlandRaptorSimpleConcurrentParser() { } class TParseOffsetData { public: TParseOffsetData(cint64 startOffset = 0, cint64 endOffset = 0) { mStartOffset = startOffset; mEndOffset = endOffset; } TParseOffsetData(const TParseOffsetData& data) { mStartOffset = data.mStartOffset; mEndOffset = data.mEndOffset; mIndex = data.mIndex; } cint64 mIndex = 0; cint64 mStartOffset = 0; cint64 mLastPos = 0; cint64 mEndOffset = 0; QIODevice* mDevice; bool mStartCharFound = false; bool mEndCharFound = false; QByteArray* mBasePrefixDataArray = nullptr; cint64 mBasePrefixDataArrayReadPos = 0; bool mBasePrefixDataArrayRead = false; }; int raptor_iostream_read_bytes_from_qiodevice_with_offset(void *context, void *ptr, size_t size, size_t nmemb) { TParseOffsetData* parseOffsetData = (TParseOffsetData*)context; char* begCharPtr = (char*)ptr; qint64 readElements = 0; if (!parseOffsetData->mBasePrefixDataArrayRead && parseOffsetData->mBasePrefixDataArray) { const char* data = parseOffsetData->mBasePrefixDataArray->constData(); int length = parseOffsetData->mBasePrefixDataArray->length(); while (readElements < nmemb && parseOffsetData->mBasePrefixDataArrayReadPos < length) { char* charPointer = (char*)ptr; int nextElByteCount = 0; while ((size_t)nextElByteCount < size && parseOffsetData->mBasePrefixDataArrayReadPos < length) { charPointer[nextElByteCount++] = data[parseOffsetData->mBasePrefixDataArrayReadPos++]; } if ((size_t)nextElByteCount >= size) { readElements++; nextElByteCount = 0; ptr = (void*)((char*)ptr + size); } } if (parseOffsetData->mBasePrefixDataArrayReadPos >= length) { parseOffsetData->mBasePrefixDataArrayRead = true; } } QIODevice* ioDevice = parseOffsetData->mDevice; parseOffsetData->mLastPos = ioDevice->pos(); cint64 maxSize = parseOffsetData->mEndOffset - parseOffsetData->mLastPos; if (maxSize > 0 && readElements < nmemb) { if (size == 1) { size_t nmembCount = qMin((size_t)(nmemb - readElements), (size_t)maxSize); readElements += ioDevice->read((char*)ptr, nmembCount); ptr = (void*)((char*)ptr + nmembCount); } else { size_t nmembCount = qMin((size_t)(nmemb - readElements), (size_t)maxSize/size); for (size_t i = 0; i < nmembCount && !ioDevice->atEnd(); ++i) { qint64 read = ioDevice->read((char*)ptr, size); if (read >= size) { ptr = (void*)((char*)ptr + size); readElements++; } else { break; } } } } bool hasSpace = false; bool hasPoint = false; while (readElements < nmemb && !parseOffsetData->mEndCharFound && !ioDevice->atEnd()) { char dataChar[2]; int nextElByteCount = 0; while (!ioDevice->atEnd() && (size_t)nextElByteCount < size) { cint64 read = ioDevice->read(dataChar, 1); if (read == 1) { if (dataChar[0] == ' ') { hasSpace = true; } else if (dataChar[0] == '.' && hasSpace && !hasPoint) { hasPoint = true; } else if (dataChar[0] == '\r' && hasPoint) { } else if (dataChar[0] == '\n' && hasPoint) { parseOffsetData->mEndCharFound = true; } else { hasSpace = false; } char* charPointer = (char*)ptr; charPointer[nextElByteCount++] = dataChar[0]; } } if ((size_t)nextElByteCount >= size) { readElements++; nextElByteCount = 0; ptr = (void*)((char*)ptr + size); } } if (readElements > nmemb) { cout << "elements buffer length overwritten" << endl; } if ((char*)ptr > begCharPtr + nmemb) { cout << "pointer buffer length overwritten" << endl; } return readElements; } int raptor_iostream_read_eof_func_from_qiodevice_with_offset(void *context) { TParseOffsetData* parseOffsetData = (TParseOffsetData*)context; QIODevice* ioDevice = parseOffsetData->mDevice; if (ioDevice->atEnd() || parseOffsetData->mEndCharFound) { return 1; } return 0; } bool CRDFRedlandRaptorSimpleConcurrentParser::parseTriples(QIODevice* ioDevice, const QString& baseUriString) { QString fileString = baseUriString; if (fileString.startsWith("file:")) { fileString = fileString.replace("file:", ""); } QFile* file = new QFile(fileString); while (!file->open(QIODevice::ReadOnly) && fileString.startsWith("/")) { fileString = fileString.mid(1); delete file; file = new QFile(fileString); } delete file; file = new QFile(fileString); if (!file->open(QIODevice::ReadOnly) || mConfLoadTriplesIntoStore || (mRedlandParsingFormat != "turtle" && mRedlandParsingFormat != "ntriples")) { delete file; return CRDFRedlandRaptorParser::parseTriples(ioDevice, baseUriString); } cint64 bytesAvailable = ioDevice->bytesAvailable(); delete file; class TParsedTripleData { public: CRedlandStoredTriplesData* mTriplesData = nullptr; }; QVector parseOffsetDataVec((bytesAvailable / mConcurrentOffsetSplitSize)+1); cint64 idx = 0; for (cint64 i = 0; i < bytesAvailable; i += mConcurrentOffsetSplitSize) { TParseOffsetData data = TParseOffsetData(i, i + mConcurrentOffsetSplitSize); data.mIndex = idx; parseOffsetDataVec[idx++] = data; } LOG(INFO, getLogDomain(), logTr("Starting concurrent triple parsing for file '%1' with %2 partitions.").arg(fileString).arg(parseOffsetDataVec.size()), this); QString* raptorParsingErrorString = nullptr; bool warningNoStore = false; QByteArray basePrefixDataArray; while (!ioDevice->atEnd()) { QByteArray data = ioDevice->readLine().trimmed(); QString dataString = data; dataString = dataString.toUpper(); bool isBasePrefixData = false; if (dataString.startsWith("PREFIX") || dataString.startsWith("@PREFIX")) { isBasePrefixData = true; } if (dataString.startsWith("BASE") || dataString.startsWith("@BASE")) { isBasePrefixData = true; } if (isBasePrefixData) { basePrefixDataArray.append(data); basePrefixDataArray.append(QString("\r\n")); } else if (!dataString.startsWith("#")) { break; } } function parseTriplesMapFunc = [&](const TParseOffsetData& parseData) ->CRedlandStoredTriplesData* { QFile file(fileString); if (parseData.mStartOffset != parseData.mEndOffset && file.open(QIODevice::ReadOnly)) { TParseOffsetData tmpParseData(parseData); tmpParseData.mDevice = &file; file.seek(parseData.mStartOffset); cint64 startSkipCount = 0; if (!tmpParseData.mStartCharFound && parseData.mStartOffset > 0) { tmpParseData.mBasePrefixDataArray = &basePrefixDataArray; char dataChar[2]; bool hasSpace = false; bool hasPoint = false; while (!tmpParseData.mStartCharFound && !tmpParseData.mDevice->atEnd()) { cint64 read = tmpParseData.mDevice->read(dataChar, 1); if (read == 1) { if (dataChar[0] == ' ') { hasSpace = true; } else if (dataChar[0] == '.' && hasSpace && !hasPoint) { hasPoint = true; } else if (dataChar[0] == '\r' && hasPoint) { } else if (dataChar[0] == '\n' && hasPoint) { tmpParseData.mStartCharFound = true; } else { hasSpace = false; } startSkipCount++; } } } else { tmpParseData.mBasePrefixDataArrayRead = true; tmpParseData.mStartCharFound = true; } raptor_iostream_handler qioDeviceIOStreamHandler; qioDeviceIOStreamHandler.version = 2; qioDeviceIOStreamHandler.read_bytes = &raptor_iostream_read_bytes_from_qiodevice_with_offset; qioDeviceIOStreamHandler.read_eof = &raptor_iostream_read_eof_func_from_qiodevice_with_offset; qioDeviceIOStreamHandler.init = nullptr; qioDeviceIOStreamHandler.finish = nullptr; qioDeviceIOStreamHandler.write_byte = nullptr; qioDeviceIOStreamHandler.write_bytes = nullptr; qioDeviceIOStreamHandler.write_end = nullptr; CRedlandStoredTriplesData* tripleData = getUpdatingTripleData(); raptor_world* raptor_world_ptr = librdf_world_get_raptor(tripleData->getRedlandWorld()); raptor_world_set_log_handler(raptor_world_ptr, &raptorParsingErrorString, &raptor_log_handler_error_notifier); raptor_iostream* raptor_stream = raptor_new_iostream_from_handler(raptor_world_ptr, &tmpParseData, &qioDeviceIOStreamHandler); QByteArray baseUriStringByteArray(baseUriString.toUtf8()); librdf_uri *baseUri = librdf_new_uri(tripleData->getRedlandWorld(), (const unsigned char*)baseUriStringByteArray.constData()); QByteArray mRedlandParsingFormatByteArray = mRedlandParsingFormat.toUtf8(); const char* parser_name = mRedlandParsingFormatByteArray.constData(); librdf_parser* parser = librdf_new_parser(tripleData->getRedlandWorld(), parser_name, NULL, NULL); if (!parser) { mErrorString = QString("Failed to create parser"); mParsingError = true; } //if (parser && raptor_stream && librdf_parser_parse_iostream_into_model(parser, raptor_stream, baseUri, tripleData->getRedlandStreamedModel())) { // mParsingError = true; //} librdf_stream* tripleStream = nullptr; if (parser && raptor_stream) { tripleStream = librdf_parser_parse_iostream_as_stream(parser, raptor_stream, baseUri); if (!tripleStream) { mErrorString = QString("Failed to load data stream"); mParsingError = true; } librdf_node* critPredicate1 = librdf_new_node_from_uri_string(tripleData->getRedlandWorld(), (const unsigned char*)PREFIX_RDFS_SUBCLASS_OF); librdf_node* critPredicate2 = librdf_new_node_from_uri_string(tripleData->getRedlandWorld(), (const unsigned char*)PREFIX_OWL_COMPLEMENT_OF); CXLinker* statementLinker = nullptr; CXLinker* lastStatementLinker = nullptr; while (!librdf_stream_end(tripleStream)) { librdf_statement* statement = librdf_stream_get_object(tripleStream); if (!librdf_model_contains_statement(tripleData->getRedlandIndexedModel(), statement)) { librdf_statement* statementCopy = librdf_new_statement_from_statement(statement); CXLinker* newStatementLinker = new CXLinker(); newStatementLinker->initLinker(statementCopy, nullptr); if (statementLinker) { lastStatementLinker->setNext(newStatementLinker); lastStatementLinker = newStatementLinker; } else { statementLinker = newStatementLinker; lastStatementLinker = newStatementLinker; } if (!warningNoStore && (librdf_node_equals(critPredicate1, librdf_statement_get_predicate(statementCopy)) || librdf_node_equals(critPredicate2, librdf_statement_get_predicate(statementCopy)))) { warningNoStore = true; LOG(WARN, getLogDomain(), logTr("(Nontrivial) OWL axioms and expressions cannot be extracted from triples since they are not loaded into the triple store."), this); } } librdf_stream_next(tripleStream); } librdf_free_node(critPredicate1); librdf_free_node(critPredicate2); tripleData->setRedlandStatementLinker(statementLinker); } if (raptor_stream) { raptor_free_iostream(raptor_stream); } if (parser) { librdf_free_parser(parser); } return tripleData; } return nullptr; }; QVector triplesDataVec = QtConcurrent::blockingMapped(parseOffsetDataVec, parseTriplesMapFunc); if (raptorParsingErrorString) { mParsingError = true; mErrorString = *raptorParsingErrorString; delete raptorParsingErrorString; raptorParsingErrorString = nullptr; } if (mParsingError) { qDeleteAll(triplesDataVec); } else { for (CRedlandStoredTriplesData* tripleData : triplesDataVec) { mOntologyBuilder->addTriplesData(tripleData); } } return !mParsingError; } }; // end namespace Parser }; // end namespace Konclude #endif // !KONCLUDE_REDLAND_INTEGRATION