/**************************************************************************************** * Copyright (c) 2007 Bart Cerneels * * 2009 Mathias Panzenböck * * 2013 Ralf Engels * * * * This program is free software; you can redistribute it and/or modify it under * * the terms of the GNU General Public License as published by the Free Software * * Foundation; either version 2 of the License, or (at your option) any later * * version. * * * * This program is distributed in the hope that it will be useful, but WITHOUT ANY * * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A * * PARTICULAR PURPOSE. See the GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License along with * * this program. If not, see . * ****************************************************************************************/ #include "core/podcasts/PodcastReader.h" #include "core/support/Amarok.h" #include "core/support/Components.h" #include "core/support/Debug.h" #include "core/meta/support/MetaUtility.h" #include #include #include #include using namespace Podcasts; #define ITUNES_NS QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtd") #define RDF_NS "http://www.w3.org/1999/02/22-rdf-syntax-ns#" #define RSS10_NS QStringLiteral("http://purl.org/rss/1.0/") #define RSS20_NS QStringLiteral("") #define ATOM_NS "http://www.w3.org/2005/Atom" #define ENC_NS "http://purl.oclc.org/net/rss_2.0/enc#" #define CONTENT_NS QStringLiteral("http://purl.org/rss/1.0/modules/content") #define DC_NS QStringLiteral("http://purl.org/dc/elements/1.1/") // regular expressions for linkification: #define RE_USER "[-+_%\\.\\w]+" #define RE_PASSWD RE_USER #define RE_DOMAIN "[-a-zA-Z0-9]+(?:\\.[-a-zA-Z0-9]+)*" #define RE_PROT "[a-zA-Z]+://" #define RE_URL RE_PROT "(?:" RE_USER "(?::" RE_PASSWD ")?@)?" RE_DOMAIN \ "(?::\\d+)?(?:/[-\\w\\?&=%+.,;:_#~/!@]*)?" #define RE_MAIL RE_USER "@" RE_DOMAIN const PodcastReader::StaticData PodcastReader::sd; PodcastReader::PodcastReader( PodcastProvider *podcastProvider, QObject *parent ) : QObject( parent ) , m_xmlReader() , m_podcastProvider( podcastProvider ) , m_transferJob( ) , m_current( nullptr ) , m_actionStack() , m_contentType( TextContent ) , m_buffer() {} void PodcastReader::Action::begin( PodcastReader *podcastReader ) const { if( m_begin ) (( *podcastReader ).*m_begin )(); } void PodcastReader::Action::end( PodcastReader *podcastReader ) const { if( m_end ) (( *podcastReader ).*m_end )(); } void PodcastReader::Action::characters( PodcastReader *podcastReader ) const { if( m_characters ) (( *podcastReader ).*m_characters )(); } // initialization of the feed parser automata: PodcastReader::StaticData::StaticData() : removeScripts( QStringLiteral("|]*>"), QRegularExpression::CaseInsensitiveOption ) , mightBeHtml( QStringLiteral("<\\?xml[^>]*\\?>|]*>|]*>|<|>|&|"|" "<([-:\\w\\d]+)[^>]*(/>|>.*)|]*>|&#\\d+;|&#x[a-fA-F\\d]+;"), QRegularExpression::CaseInsensitiveOption ) , linkify( QStringLiteral("\\b(" RE_URL ")|\\b(" RE_MAIL ")|(\n)") ) , startAction( rootMap ) , docAction( docMap, nullptr, &PodcastReader::endDocument ) , xmlAction( xmlMap, &PodcastReader::beginXml, &PodcastReader::endXml, &PodcastReader::readEscapedCharacters ) , skipAction( skipMap ) , noContentAction( noContentMap, &PodcastReader::beginNoElement, nullptr, &PodcastReader::readNoCharacters ) , rdfAction( rdfMap, &PodcastReader::beginRdf ) , rssAction( rssMap, &PodcastReader::beginRss ) , feedAction( feedMap, &PodcastReader::beginFeed ) , htmlAction( skipMap, &PodcastReader::beginHtml ) , unknownFeedTypeAction( skipMap, &PodcastReader::beginUnknownFeedType ) // RSS 1.0+2.0 , rss10ChannelAction( rss10ChannelMap, &PodcastReader::beginChannel ) , rss20ChannelAction( rss20ChannelMap, &PodcastReader::beginChannel ) , titleAction( textMap, &PodcastReader::beginText, &PodcastReader::endTitle, &PodcastReader::readCharacters ) , subtitleAction( textMap, &PodcastReader::beginText, &PodcastReader::endSubtitle, &PodcastReader::readCharacters ) , descriptionAction( textMap, &PodcastReader::beginText, &PodcastReader::endDescription, &PodcastReader::readCharacters ) , encodedAction( textMap, &PodcastReader::beginText, &PodcastReader::endEncoded, &PodcastReader::readCharacters ) , bodyAction( xmlMap, &PodcastReader::beginText, &PodcastReader::endBody, &PodcastReader::readEscapedCharacters ) , linkAction( textMap, &PodcastReader::beginText, &PodcastReader::endLink, &PodcastReader::readCharacters ) , imageAction( imageMap, &PodcastReader::beginImage ) , itemAction( itemMap, &PodcastReader::beginItem, &PodcastReader::endItem ) , urlAction( textMap, &PodcastReader::beginText, &PodcastReader::endImageUrl, &PodcastReader::readCharacters ) , authorAction( textMap, &PodcastReader::beginText, &PodcastReader::endAuthor, &PodcastReader::readCharacters ) , creatorAction( textMap, &PodcastReader::beginText, &PodcastReader::endCreator, &PodcastReader::readCharacters ) , enclosureAction( noContentMap, &PodcastReader::beginEnclosure ) , guidAction( textMap, &PodcastReader::beginText, &PodcastReader::endGuid, &PodcastReader::readCharacters ) , pubDateAction( textMap, &PodcastReader::beginText, &PodcastReader::endPubDate, &PodcastReader::readCharacters ) , keywordsAction( textMap, &PodcastReader::beginText, &PodcastReader::endKeywords, &PodcastReader::readCharacters ) , newFeedUrlAction( textMap, &PodcastReader::beginText, &PodcastReader::endNewFeedUrl, &PodcastReader::readCharacters ) // Atom , atomLogoAction( textMap, &PodcastReader::beginText, &PodcastReader::endImageUrl, &PodcastReader::readCharacters ) , atomIconAction( textMap, &PodcastReader::beginText, &PodcastReader::endAtomIcon, &PodcastReader::readCharacters ) , atomEntryAction( atomEntryMap, &PodcastReader::beginItem, &PodcastReader::endItem ) , atomTitleAction( atomTextMap, &PodcastReader::beginAtomText, &PodcastReader::endAtomTitle, &PodcastReader::readAtomTextCharacters ) , atomSubtitleAction( atomTextMap, &PodcastReader::beginAtomText, &PodcastReader::endAtomSubtitle, &PodcastReader::readAtomTextCharacters ) , atomAuthorAction( atomAuthorMap ) , atomFeedLinkAction( noContentMap, &PodcastReader::beginAtomFeedLink, nullptr, &PodcastReader::readNoCharacters ) , atomEntryLinkAction( noContentMap, &PodcastReader::beginAtomEntryLink, nullptr, &PodcastReader::readNoCharacters ) , atomIdAction( textMap, &PodcastReader::beginText, &PodcastReader::endGuid, &PodcastReader::readCharacters ) , atomPublishedAction( textMap, &PodcastReader::beginText, &PodcastReader::endAtomPublished, &PodcastReader::readCharacters ) , atomUpdatedAction( textMap, &PodcastReader::beginText, &PodcastReader::endAtomUpdated, &PodcastReader::readCharacters ) , atomSummaryAction( atomTextMap, &PodcastReader::beginAtomText, &PodcastReader::endAtomSummary, &PodcastReader::readAtomTextCharacters ) , atomContentAction( atomTextMap, &PodcastReader::beginAtomText, &PodcastReader::endAtomContent, &PodcastReader::readAtomTextCharacters ) , atomTextAction( atomTextMap, &PodcastReader::beginAtomTextChild, &PodcastReader::endAtomTextChild, &PodcastReader::readAtomTextCharacters ) { // known elements: knownElements[ QStringLiteral("rss") ] = Rss; knownElements[ QStringLiteral("RDF") ] = Rdf; knownElements[ QStringLiteral("feed") ] = Feed; knownElements[ QStringLiteral("channel") ] = Channel; knownElements[ QStringLiteral("item") ] = Item; knownElements[ QStringLiteral("image") ] = Image; knownElements[ QStringLiteral("link") ] = Link; knownElements[ QStringLiteral("url") ] = Url; knownElements[ QStringLiteral("title") ] = Title; knownElements[ QStringLiteral("author") ] = Author; knownElements[ QStringLiteral("enclosure") ] = EnclosureElement; knownElements[ QStringLiteral("guid") ] = Guid; knownElements[ QStringLiteral("pubDate") ] = PubDate; knownElements[ QStringLiteral("description") ] = Description; knownElements[ QStringLiteral("summary") ] = Summary; knownElements[ QStringLiteral("body") ] = Body; knownElements[ QStringLiteral("entry") ] = Entry; knownElements[ QStringLiteral("content") ] = Content; knownElements[ QStringLiteral("name") ] = Name; knownElements[ QStringLiteral("id") ] = Id; knownElements[ QStringLiteral("subtitle") ] = Subtitle; knownElements[ QStringLiteral("updated") ] = Updated; knownElements[ QStringLiteral("published") ] = Published; knownElements[ QStringLiteral("logo") ] = Logo; knownElements[ QStringLiteral("icon") ] = Icon; knownElements[ QStringLiteral("encoded") ] = Encoded; knownElements[ QStringLiteral("creator") ] = Creator; knownElements[ QStringLiteral("keywords") ] = Keywords; knownElements[ QStringLiteral("new-feed-url") ] = NewFeedUrl; knownElements[ QStringLiteral("html") ] = Html; knownElements[ QStringLiteral("HTML") ] = Html; // before start document/after end document rootMap.insert( Document, &docAction ); // parse document docMap.insert( Rss, &rssAction ); docMap.insert( Html, &htmlAction ); docMap.insert( Rdf, &rdfAction ); docMap.insert( Feed, &feedAction ); docMap.insert( Any, &unknownFeedTypeAction ); // parse "RSS 2.0" rssMap.insert( Channel, &rss20ChannelAction ); // parse "RSS 1.0" rdfMap.insert( Channel, &rss10ChannelAction ); rdfMap.insert( Item, &itemAction ); // parse "RSS 2.0" rss20ChannelMap.insert( Title, &titleAction ); rss20ChannelMap.insert( ItunesSubtitle, &subtitleAction ); rss20ChannelMap.insert( ItunesAuthor, &authorAction ); rss20ChannelMap.insert( Creator, &creatorAction ); rss20ChannelMap.insert( Description, &descriptionAction ); rss20ChannelMap.insert( Encoded, &encodedAction ); rss20ChannelMap.insert( ItunesSummary, &descriptionAction ); rss20ChannelMap.insert( Body, &bodyAction ); rss20ChannelMap.insert( Link, &linkAction ); rss20ChannelMap.insert( Image, &imageAction ); rss20ChannelMap.insert( ItunesKeywords, &keywordsAction ); rss20ChannelMap.insert( NewFeedUrl, &newFeedUrlAction ); rss20ChannelMap.insert( Item, &itemAction ); // parse "RSS 1.0" rss10ChannelMap.insert( Title, &titleAction ); rss10ChannelMap.insert( ItunesSubtitle, &subtitleAction ); rss10ChannelMap.insert( ItunesAuthor, &authorAction ); rss10ChannelMap.insert( Creator, &creatorAction ); rss10ChannelMap.insert( Description, &descriptionAction ); rss10ChannelMap.insert( Encoded, &encodedAction ); rss10ChannelMap.insert( ItunesSummary, &descriptionAction ); rss10ChannelMap.insert( Body, &bodyAction ); rss10ChannelMap.insert( Link, &linkAction ); rss10ChannelMap.insert( Image, &imageAction ); rss10ChannelMap.insert( ItunesKeywords, &keywordsAction ); rss10ChannelMap.insert( NewFeedUrl, &newFeedUrlAction ); // parse imageMap.insert( Title, &skipAction ); imageMap.insert( Link, &skipAction ); imageMap.insert( Url, &urlAction ); // parse itemMap.insert( Title, &titleAction ); itemMap.insert( ItunesSubtitle, &subtitleAction ); itemMap.insert( Author, &authorAction ); itemMap.insert( ItunesAuthor, &authorAction ); itemMap.insert( Creator, &creatorAction ); itemMap.insert( Description, &descriptionAction ); itemMap.insert( Encoded, &encodedAction ); itemMap.insert( ItunesSummary, &descriptionAction ); itemMap.insert( Body, &bodyAction ); itemMap.insert( EnclosureElement, &enclosureAction ); itemMap.insert( Guid, &guidAction ); itemMap.insert( PubDate, &pubDateAction ); itemMap.insert( ItunesKeywords, &keywordsAction ); // TODO: move the link field from PodcastChannel to PodcastMetaCommon // itemMap.insert( Link, &linkAction ); // parse "Atom" feedMap.insert( Title, &atomTitleAction ); feedMap.insert( Subtitle, &atomSubtitleAction ); feedMap.insert( Icon, &atomIconAction ); feedMap.insert( Logo, &atomLogoAction ); feedMap.insert( Author, &atomAuthorAction ); feedMap.insert( Link, &atomFeedLinkAction ); feedMap.insert( Entry, &atomEntryAction ); // parse "Atom" atomEntryMap.insert( Title, &atomTitleAction ); atomEntryMap.insert( Subtitle, &atomSubtitleAction ); atomEntryMap.insert( Author, &atomAuthorAction ); atomEntryMap.insert( Id, &atomIdAction ); atomEntryMap.insert( Published, &atomPublishedAction ); atomEntryMap.insert( Updated, &atomUpdatedAction ); atomEntryMap.insert( Summary, &atomSummaryAction ); atomEntryMap.insert( Link, &atomEntryLinkAction ); atomEntryMap.insert( SupportedContent, &atomContentAction ); // parse "Atom" atomAuthorMap.insert( Name, &authorAction ); // parse atom text atomTextMap.insert( Any, &atomTextAction ); // parse arbitrary xml xmlMap.insert( Any, &xmlAction ); // skip elements skipMap.insert( Any, &skipAction ); } PodcastReader::~PodcastReader() { DEBUG_BLOCK } bool PodcastReader::mightBeHtml( const QString& text ) //Static { return text.indexOf( sd.mightBeHtml ) != -1; } bool PodcastReader::read( QIODevice *device ) { DEBUG_BLOCK m_xmlReader.setDevice( device ); return read(); } bool PodcastReader::read( const QUrl &url ) { DEBUG_BLOCK m_url = url; m_transferJob = KIO::get( m_url, KIO::Reload, KIO::HideProgressInfo ); connect( m_transferJob, &KIO::TransferJob::data, this, &PodcastReader::slotAddData ); connect( m_transferJob, &KIO::TransferJob::result, this, &PodcastReader::downloadResult ); connect( m_transferJob, &KIO::TransferJob::redirection, this, &PodcastReader::slotRedirection ); connect( m_transferJob, &KIO::TransferJob::permanentRedirection, this, &PodcastReader::slotPermanentRedirection ); QString description = i18n( "Importing podcast channel from %1", url.url() ); if( m_channel ) { description = m_channel->title().isEmpty() ? i18n( "Updating podcast channel" ) : i18n( "Updating \"%1\"", m_channel->title() ); } Q_EMIT statusBarNewProgressOperation( m_transferJob, description, this ); // parse data return read(); } void PodcastReader::slotAbort() { DEBUG_BLOCK } bool PodcastReader::update( const PodcastChannelPtr &channel ) { DEBUG_BLOCK m_channel = channel; return read( m_channel->url() ); } void PodcastReader::slotAddData( KIO::Job *job, const QByteArray &data ) { DEBUG_BLOCK Q_UNUSED( job ) m_xmlReader.addData( data ); // parse more data continueRead(); } void PodcastReader::downloadResult( KJob * job ) { DEBUG_BLOCK // parse more data continueRead(); KIO::TransferJob *transferJob = dynamic_cast( job ); if( transferJob && transferJob->isErrorPage() ) { QString errorMessage = i18n( "Importing podcast from %1 failed with error:\n", m_url.url() ); if( m_channel ) { errorMessage = m_channel->title().isEmpty() ? i18n( "Updating podcast from %1 failed with error:\n", m_url.url() ) : i18n( "Updating \"%1\" failed with error:\n", m_channel->title() ); } errorMessage = errorMessage.append( job->errorString() ); Q_EMIT statusBarErrorMessage( errorMessage ); } else if( job->error() ) { QString errorMessage = i18n( "Importing podcast from %1 failed with error:\n", m_url.url() ); if( m_channel ) { errorMessage = m_channel->title().isEmpty() ? i18n( "Updating podcast from %1 failed with error:\n", m_url.url() ) : i18n( "Updating \"%1\" failed with error:\n", m_channel->title() ); } errorMessage = errorMessage.append( job->errorString() ); Q_EMIT statusBarErrorMessage( errorMessage ); } m_transferJob = nullptr; } PodcastReader::ElementType PodcastReader::elementType() const { if( m_xmlReader.isEndDocument() || m_xmlReader.isStartDocument() ) return Document; if( m_xmlReader.isCDATA() || m_xmlReader.isCharacters() ) return CharacterData; ElementType elementType = sd.knownElements[ m_xmlReader.name().toString()]; // This is a bit hacky because my automata does not support conditions. // Therefore I put the decision logic in here and declare some pseudo elements. // I don't think it is worth it to extend the automata to support such conditions. switch( elementType ) { case Summary: if( m_xmlReader.namespaceUri() == ITUNES_NS ) { elementType = ItunesSummary; } break; case Subtitle: if( m_xmlReader.namespaceUri() == ITUNES_NS ) { elementType = ItunesSubtitle; } break; case Author: if( m_xmlReader.namespaceUri() == ITUNES_NS ) { elementType = ItunesAuthor; } break; case Keywords: if( m_xmlReader.namespaceUri() == ITUNES_NS ) { elementType = ItunesKeywords; } break; case Content: if( m_xmlReader.namespaceUri() == QStringLiteral( ATOM_NS ) && // ignore atom:content elements that do not // have content but only refer to some url: !hasAttribute( ATOM_NS, "src" ) ) { // Atom supports arbitrary Base64 encoded content. // Because we can only something with text/html/xhtml I ignore // anything else. // See: // http://tools.ietf.org/html/rfc4287#section-4.1.3 if( hasAttribute( ATOM_NS, "type" ) ) { QStringView type( attribute( ATOM_NS, "type" ) ); if( type == QStringLiteral("text") || type == QStringLiteral("html") || type == QStringLiteral("xhtml") ) { elementType = SupportedContent; } } else { elementType = SupportedContent; } } break; default: break; } return elementType; } bool PodcastReader::read() { DEBUG_BLOCK m_current = nullptr; m_item = nullptr; m_contentType = TextContent; m_buffer.clear(); m_actionStack.clear(); m_actionStack.push( &( PodcastReader::sd.startAction ) ); m_xmlReader.setNamespaceProcessing( true ); return continueRead(); } bool PodcastReader::continueRead() { // this is some kind of pushdown automata // with this it should be possible to parse feeds in parallel // without using threads DEBUG_BLOCK while( !m_xmlReader.atEnd() && m_xmlReader.error() != QXmlStreamReader::CustomError ) { QXmlStreamReader::TokenType token = m_xmlReader.readNext(); if( m_xmlReader.error() == QXmlStreamReader::PrematureEndOfDocumentError && m_transferJob ) { return true; } if( m_xmlReader.hasError() ) { Q_EMIT finished( this ); return false; } if( m_actionStack.isEmpty() ) { debug() << "expected element on stack!"; return false; } const Action* action = m_actionStack.top(); const Action* subAction = nullptr; switch( token ) { case QXmlStreamReader::Invalid: return false; case QXmlStreamReader::StartDocument: case QXmlStreamReader::StartElement: subAction = action->actionMap()[ elementType()]; if( !subAction ) subAction = action->actionMap()[ Any ]; if( !subAction ) subAction = &( PodcastReader::sd.skipAction ); m_actionStack.push( subAction ); subAction->begin( this ); break; case QXmlStreamReader::EndDocument: case QXmlStreamReader::EndElement: action->end( this ); if( m_actionStack.pop() != action ) { debug() << "popped other element than expected!"; } break; case QXmlStreamReader::Characters: if( !m_xmlReader.isWhitespace() || m_xmlReader.isCDATA() ) { action->characters( this ); } break; // ignorable whitespaces case QXmlStreamReader::Comment: case QXmlStreamReader::EntityReference: case QXmlStreamReader::ProcessingInstruction: case QXmlStreamReader::DTD: case QXmlStreamReader::NoToken: // ignore break; } } return !m_xmlReader.hasError(); } void PodcastReader::stopWithError( const QString &message ) { m_xmlReader.raiseError( message ); if( m_transferJob ) { m_transferJob->kill(KJob::EmitResult); m_transferJob = nullptr; } Q_EMIT finished( this ); } void PodcastReader::beginText() { m_buffer.clear(); } void PodcastReader::endTitle() { m_current->setTitle( m_buffer.trimmed() ); } void PodcastReader::endSubtitle() { m_current->setSubtitle( m_buffer.trimmed() ); } QString PodcastReader::atomTextAsText() { switch( m_contentType ) { case HtmlContent: case XHtmlContent: // TODO: strip tags (there should not be any non-xml entities here) return unescape( m_buffer ); case TextContent: default: return m_buffer; } } QString PodcastReader::atomTextAsHtml() { switch( m_contentType ) { case HtmlContent: case XHtmlContent: // strip