| 1 |
#!/usr/bin/env python |
| 2 |
|
| 3 |
__license__ = 'GPL v3' |
| 4 |
__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>' |
| 5 |
''' |
| 6 |
politico.eu |
| 7 |
''' |
| 8 |
|
| 9 |
import re |
| 10 |
|
| 11 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 12 |
|
| 13 |
|
| 14 |
class Politico(BasicNewsRecipe): |
| 15 |
title = 'Politico.eu' |
| 16 |
__author__ = 'unkn0wn, Darko Miletic and Sujata Raman' |
| 17 |
description = ( |
| 18 |
'We connect and empower professionals through nonpartisan journalism and actionable intelligence about European politics and policy. Download Weekly.' |
| 19 |
) |
| 20 |
publisher = 'Axel Springer SE.' |
| 21 |
category = 'news, politics, Europe' |
| 22 |
oldest_article = 7 # days |
| 23 |
max_articles_per_feed = 20 |
| 24 |
use_embedded_content = False |
| 25 |
no_stylesheets = True |
| 26 |
remove_javascript = True |
| 27 |
encoding = 'UTF-8' |
| 28 |
language = 'en' |
| 29 |
|
| 30 |
recipe_specific_options = { |
| 31 |
'days': { |
| 32 |
'short': 'Oldest article to download from this news source. In days ', |
| 33 |
'long': 'For example, 0.5, gives you articles from the past 12 hours', |
| 34 |
'default': str(oldest_article), |
| 35 |
} |
| 36 |
} |
| 37 |
|
| 38 |
def __init__(self, *args, **kwargs): |
| 39 |
BasicNewsRecipe.__init__(self, *args, **kwargs) |
| 40 |
d = self.recipe_specific_options.get('days') |
| 41 |
if d and isinstance(d, str): |
| 42 |
self.oldest_article = float(d) |
| 43 |
|
| 44 |
remove_empty_feeds = True |
| 45 |
ignore_duplicate_articles = ['url'] |
| 46 |
|
| 47 |
html2lrf_options = ['--comment', description, '--category', category, '--publisher', publisher, '--ignore-tables'] |
| 48 |
|
| 49 |
html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' |
| 50 |
|
| 51 |
keep_only_tags = [ |
| 52 |
dict(name='div', attrs={'class': ['container']}), |
| 53 |
] |
| 54 |
|
| 55 |
remove_tags = [ |
| 56 |
dict(name=['notags', 'embed', 'aside', 'object', 'link']), |
| 57 |
dict(name='div', attrs={'class': 'module-article-inline related-articles'}), |
| 58 |
dict(name='header', attrs={'class': 'summary-header'}), |
| 59 |
dict(name='div', attrs={'class': 'amazon-polly-wrapper'}), |
| 60 |
dict(name='div', attrs={'class': lambda x: x and 'story-meta' in x.split()}), |
| 61 |
dict(attrs={'class': lambda x: x and 'story-tools' in x.split()}), |
| 62 |
dict(attrs={'class': lambda x: x and 'social-sharing' in x.split()}), |
| 63 |
dict(attrs={'class': lambda x: x and 'story-meta__authors' in x.split()}), |
| 64 |
dict(attrs={'class': lambda x: x and 'story-meta__timestamp' in x.split()}), |
| 65 |
dict(attrs={'class': lambda x: x and 'story-continued' in x.split()}), |
| 66 |
dict(attrs={'class': lambda x: x and 'story-supplement' in x.split()}), |
| 67 |
dict(attrs={'class': lambda x: x and 'story-share' in x.split()}), |
| 68 |
dict(attrs={'class': lambda x: x and 'suggested' in x.split()}), |
| 69 |
dict(attrs={'class': lambda x: x and 'article__sidebar' in x.split()}), |
| 70 |
dict(name='svg'), |
| 71 |
dict(name='footer'), |
| 72 |
dict(name='span', attrs={'class': 'ad-label'}), |
| 73 |
dict(name='div', attrs={'class': 'pro-pill'}), |
| 74 |
dict(name='ol', attrs={'class': 'breadcrumbs__list'}), |
| 75 |
] |
| 76 |
|
| 77 |
preprocess_regexps = [(re.compile(r'<a\s[^>]*>([^<]*)</a>', re.DOTALL | re.IGNORECASE), lambda match: match.group(1))] |
| 78 |
|
| 79 |
remove_tags_after = dict(attrs={'class': lambda x: x and 'article__more-from' in x.split()}) |
| 80 |
|
| 81 |
extra_css = ''' |
| 82 |
body{font-family:Arial,Sans-serif;} |
| 83 |
element.style{color:#FF0000;font-family:Arial,Sans-serif;} |
| 84 |
.author{color:#808080;font-size:x-small;} |
| 85 |
a{ color:#003399;} |
| 86 |
.byline{color:#696969 ; font-size:x-small;} |
| 87 |
.story{color:#000000;} |
| 88 |
td{color:#000000;} |
| 89 |
.figcaption__inner, .article-meta, .authors, .datetime {font-size:small; } |
| 90 |
''' |
| 91 |
|
| 92 |
def get_article_url(self, article): |
| 93 |
url = BasicNewsRecipe.get_article_url(self, article) |
| 94 |
if 'politico.com' not in url: |
| 95 |
return url.split('?')[0] |
| 96 |
|
| 97 |
masthead_url = 'https://www.politico.eu/cdn-cgi/image/width=573,quality=80,onerror=redirect,format=auto/wp-content/uploads/2021/02/25/Politico_logo-01.png' |
| 98 |
|
| 99 |
def get_cover_url(self): |
| 100 |
soup = self.index_to_soup('https://www.politico.eu/') |
| 101 |
for cov in soup.findAll(attrs={'class': 'cta__image'}): |
| 102 |
if cov.find('a', attrs={'href': lambda x: x and 'edition.pagesuite-professional' in x}): |
| 103 |
return cov.a.img['src'] |
| 104 |
|
| 105 |
feeds = [ |
| 106 |
('Policy', 'https://www.politico.eu/section/policy/feed'), |
| 107 |
('Opinion', 'https://www.politico.eu/section/opinion/feed'), |
| 108 |
('Newsletter', 'https://www.politico.eu/newsletter/feed'), |
| 109 |
('Others', 'https://www.politico.eu/feed'), |
| 110 |
] |
| 111 |
|
| 112 |
def postprocess_html(self, soup, first): |
| 113 |
for tag in soup.findAll(name=['table', 'tr', 'td']): |
| 114 |
tag.name = 'div' |
| 115 |
return soup |