| 1 |
#!/usr/bin/env python |
| 2 |
from calibre.web.feeds.recipes import BasicNewsRecipe |
| 3 |
|
| 4 |
|
| 5 |
class Volkskrant(BasicNewsRecipe): |
| 6 |
title = 'Internazionale' |
| 7 |
__author__ = 'Cristi Ghera' |
| 8 |
max_articles_per_feed = 100 |
| 9 |
description = 'Internazionale - Notizie dall’Italia e dal mondo' |
| 10 |
needs_subscription = False |
| 11 |
language = 'it' |
| 12 |
country = 'IT' |
| 13 |
category = 'news, politics, Italy, world' |
| 14 |
resolve_internal_links = True |
| 15 |
remove_tags_before = {'name': 'article'} |
| 16 |
remove_tags_after = {'name': 'article'} |
| 17 |
remove_tags = [ |
| 18 |
dict( |
| 19 |
attrs={ |
| 20 |
'class': [ |
| 21 |
'item-banner', |
| 22 |
'hentryfeed__side', |
| 23 |
'magazine-article-share-tools', |
| 24 |
'magazine-article-share-popup', |
| 25 |
'article_next', |
| 26 |
'cta_nl_ext_container', |
| 27 |
'article_others_authors', # Remove link of other articles at the bottom of the article |
| 28 |
'item_note2', # Remove 'lettere' and 'numero' at the bottom of the article |
| 29 |
'article_spotlight', # Remove 'Da non perdere' at the end of the article |
| 30 |
] |
| 31 |
} |
| 32 |
), |
| 33 |
dict(name=['script', 'style']), |
| 34 |
] |
| 35 |
remove_attributes = ['class', 'id', 'name', 'style'] |
| 36 |
encoding = 'utf-8' |
| 37 |
no_stylesheets = True |
| 38 |
ignore_duplicate_articles = {'url'} |
| 39 |
|
| 40 |
current_number_url = 'https://www.internazionale.it/sommario' |
| 41 |
home_url = 'https://www.internazionale.it' |
| 42 |
cover_url = None |
| 43 |
|
| 44 |
def extract_article(self, article): |
| 45 |
url = article.find('a')['href'] |
| 46 |
if url[0] == '/': |
| 47 |
url = self.home_url + url |
| 48 |
title_parts = [] |
| 49 |
tag = article.find('div', {'class': 'abstract-article__tag'}) |
| 50 |
if tag: |
| 51 |
title_parts.append(self.tag_to_string(tag).upper()) |
| 52 |
title_parts.append(self.tag_to_string(article.find('div', {'class': 'abstract-article__title'}))) |
| 53 |
article_title = ' \u2022 '.join(title_parts) |
| 54 |
pubdate = '' |
| 55 |
description_parts = [] |
| 56 |
author = article.find('div', {'class': 'abstract-article__author'}) |
| 57 |
if author: |
| 58 |
description_parts.append(self.tag_to_string(author)) |
| 59 |
summary = article.find('div', {'class': 'abstract-article__content'}) |
| 60 |
if summary: |
| 61 |
description_parts.append(self.tag_to_string(summary)) |
| 62 |
description = ' \u2022 '.join(description_parts) |
| 63 |
return dict(title=article_title, url=url, date=pubdate, description=description, content='') |
| 64 |
|
| 65 |
def parse_index(self): |
| 66 |
soup = self.index_to_soup(self.current_number_url) |
| 67 |
self.cover_url = soup.find('span', {'class': 'img_expand'})['data-src'] |
| 68 |
main_container = soup.find('div', {'class': 'content_data'}) |
| 69 |
children = main_container.findAll('div', recursive=False) |
| 70 |
sections = [] |
| 71 |
current_section = None |
| 72 |
for container in children: |
| 73 |
if 'abstract-testatina' in container['class'] or 'abstract-testatina-cultura' in container['class']: |
| 74 |
if current_section: |
| 75 |
sections.append(current_section) |
| 76 |
current_section = (self.tag_to_string(container), []) |
| 77 |
continue |
| 78 |
|
| 79 |
if 'masonry-items' in container['class']: |
| 80 |
for article in container.findAll('div', {'class': 'abstract-article'}): |
| 81 |
current_section[1].append(self.extract_article(article)) |
| 82 |
continue |
| 83 |
|
| 84 |
if 'abstract-article' in container['class']: |
| 85 |
current_section[1].append(self.extract_article(container)) |
| 86 |
continue |
| 87 |
|
| 88 |
# print(container['class']) |
| 89 |
if current_section: |
| 90 |
sections.append(current_section) |
| 91 |
return sections |
| 92 |
|
| 93 |
def preprocess_html(self, soup): |
| 94 |
for node in soup.findAll('figure'): |
| 95 |
img_src = None |
| 96 |
image_attributes = [ |
| 97 |
'data-media1024', |
| 98 |
'data-media1025', |
| 99 |
'data-media641', |
| 100 |
'data-media321', |
| 101 |
'data-media', |
| 102 |
] |
| 103 |
for attr in image_attributes: |
| 104 |
if node.has_attr(attr): |
| 105 |
img_src = node[attr] |
| 106 |
break |
| 107 |
node.name = 'div' |
| 108 |
if img_src: |
| 109 |
img = soup.new_tag('img', src=img_src) |
| 110 |
node.insert(0, img) |
| 111 |
for node in soup.findAll('figcaption'): |
| 112 |
node.name = 'div' |
| 113 |
# if self.browser.cookiejar: |
| 114 |
# self.browser.cookiejar.clear() |
| 115 |
return soup |
| 116 |
|
| 117 |
def get_cover_url(self): |
| 118 |
return self.cover_url |