Debian Codegraph @

recipes/internazionale.recipe

raw
1 #!/usr/bin/env python
2 from calibre.web.feeds.recipes import BasicNewsRecipe
3
4
5 class Volkskrant(BasicNewsRecipe):
6 title = 'Internazionale'
7 __author__ = 'Cristi Ghera'
8 max_articles_per_feed = 100
9 description = 'Internazionale - Notizie dall’Italia e dal mondo'
10 needs_subscription = False
11 language = 'it'
12 country = 'IT'
13 category = 'news, politics, Italy, world'
14 resolve_internal_links = True
15 remove_tags_before = {'name': 'article'}
16 remove_tags_after = {'name': 'article'}
17 remove_tags = [
18 dict(
19 attrs={
20 'class': [
21 'item-banner',
22 'hentryfeed__side',
23 'magazine-article-share-tools',
24 'magazine-article-share-popup',
25 'article_next',
26 'cta_nl_ext_container',
27 'article_others_authors', # Remove link of other articles at the bottom of the article
28 'item_note2', # Remove 'lettere' and 'numero' at the bottom of the article
29 'article_spotlight', # Remove 'Da non perdere' at the end of the article
30 ]
31 }
32 ),
33 dict(name=['script', 'style']),
34 ]
35 remove_attributes = ['class', 'id', 'name', 'style']
36 encoding = 'utf-8'
37 no_stylesheets = True
38 ignore_duplicate_articles = {'url'}
39
40 current_number_url = 'https://www.internazionale.it/sommario'
41 home_url = 'https://www.internazionale.it'
42 cover_url = None
43
44 def extract_article(self, article):
45 url = article.find('a')['href']
46 if url[0] == '/':
47 url = self.home_url + url
48 title_parts = []
49 tag = article.find('div', {'class': 'abstract-article__tag'})
50 if tag:
51 title_parts.append(self.tag_to_string(tag).upper())
52 title_parts.append(self.tag_to_string(article.find('div', {'class': 'abstract-article__title'})))
53 article_title = ' \u2022 '.join(title_parts)
54 pubdate = ''
55 description_parts = []
56 author = article.find('div', {'class': 'abstract-article__author'})
57 if author:
58 description_parts.append(self.tag_to_string(author))
59 summary = article.find('div', {'class': 'abstract-article__content'})
60 if summary:
61 description_parts.append(self.tag_to_string(summary))
62 description = ' \u2022 '.join(description_parts)
63 return dict(title=article_title, url=url, date=pubdate, description=description, content='')
64
65 def parse_index(self):
66 soup = self.index_to_soup(self.current_number_url)
67 self.cover_url = soup.find('span', {'class': 'img_expand'})['data-src']
68 main_container = soup.find('div', {'class': 'content_data'})
69 children = main_container.findAll('div', recursive=False)
70 sections = []
71 current_section = None
72 for container in children:
73 if 'abstract-testatina' in container['class'] or 'abstract-testatina-cultura' in container['class']:
74 if current_section:
75 sections.append(current_section)
76 current_section = (self.tag_to_string(container), [])
77 continue
78
79 if 'masonry-items' in container['class']:
80 for article in container.findAll('div', {'class': 'abstract-article'}):
81 current_section[1].append(self.extract_article(article))
82 continue
83
84 if 'abstract-article' in container['class']:
85 current_section[1].append(self.extract_article(container))
86 continue
87
88 # print(container['class'])
89 if current_section:
90 sections.append(current_section)
91 return sections
92
93 def preprocess_html(self, soup):
94 for node in soup.findAll('figure'):
95 img_src = None
96 image_attributes = [
97 'data-media1024',
98 'data-media1025',
99 'data-media641',
100 'data-media321',
101 'data-media',
102 ]
103 for attr in image_attributes:
104 if node.has_attr(attr):
105 img_src = node[attr]
106 break
107 node.name = 'div'
108 if img_src:
109 img = soup.new_tag('img', src=img_src)
110 node.insert(0, img)
111 for node in soup.findAll('figcaption'):
112 node.name = 'div'
113 # if self.browser.cookiejar:
114 # self.browser.cookiejar.clear()
115 return soup
116
117 def get_cover_url(self):
118 return self.cover_url