Debian Codegraph @

recipes/clarin.recipe

raw
1 #!/usr/bin/env python
2 # -*- mode: python -*-
3 # -*- coding: utf-8 -*-
4
5 from __future__ import unicode_literals
6
7 __license__ = 'GPL v3'
8 __copyright__ = '2008-2016, Darko Miletic <darko.miletic at gmail.com>'
9 '''
10 clarin.com
11 '''
12
13 try:
14 from urllib.parse import urlencode
15 except ImportError:
16 from urllib import urlencode
17 from calibre import strftime
18 from calibre.web.feeds.news import BasicNewsRecipe
19
20
21 class Clarin(BasicNewsRecipe):
22 title = 'Clarín'
23 __author__ = 'Darko Miletic, updated by GGsalas'
24 description = 'Clarin.com. Noticias de la Argentina y el mundo. Información actualizada las 24 horas y en español. Informate ya'
25 publisher = 'Grupo Clarin'
26 category = 'news, politics, Argentina'
27 oldest_article = 1
28 max_articles_per_feed = 100
29 use_embedded_content = False
30 no_stylesheets = True
31 encoding = 'utf8'
32 delay = 1
33 language = 'es_AR'
34 publication_type = 'newspaper'
35 needs_subscription = 'optional'
36 INDEX = 'http://www.clarin.com'
37 LOGIN = 'https://app-pase.clarin.com/pase-registracion/app/pase/ingresarNavegable?execution=e1s1'
38 cover_url = strftime('http://tapas.clarin.com/tapa/%Y/%m/%d/%Y%m%d_thumb.jpg')
39
40 compress_news_images = True
41 scale_news_images_to_device = True
42 compress_news_images_max_size = 10 # kB
43
44 # To get all the data (images)
45 auto_cleanup = False
46
47 extra_css = '''
48 h1#title {
49 line-height: 1em;
50 margin: 0 0 .5em 0;
51 }
52 p.volanta {
53 font-size: .7em;
54 margin-bottom: .5em;
55 }
56 .bajada h2 {
57 font-size: 1em;
58 line-height: 1em;
59 color: #666666;
60 margin: 0 0 1em 0;
61 }
62 .figcaption {
63 font-style: italic;
64 font-size: .9em;
65 margin-bottom: .5em;
66 }
67 '''
68
69 conversion_options = {'comment': description, 'tags': category, 'publisher': publisher, 'language': language}
70
71 recipe_specific_options = {
72 'days': {
73 'short': 'Oldest article to download from this news source. In days ',
74 'long': 'For example, 0.5, gives you articles from the past 12 hours',
75 'default': str(oldest_article),
76 }
77 }
78
79 def __init__(self, *args, **kwargs):
80 BasicNewsRecipe.__init__(self, *args, **kwargs)
81 d = self.recipe_specific_options.get('days')
82 if d and isinstance(d, str):
83 self.oldest_article = float(d)
84
85 keep_only_tags = [
86 dict(name='p', attrs={'class': 'volanta'}),
87 dict(name='h1', attrs={'id': 'title'}),
88 dict(name='div', attrs={'class': 'bajada'}),
89 dict(name='div', attrs={'id': 'galeria-trigger'}),
90 dict(name='div', attrs={'class': 'body-nota'}),
91 ]
92
93 remove_tags = [
94 dict(name=['meta', 'base', 'link', 'iframe', 'embed', 'object']),
95 dict(attrs={'class': ['tags-bar', 'breadcrumb', 'share-bar', 'share', 'sp__SM']}),
96 dict(name='div', attrs={'class': lambda x: x and 'r-nota' in x.split()}),
97 dict(attrs={'id': ['relacionadas']}),
98 dict(name='a', attrs={'class': 'content-new'}),
99 ]
100
101 remove_tags_after = dict(name='div', attrs={'id': 'relacionadas'})
102
103 remove_attributes = ['lang']
104
105 # Images on highlights view
106 def populate_article_metadata(self, article, soup, first):
107 if first and hasattr(self, 'add_toc_thumbnail'):
108 picdiv = soup.find('img')
109 if picdiv is not None:
110 self.add_toc_thumbnail(article, picdiv['src'])
111
112 feeds = [
113 ('Lo Ultimo', 'http://www.clarin.com/rss/lo-ultimo/'),
114 ('Politica', 'http://www.clarin.com/rss/politica/'),
115 ('Opinion', 'https://www.clarin.com/rss/opinion/'),
116 ('Cultura', 'https://www.clarin.com/rss/cultura/'),
117 ('Economia', 'https://www.clarin.com/rss/economia/'),
118 ('Tecnologia', 'https://www.clarin.com/rss/tecnologia/'),
119 ('RevistaN', 'https://www.clarin.com/rss/revista-enie/'),
120 ('Viva', 'https://www.clarin.com/rss/viva/'),
121 ('Deportes', 'http://www.clarin.com/rss/deportes/'),
122 ('Mundo', 'http://www.clarin.com/rss/mundo/'),
123 ('Espectaculos', 'http://www.clarin.com/rss/espectaculos/'),
124 ('Sociedad', 'http://www.clarin.com/rss/sociedad/'),
125 ('Ciudades', 'http://www.clarin.com/rss/ciudades/'),
126 ('Policiales', 'http://www.clarin.com/rss/policiales/'),
127 ('Internet', 'http://www.clarin.com/rss/internet/'),
128 ]
129
130 def get_browser(self):
131 br = BasicNewsRecipe.get_browser(self)
132 br.open(self.INDEX)
133 if self.username is not None and self.password is not None:
134 data = urlencode({
135 'ingresar_ingresar_paseForm': 'ingresar_ingresar_paseForm',
136 'ingresar_ingresar_email_paseInputComponent': self.username,
137 'ingresar_ingresar_palabraClave_paseInputComponent': self.password,
138 'ingresar_ingresar_ingresar_paseButton': 'Ingresar',
139 'javax.faces.ViewState': 'e1s1',
140 })
141 br.open(self.LOGIN, data)
142 return br
143
144 def preprocess_html(self, soup):
145 for img in soup.findAll(['img'], attrs={'data-big': True}):
146 img['src'] = img['data-big']
147
148 for figCaption in soup.findAll(['figcaption']):
149 figCaption.name = 'div'
150 figCaption['class'] = 'figcaption'
151
152 return soup
153
154 def postprocess_html(self, soup, first_fetch):
155 for t in soup.findAll(['a']):
156 t.name = 'strong'
157
158 nonBreakSpace = '\xa0'
159 for empty in soup.findAll('p', string=nonBreakSpace):
160 empty.extract()
161
162 return soup