Debian Codegraph @

recipes/le_monde_sub.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
3 from __future__ import absolute_import, division, print_function, unicode_literals
4
5 __author__ = 'S. Durand <[email protected]>'
6 __license__ = 'GPL v3'
7
8 '''
9 lemonde.fr
10 '''
11
12 import re
13 from datetime import date
14
15 from calibre.web.feeds.news import BasicNewsRecipe, classes
16
17
18 class LeMondeNumerique(BasicNewsRecipe):
19 title = 'Le Monde: Édition abonnés'
20 __author__ = 'Sylvain Durand'
21 description = 'La version numérique du quotidien Le Monde'
22 publisher = 'Société Editrice du Monde'
23 publication_type = 'newspaper'
24 needs_subscription = True
25 language = 'fr'
26
27 no_stylesheets = True
28 ignore_duplicate_articles = {'title', 'url'}
29
30 conversion_options = {'publisher': publisher}
31
32 masthead_url = 'http://upload.wikimedia.org/wikipedia/commons/thumb/5/54/Le_monde_logo.svg/800px-Le_monde_logo.svg.png'
33
34 lm_sections = [
35 'international:International',
36 'politique:Politique',
37 'societe:Société',
38 'economie:Éco',
39 'culture:Culture',
40 'idees:Idées',
41 'planete:Planète',
42 'sport:Sport',
43 'sciences:Sciences',
44 'pixels:Pixels',
45 'campus:Campus',
46 ]
47
48 keep_only_tags = [classes('article__header'), dict(name='section', attrs={'class': ['article__content', 'article__heading', 'article__wrapper']})]
49
50 remove_tags = [
51 classes('article__status meta__date meta__reading-time meta__social multimedia-embed'),
52 dict(name=['footer', 'link']),
53 dict(name='img', attrs={'class': ['article__author-picture']}),
54 dict(name='section', attrs={'class': ['article__reactions', 'author', 'catcher', 'portfolio', 'services-inread']}),
55 ]
56
57 remove_attributes = ['data-sizes', 'height', 'sizes', 'width']
58
59 preprocess_regexps = [
60 # insert space between author name and description
61 (re.compile(r'(<span class="[^"]*author__desc[^>]*>)([^<]*</span>)', re.IGNORECASE), lambda match: match.group(1) + ' ' + match.group(2)),
62 # insert " | " between article type and description
63 (re.compile(r'(<span class="[^"]*article__kicker[^>]*>[^<]*)(</span>)', re.IGNORECASE), lambda match: match.group(1) + ' | ' + match.group(2)),
64 ]
65
66 extra_css = '''
67 h2 { font-size: 1em; }
68 h3 { font-size: 1em; }
69 .article__desc { font-weight: bold; }
70 .article__fact { font-weight: bold; text-transform: uppercase; }
71 .article__kicker { text-transform: uppercase; }
72 .article__legend { font-size: 0.6em; margin-bottom: 1em; }
73 .article__title { margin-top: 0em; }
74 '''
75
76 def get_browser(self):
77 br = BasicNewsRecipe.get_browser(self)
78 if self.username is not None and self.password is not None:
79 try:
80 br.open('https://secure.lemonde.fr/sfuser/connexion')
81 br.select_form(nr=0)
82 br['email'] = self.username
83 br['password'] = self.password
84 br.submit()
85 except Exception as e:
86 self.log('Login failed with error:', str(e))
87 return br
88
89 def get_cover_url(self):
90 # today's date is a reasonable guess for the ID of the cover
91 cover_id = date.today().strftime('%Y%m%d')
92 soup = self.index_to_soup('https://www.lemonde.fr/')
93 a = soup.find('a', {'id': 'jelec_link', 'style': True})
94 if a and a['style']:
95 url = a['style'].split('/')
96 if len(url) > 5 and url[3].isdigit():
97 # overwrite guess if actual cover ID was found
98 cover_id = url[3]
99 return 'https://www.lemonde.fr/thumbnail/journal/' + cover_id + '/1000/1490'
100
101 def parse_index(self):
102 ans = []
103 for x in self.lm_sections:
104 s, section_title = x.partition(':')[::2]
105 self.log('Processing section', section_title, '...')
106 articles = list(self.parse_section('https://www.lemonde.fr/%s/' % s))
107 if articles:
108 ans.append((section_title, articles))
109 return ans
110
111 def parse_section(self, url):
112 soup = self.index_to_soup(url)
113 for article in soup.find_all('section', {'class': 'teaser'}):
114 # extract URL
115 a = article.find('a', {'class': 'teaser__link', 'href': True})
116 if a is None:
117 continue
118 url = a['href']
119 # skip articles without relevant content (e.g., videos)
120 for el in 'blog chat live newsletters podcasts portfolio video visuel'.split():
121 if '/' + el + '/' in url:
122 url = None
123 break
124 if url is None:
125 continue
126 # extract title
127 h3 = article.find('h3', {'class': 'teaser__title'})
128 if h3 is None:
129 continue
130 title = self.tag_to_string(h3)
131 # extract description
132 desc = ''
133 p = article.find('p', {'class': 'teaser__desc'})
134 if p is not None:
135 desc = self.tag_to_string(p)
136 self.log('\tFound article', title, 'at', url)
137 yield {'title': title, 'url': url, 'description': desc}
138
139 def preprocess_html(self, soup):
140 # when an image is available in multiple sizes, select the smallest one
141 for img in soup.find_all('img', {'data-srcset': True}):
142 data_srcset = img['data-srcset'].split()
143 if len(data_srcset) > 1:
144 img['src'] = data_srcset[-2]
145 del img['data-srcset']
146 return soup
147
148 def postprocess_html(self, soup, first_fetch):
149 # remove local hyperlinks
150 for a in soup.find_all('a', {'href': True}):
151 if '.lemonde.fr/' in a['href']:
152 a.replace_with(self.tag_to_string(a))
153 # clean up header
154 for ul in soup.find_all('ul', {'class': 'breadcrumb'}):
155 div = soup.new_tag('div')
156 category = ''
157 for li in ul.find_all('li', {'class': True}):
158 category += self.tag_to_string(li).strip().upper() + ' - '
159 div.string = category[:-3]
160 ul.replace_with(div)
161 return soup