Debian Codegraph @

recipes/le_monde_en.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3
4 __license__ = 'GPL v3'
5 __copyright__ = '2022'
6
7 '''
8 lemonde.fr/en
9 '''
10
11 import re
12
13 from calibre.web.feeds.news import BasicNewsRecipe
14
15
16 def classes(classes):
17 q = frozenset(classes.split(' '))
18 return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)})
19
20
21 class LeMonde(BasicNewsRecipe):
22 title = 'Le Monde in English'
23 __author__ = 'Darko Miletic (based on veezh recipe)'
24 description = 'Le Monde in English'
25 publisher = 'Société Editrice du Monde'
26 publication_type = 'newspaper'
27 needs_subscription = 'optional'
28 language = 'en'
29
30 oldest_article = 2
31 max_articles_per_feed = 15
32 no_stylesheets = True
33 remove_empty_feeds = True
34 ignore_duplicate_articles = {'title', 'url'}
35
36 conversion_options = {'publisher': publisher}
37
38 masthead_url = 'http://upload.wikimedia.org/wikipedia/commons/thumb/5/54/Le_monde_logo.svg/800px-Le_monde_logo.svg.png'
39
40 feeds = [
41 ('Frontpage', 'https://www.lemonde.fr/en/rss/une.xml'),
42 ('International', 'https://www.lemonde.fr/en/international/rss_full.xml'),
43 ('Politics', 'https://www.lemonde.fr/en/politics/rss_full.xml'),
44 ('France', 'https://www.lemonde.fr/en/france/rss_full.xml'),
45 ('Economy', 'https://www.lemonde.fr/en/economy/rss_full.xml'),
46 ('Environment', 'https://www.lemonde.fr/en/environment/rss_full.xml'),
47 ('Science', 'https://www.lemonde.fr/en/science/rss_full.xml'),
48 ('Pixels', 'https://www.lemonde.fr/en/pixels/rss_full.xml'),
49 ('Culture', 'https://www.lemonde.fr/en/culture/rss_full.xml'),
50 ('Opinion', 'https://www.lemonde.fr/en/opinion/rss_full.xml'),
51 ('Our Times', 'https://www.lemonde.fr/en/our-times/rss_full.xml'),
52 ]
53
54 keep_only_tags = [classes('article__header'), dict(name='section', attrs={'class': ['article__content', 'article__heading', 'article__wrapper']})]
55
56 remove_tags = [
57 classes('article__status meta__date meta__reading-time meta__social multimedia-embed'),
58 dict(name=['footer', 'link']),
59 dict(name='img', attrs={'class': ['article__author-picture']}),
60 dict(name='section', attrs={'class': ['article__reactions', 'author', 'catcher', 'portfolio', 'services-inread']}),
61 ]
62
63 remove_attributes = ['data-sizes', 'height', 'sizes', 'width']
64
65 preprocess_regexps = [
66 # insert space between author name and description
67 (re.compile(r'(<span class="[^"]*author__desc[^>]*>)([^<]*</span>)', re.IGNORECASE), lambda match: match.group(1) + ' ' + match.group(2)),
68 # insert " | " between article type and description
69 (re.compile(r'(<span class="[^"]*article__kicker[^>]*>[^<]*)(</span>)', re.IGNORECASE), lambda match: match.group(1) + ' | ' + match.group(2)),
70 ]
71
72 extra_css = '''
73 h2 { font-size: 1em; }
74 h3 { font-size: 1em; }
75 .article__desc { font-weight: bold; }
76 .article__fact { font-weight: bold; text-transform: uppercase; }
77 .article__kicker { text-transform: uppercase; }
78 .article__legend { font-size: 0.6em; margin-bottom: 1em; }
79 .article__title { margin-top: 0em; }
80 '''
81
82 def get_browser(self):
83 br = BasicNewsRecipe.get_browser(self)
84 if self.username is not None and self.password is not None:
85 br.open('https://secure.lemonde.fr/sfuser/connexion')
86 br.select_form(nr=0)
87 br['email'] = self.username
88 br['password'] = self.password
89 br.submit()
90 return br
91
92 def get_article_url(self, article):
93 url = BasicNewsRecipe.get_article_url(self, article)
94 # skip articles without relevant content (e.g., videos)
95 for el in 'blog chat live podcasts portfolio video visuel'.split():
96 if '/' + el + '/' in url:
97 self.log(url)
98 self.abort_article()
99 return url
100
101 def preprocess_html(self, soup):
102 # when an image is available in multiple sizes, select the smallest one
103 for img in soup.find_all('img', {'data-srcset': True}):
104 data_srcset = img['data-srcset'].split()
105 if len(data_srcset) > 1:
106 img['src'] = data_srcset[-2]
107 del img['data-srcset']
108 return soup
109
110 def postprocess_html(self, soup, first_fetch):
111 # remove local hyperlinks
112 for a in soup.find_all('a', {'href': True}):
113 if '.lemonde.fr/' in a['href']:
114 a.replace_with(self.tag_to_string(a))
115 # clean up header
116 for ul in soup.find_all('ul', {'class': 'breadcrumb'}):
117 div = soup.new_tag('div')
118 category = ''
119 for li in ul.find_all('li', {'class': True}):
120 category += self.tag_to_string(li).strip().upper() + ' - '
121 div.string = category[:-3]
122 ul.replace_with(div)
123 return soup