Debian Codegraph @

recipes/le_monde_sub_paper.recipe

raw
1 #!/usr/bin/env python
2
3 __license__ = 'GPL v3'
4 __copyright__ = '2012, 2013, Rémi Vanicat <vanicat at debian.org>'
5 '''
6 Lemonde.fr: Version abonnée
7 '''
8
9 import os
10 import re
11 import time
12 import zipfile
13
14 from calibre.constants import preferred_encoding
15 from calibre.ptempfile import PersistentTemporaryFile
16 from calibre.web.feeds.news import BasicNewsRecipe
17
18
19 class LeMondeAbonne(BasicNewsRecipe):
20 title = 'Le Monde: Édition abonnés papier'
21 __author__ = 'Rémi Vanicat'
22 description = 'Actualités'
23 category = 'Actualités, France, Monde'
24 publisher = 'Le Monde'
25 language = 'fr'
26 needs_subscription = True
27 no_stylesheets = True
28 smarten_punctuation = True
29 remove_attributes = ['border', 'cellspacing', 'display', 'align', 'cellpadding', 'colspan', 'valign', 'vscape', 'hspace', 'alt', 'width', 'height']
30 extra_css = ''' li{margin:6pt 0}
31 ul{margin:0}
32
33 div.photo img{max-width:100%; border:0px transparent solid;}
34 div.photo{font-family:inherit; color:#333; text-align:center;}
35 div.photo p{text-align:justify;font-size:.9em; line-height:.9em;}
36
37 @page{margin:10pt}
38 .ar-txt {color:#000; text-align:justify;}
39 h1{text-align:left; font-size:1.25em;}
40
41 .auteur{text-align:right; font-weight:bold}
42 .feed{text-align:right; font-weight:bold}
43 .po-ti2{font-weight:bold}
44 .fen-tt{font-weight:bold;font-size:1.1em}
45 '''
46
47 zipurl_format = 'http://medias.lemonde.fr/abonnes/editionelectronique/%Y%m%d/html/%y%m%d.zip'
48 coverurl_format = '/img/%y%m%d01.jpg'
49 masthead_url = 'http://upload.wikimedia.org/wikipedia/commons/thumb/5/54/Le_monde_logo.svg/800px-Le_monde_logo.svg.png'
50 path_format = '%y%m%d'
51
52 keep_only_tags = [
53 dict(name=['h1']),
54 dict(name='div', attrs={'class': 'photo'}),
55 dict(name='div', attrs={'class': 'po-ti2'}),
56 dict(name='div', attrs={'class': 'ar-txt'}),
57 dict(name='div', attrs={'class': 'po_rtcol'}),
58 ]
59
60 remove_tags = [dict(name='div', attrs={'class': 'po-ti'}), dict(name='div', attrs={'class': 'po-copy'})]
61
62 article_id_pattern = re.compile(r'[0-9]+\.html')
63 article_url_format = 'http://www.lemonde.fr/journalelectronique/donnees/protege/%Y%m%d/html/'
64
65 def get_browser(self):
66 br = BasicNewsRecipe.get_browser(self)
67 br.open('https://secure.lemonde.fr/sfuser/connexion')
68 br.select_form(name='connection')
69 br['connection[mail]'] = self.username
70 br['connection[password]'] = self.password
71 br.submit()
72 return br
73
74 decalage = 24 * 60 * 60 # today Monde has tomorow date
75
76 def get_cover_url(self):
77 url = time.strftime(self.coverurl_format, self.ltime)
78 return self.articles_path + url
79
80 def parse_index(self):
81 browser = self.get_browser()
82
83 second = time.time()
84 second += self.decalage
85
86 for i in range(7):
87 self.ltime = time.gmtime(second)
88 self.timefmt = time.strftime(' %A %d %B %Y', self.ltime).decode(preferred_encoding)
89 url = time.strftime(self.zipurl_format, self.ltime)
90 try:
91 response = browser.open(url)
92 continue
93 except Exception:
94 second -= 24 * 60 * 60
95
96 tmp = PersistentTemporaryFile(suffix='.zip')
97 self.report_progress(0.1, _('downloading zip file'))
98 tmp.write(response.read())
99 tmp.close()
100
101 zfile = zipfile.ZipFile(tmp.name, 'r')
102 self.report_progress(0.1, _('extracting zip file'))
103
104 zfile.extractall(self.output_dir)
105 zfile.close()
106
107 path = os.path.join(self.output_dir, time.strftime(self.path_format, self.ltime), 'data')
108
109 self.articles_path = path
110
111 files = os.listdir(path)
112
113 nb_index_files = len([name for name in files if re.match(r'frame_gauche_[0-9]+.html', name)])
114
115 flux = []
116
117 article_url = time.strftime(self.article_url_format, self.ltime)
118
119 for i in range(nb_index_files):
120 filename = os.path.join(path, 'selection_%d.html' % (i + 1))
121 with open(filename, 'rb') as tmp:
122 soup = self.index_to_soup(tmp.read())
123 title = soup.find('span').contents[0]
124 if title == 'Une':
125 title = 'À la une'
126 if title == 'Evenement':
127 title = "L'événement"
128 if title == 'Planete':
129 title = 'Planète'
130 if title == 'Economie - Entreprises':
131 title = 'Économie'
132 if title == "L'Oeil du Monde":
133 title = "L'œil du Monde"
134 if title == 'Enquete':
135 title = 'Enquête'
136 if title == 'Editorial - Analyses':
137 title = 'Horizons'
138 if title == 'Le Monde Economie':
139 title = 'Économie'
140 if title == 'Lettre et chronique':
141 title = 'Idées'
142 if title == 'Le Monde Géo et politique':
143 title = 'Géopolitique'
144 if title == 'Météo - Jeux - Ecrans':
145 title = 'Économie & Entreprise'
146 tmp.close()
147
148 filename = os.path.join(path, 'frame_gauche_%d.html' % (i + 1))
149 with open(filename, 'rb') as tmp:
150 soup = self.index_to_soup(tmp.read())
151 articles = []
152 for link in soup.findAll('a'):
153 article_file = link['href']
154 article_id = self.article_id_pattern.search(article_file).group()
155 article = {'title': link.contents[0], 'url': article_url + article_id, 'description': '', 'content': ''}
156 articles.append(article)
157 tmp.close()
158
159 flux.append((title, articles))
160
161 return flux
162
163
164 # Local Variables:
165 # mode: python
166 # End: