Debian Codegraph @

recipes/private_eye.recipe

raw
1 """
2 Fetch Private Eye (Online Edition)
3 """
4
5 import re
6 from datetime import datetime, timedelta
7
8 from calibre.web.feeds.news import BasicNewsRecipe
9
10
11 class PrivateEyeRecipe(BasicNewsRecipe):
12 ##
13 # Last Edited: 2026-01-11 by Sophist-UK
14 #
15 # Remark: Version 3.4 2026-01-13 by Sophist-UK
16 # Tweak #block-sections spacing
17 # Version 3.3 2026-01-11 by Sophist-UK
18 # Fix recipe after web-site changes
19 # Get next publication date from home page
20 # Get articles URL list from sub-menu on News page
21 # Add classified ads from https://www.eyeads.co.uk/
22 # Version 3.2 2025-04-02 by Sophist-UK
23 # Fix recipe after web-site changes
24 # Version 3.1 2023-07-14 by Sophist-UK
25 # Show crossword on right so clues are continuous down left
26 # Link to crossword image removed
27 # Improve many image layouts
28 # Version 3.0 2023-07-01 by Sophist-UK
29 # Rewrite (by Sophist-UK) to fit latest web pages,
30 # correctly identify pages to include
31 # and improve formatting.
32 # Added: inclusion of About page,
33 # identifying series number and publication date and setting metadata.
34
35 title = 'Private Eye (Online Edition)'
36 description = 'Private Eye is a fortnightly British satirical and current affairs magazine, edited by Ian Hislop'
37 publication_type = 'magazine'
38 language = 'en_GB'
39 encoding = 'utf-8'
40 oldest_article = 13
41 max_articles_per_feed = 100
42 remove_javascript = True
43 ignore_duplicate_articles = {'url'}
44
45 __author__ = 'Martyn Pritchard & Sophist-UK'
46 __copyright__ = '© 2020-2026, Martyn Pritchard <[email protected]> & Sophist-UK <[email protected]>'
47
48 base_url = 'https://www.private-eye.co.uk/'
49 current_issue = 'https://www.private-eye.co.uk/news'
50 about_page = 'https://www.private-eye.co.uk/about'
51 number_crunching = 'https://www.private-eye.co.uk/number-crunching'
52 masthead_url = 'https://www.private-eye.co.uk/grfx/logos/logo-new.png'
53 classified_ads = 'https://www.eyeads.co.uk'
54 next_issue_date = 'Next issue on sale:'
55
56 author = 'Private Eye'
57 series = title = 'Private Eye Online'
58 conversion_options = {
59 'authors': author,
60 'author_sort': author,
61 'series': series,
62 'series_index': 0,
63 'title': title,
64 'title_sort': title,
65 }
66
67 index_attrs_to_include = [
68 {'id': 'sub-nav-box'},
69 {'class': 'sub-nav-bar'},
70 ]
71
72 titles_to_skip = [
73 'Home',
74 'more',
75 'In This Issue',
76 ]
77
78 url_to_section_name = {
79 'news': 'News',
80 'street-of-shame': 'Street of Shame',
81 'hp-sauce': 'HP Sauce',
82 'cartoons': 'Strips and Cartoons',
83 'pms-whatsapp': "PM's Whatsapp",
84 'mediaballs': 'Commentatorballs',
85 'lookalikes': 'Lookalike',
86 'crossword': 'Eye Crossword',
87 'in-the-back': 'In the Back',
88 'media-news': 'Media News',
89 'columnists': 'Columnists',
90 'rotten-boroughs': 'Rotten Boroughs',
91 'number-crunching': 'Number Crunching',
92 'www.eyeads.co.uk': 'Private Eye Classified Ads',
93 }
94
95 re_extract_date = re.compile(r'\d.*\d')
96 articles = []
97 urls = []
98
99 def add_article(self, title, url):
100 if url in self.urls:
101 return
102 known_url = url.rsplit('/', 1)[-1]
103 if known_url and known_url in self.url_to_section_name:
104 title = self.url_to_section_name[known_url]
105 if not title:
106 return
107 self.articles.append({
108 'title': title,
109 'url': url,
110 })
111 self.urls.append(url)
112 self.log('Adding:', title, '(', url, ')')
113
114 def get_cover_url(self):
115 soup = self.index_to_soup(self.current_issue)
116
117 for img in soup.findAll('img', {'class': 'issue-cover'}):
118 src = img['src']
119 if src.endswith('_big.jpg'):
120 file_name = src.rsplit('/', 1)[1]
121 if file_name is None:
122 file_name = src
123 try:
124 self.conversion_options.update({'series_index': int(file_name[: -len('_big.jpg')])})
125 self.log('series-index:', self.conversion_options['series_index'])
126 except TypeError, ValueError:
127 # wrong big image
128 continue
129 return src
130 return None
131
132 def get_next_issue_date(self, soup):
133 sidebar = soup.find('div', id='sections-sidebar')
134 if sidebar is None:
135 return None
136 for s in sidebar.stripped_strings:
137 match = self.re_extract_date.match(s)
138 if match:
139 return match.group(0)
140 return None
141
142 def parse_index(self):
143 self.log('')
144 self.log('Title:', self.title)
145 self.log('Description:', self.description)
146 self.log('Authors:', self.__author__)
147 self.log('Copyright:', self.__copyright__)
148 self.log('')
149 self.log('Support:', 'If this recipe stops working email [email protected] to let them know.')
150 self.log('')
151
152 soup = self.index_to_soup(self.current_issue)
153
154 try:
155 # Get publication date - Next issue on sale date - 13 days
156 next_issue_date = self.get_next_issue_date(soup)
157 self.log('next_issue_date:', next_issue_date)
158
159 day, month, year = next_issue_date.split(' ')
160 # remove day suffixes e.g. 2nd
161 day = ''.join(c for c in day if c.isdigit())
162 pub_date = datetime.strptime(' '.join((day, month, year)), '%d %B %Y') - timedelta(13)
163 self.log('pub-date:', pub_date)
164
165 self.conversion_options.update({'pubdate': datetime.strftime(pub_date, '%d %B %Y').lstrip('0')})
166 title = self.title + ' ' + datetime.strftime(pub_date, '%Y-%m-%d')
167 self.conversion_options.update({'title': title})
168 self.conversion_options.update({'title_sort': title})
169 except TypeError, ValueError:
170 # Bad date
171 self.log('Cannot parse next issue date from:', next_issue_date)
172
173 # Get pages from the various contents panels.
174 # Duplicates will be eliminated automatically.
175
176 # Since we are now using the first online page to generate the list of pages to include
177 # we need to include this one:
178
179 self.add_article('News', self.current_issue)
180
181 for section_attrs in self.index_attrs_to_include:
182 section = soup.find('div', attrs=section_attrs)
183
184 if not section:
185 continue
186
187 for a in section.findAll('a', href=True):
188 url = a.get('href')
189 title = a.getText().rstrip(' »\n')
190 if title in self.titles_to_skip:
191 continue
192 if not url.startswith('http'):
193 url = self.base_url + url
194 self.add_article(title, url)
195
196 if not self.articles:
197 raise ValueError('Private-Eye Online index of pages not found')
198
199 # Add the pages with no menu
200 self.add_article('Number crunching', self.number_crunching)
201
202 # Add the About page as a final article
203 self.add_article('About Private Eye', self.about_page)
204
205 # Add classified adverts - as a fun historical record
206 self.add_article('Classified Ads', self.classified_ads)
207
208 self.log('parse_index:', self.articles)
209
210 return [('Private Eye', self.articles)]
211
212 def preprocess_html(self, soup):
213 # Remove <a> tag link to crossword image
214 for tag in soup.findAll('a', {'href': re.compile(r'/pictures/crossword/')}):
215 self.log('Removing link to crossword image...')
216 tag.unwrap()
217
218 # Remove align tag in crossword image (so float right works)
219 for tag in soup.findAll('img', {'src': re.compile(r'/pictures/crossword/')}):
220 if 'align' in tag.attrs:
221 self.log('Removing crossword image align attribute...')
222 del tag.attrs['align']
223
224 return soup
225
226 # We remove vast swathes of HTML which is not part of the articles.
227 # Remove sibling content
228 remove_tags_before = [
229 {'name': 'div', 'class': 'article'},
230 {'name': 'div', 'id': 'page'},
231 {'name': 'div', 'id': 'page-wide'},
232 {'name': 'div', 'id': 'content'},
233 ]
234 remove_tags_after = remove_tags_before.copy()
235 remove_tags_after.extend([
236 {'name': 'div', 'id': 'about-covers'},
237 {'name': 'script', 'id': 'about-covers'},
238 {'name': 'a', 'attrs': {'href': 'https://shop.private-eye.co.uk'}},
239 # {'name': 'img', 'attrs': {'src': re.compile(r'/grfx/logos/')}},
240 ])
241 # Remove non-sibling content
242 remove_tags = [
243 # Top
244 {'name': 'div', 'attrs': {'id': 'header-wide'}},
245 {'name': 'div', 'attrs': {'id': 'top-bar-sticky'}},
246 {'name': 'div', 'attrs': {'id': 'nav-box-mobile'}},
247 {'name': 'div', 'attrs': {'id': 'nav-box-sections-mobile'}},
248 {'name': 'div', 'attrs': {'id': 'sub-nav-box'}},
249 {'name': 'div', 'attrs': {'class': 'sub-nav-bar'}},
250 # Content
251 {'name': 'div', 'attrs': {'id': 'why-subscribe'}},
252 {'name': 'div', 'attrs': {'id': 'issue-box'}},
253 {'name': 'div', 'attrs': {'id': 'sections-sidebar'}},
254 # Spacers
255 {'name': 'div', 'attrs': {'class': 'gap'}},
256 # {'name': 'div', 'attrs': {'class': 'gap-small'}},
257 {'name': 'div', 'attrs': {'class': 'gap-bigger'}},
258 {'name': 'div', 'attrs': {'class': 'gap-biggest'}},
259 # Bottom
260 {'name': 'div', 'attrs': {'id': 'home-content'}},
261 {'name': 'div', 'attrs': {'id': 'home-shop-content-wide'}},
262 {'name': 'div', 'attrs': {'id': 'page-94-strip-wide'}},
263 {'name': 'div', 'attrs': {'id': 'footer-wide'}},
264 {'name': 'div', 'attrs': {'id': 'about-covers'}},
265 # {'name': 'div', 'attrs': {'id': 'follow-buttons'}},
266 # {'name': 'div', 'attrs': {'id': 'sidebar'}},
267 {'name': 'a', ' attrs': {'href': 'https://www.private-eye.co.uk/shop'}},
268 {'name': 'a', ' attrs': {'href': 'https://shop.private-eye.co.uk'}},
269 {'name': 'a', ' attrs': {'href': 'https://shop.private-eye.co.uk/'}},
270 {'name': 'link', 'attrs': {'href': re.compile(r'/javastyle/lightbox/')}},
271 {'name': 'link', 'attrs': {'href': re.compile(r'/javastyle/news_ticker/')}},
272 {'name': 'link', 'attrs': {'href': re.compile(r'/javastyle/media-queries-')}},
273 {'name': 'img', 'attrs': {'src': re.compile(r'/grfx/logos/gnitty.gif')}},
274 {'name': 'img', 'attrs': {'src': re.compile(r'/grfx/stuff/subscribe_here_gnitty.gif')}},
275 {'name': 'iframe'},
276 {'name': 'script'},
277 {'name': 'style'},
278 # Classified site
279 {'name': 'a', 'attrs': {'class': 'skip-link'}},
280 {'name': 'header', 'attrs': {'id': 'masthead'}},
281 {'name': 'div', 'attrs': {'class': 'placeAdIntro'}},
282 {'name': 'img', 'attrs': {'class': 'cat-image'}},
283 {'name': 'img', 'attrs': {'class': 'catArrow'}},
284 {'name': 'div', 'attrs': {'class': 'ad-contact'}},
285 {'name': 'div', 'attrs': {'id': re.compile(r'cookie-law-')}},
286 {'name': 'div', 'attrs': {'class': re.compile(r'cli-modal')}},
287 {'name': 'footer', 'attrs': {'class': 'site-footer'}},
288 ]
289
290 preprocess_regexps = [
291 # Convert headers to h1, strapline to h4
292 (
293 re.compile(
294 r'<span class="headline(?:-new)?">(.*?)</span>\s*(?:<br/?>\s*)*((?:<span class="text">(.*?)</span>)|(?:<font .*?>(.*?)</font>))?',
295 re.DOTALL | re.IGNORECASE,
296 ),
297 lambda match: '<h1 class="myheadline">' + match[1] + '</h1>' + (('<h4 class="mystrapline">' + match[2] + '</h4>') if match[2] else ''),
298 ),
299 # Remove broken links for this issue
300 (re.compile(r'<a href="/issue-.*?".*?>(.*?)</a>', re.DOTALL | re.IGNORECASE), lambda match: match[1]),
301 ]
302
303 # The following extra css is to tweak the formatting of various elements of various article pages.
304 # Unfortunately, there are a variety of different pages styles, hence the extended tweak list.
305 # Some of these mimic the actual layout.css which does not seem to make it across into the calibre
306 # ebook without duplicating it as extra css.
307 # However some is new css to tweak output when part of an ebook.
308 extra_css = ' \n '.join([
309 '.article {font-family: Merriweather, Georgia, serif; font-size: 1em}',
310 '.myheadline {font-family: "Source Sans Pro", Arial, Helvetica, sans-serif; font-size: 2em;}',
311 '.mystrapline {font-family: "Source Sans Pro", Arial, Helvetica, sans-serif; font-size: 1.2em;}',
312 '#content img {float: right; width: 45%; minimum-width:350px;}',
313 '#content img.office {float: right; width: 45%; maximum-width:390px; margin-left: 15px;}',
314 '#content img.cartoon-left {float: left; margin-right: 15px; margin-bottom: 15px;}',
315 '#content img.cartoon-right {float: none; margin-bottom: 15px;}',
316 '#content img.strip {float: none; width: 100%;}',
317 '#content img:first-child {float: none;}',
318 '#content img.gnitty-right {float: none; width: 160px;}',
319 '#content #story > div[align=right] > img:first-child {float: none; width: 15px;}',
320 '#content #story > img:first-child {float: none; height: 100px; width: none; minimum-width: none;}',
321 '#content #block-sections div.divider {height: 1px; background-color: #eee; margin: 10px 0px; clear: both;}',
322 '#content #block-sections img {float: none; width: none;}',
323 '#content #block-sections img.lookalike {float: none; width: 100%;}',
324 '#content #block-sections img.photo-right {float: right; width: 25%; min-width:120px; margin-left: 15px;}',
325 '#content #block-sections > p:last-child > img:first-child {float: none; width: 120px;}',
326 '#content #block-sections > p:last-child > img:nth-child(2) {float: none; width: 120px;}',
327 '#content #block-sections img.crossword {float: right; width: 40%; margin-left: 15px; min-width: 350px;}',
328 '#content #article-caption-box {float: right; background: #222222; display: block; width: 40%; min-width: 250px; font-size: 90%; margin-left: 15px;}',
329 '#content #article-caption-box img {float: none; width: 100%; max-width: none;}',
330 '#caption-box {color: #ffffff; text-align: center; padding: 5px 20px 15px 20px;}',
331 '#whatsapp {border-left: 5px #f45e61 solid; border-right: 5px #f45e61 solid; border-bottom: 5px #f45e61 solid; padding: 10px 20px 20px 10px;}',
332 '#whatsapp::after {clear:both;}',
333 '#whatsapp .whatsapp-left, .whatsapp-right {margin: 20px 0; padding: 15px; border-radius: 10px;}',
334 '#whatsapp .whatsapp-left, .whatsapp-right {font-family: Helvetica, Arial, "sans-serif"; font-weight: 300; font-size: 18px; line-height: 24px;}',
335 '#whatsapp .whatsapp-left {margin-right: 30%; background-color: #eeeeee;}',
336 '#whatsapp .whatsapp-right {margin-left: 30%; background-color: #dce5ae;}',
337 '#whatsapp .whatsapp-left img, #whatsapp .whatsapp-right img {width: 35px; margin: 0 10px; vertical-align: middle;}',
338 '#whatsapp .whatsapp-left img.emoji, #whatsapp .whatsapp-right img.emoji {max-width: 35px; margin: 0 5px; vertical-align: middle;}',
339 '.container .row .catTitle {color: fff; background-color: #12174a;}',
340 ])