| 1 |
""" |
| 2 |
Fetch Private Eye (Online Edition) |
| 3 |
""" |
| 4 |
|
| 5 |
import re |
| 6 |
from datetime import datetime, timedelta |
| 7 |
|
| 8 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 9 |
|
| 10 |
|
| 11 |
class PrivateEyeRecipe(BasicNewsRecipe): |
| 12 |
## |
| 13 |
# Last Edited: 2026-01-11 by Sophist-UK |
| 14 |
# |
| 15 |
# Remark: Version 3.4 2026-01-13 by Sophist-UK |
| 16 |
# Tweak #block-sections spacing |
| 17 |
# Version 3.3 2026-01-11 by Sophist-UK |
| 18 |
# Fix recipe after web-site changes |
| 19 |
# Get next publication date from home page |
| 20 |
# Get articles URL list from sub-menu on News page |
| 21 |
# Add classified ads from https://www.eyeads.co.uk/ |
| 22 |
# Version 3.2 2025-04-02 by Sophist-UK |
| 23 |
# Fix recipe after web-site changes |
| 24 |
# Version 3.1 2023-07-14 by Sophist-UK |
| 25 |
# Show crossword on right so clues are continuous down left |
| 26 |
# Link to crossword image removed |
| 27 |
# Improve many image layouts |
| 28 |
# Version 3.0 2023-07-01 by Sophist-UK |
| 29 |
# Rewrite (by Sophist-UK) to fit latest web pages, |
| 30 |
# correctly identify pages to include |
| 31 |
# and improve formatting. |
| 32 |
# Added: inclusion of About page, |
| 33 |
# identifying series number and publication date and setting metadata. |
| 34 |
|
| 35 |
title = 'Private Eye (Online Edition)' |
| 36 |
description = 'Private Eye is a fortnightly British satirical and current affairs magazine, edited by Ian Hislop' |
| 37 |
publication_type = 'magazine' |
| 38 |
language = 'en_GB' |
| 39 |
encoding = 'utf-8' |
| 40 |
oldest_article = 13 |
| 41 |
max_articles_per_feed = 100 |
| 42 |
remove_javascript = True |
| 43 |
ignore_duplicate_articles = {'url'} |
| 44 |
|
| 45 |
__author__ = 'Martyn Pritchard & Sophist-UK' |
| 46 |
__copyright__ = '© 2020-2026, Martyn Pritchard <[email protected]> & Sophist-UK <[email protected]>' |
| 47 |
|
| 48 |
base_url = 'https://www.private-eye.co.uk/' |
| 49 |
current_issue = 'https://www.private-eye.co.uk/news' |
| 50 |
about_page = 'https://www.private-eye.co.uk/about' |
| 51 |
number_crunching = 'https://www.private-eye.co.uk/number-crunching' |
| 52 |
masthead_url = 'https://www.private-eye.co.uk/grfx/logos/logo-new.png' |
| 53 |
classified_ads = 'https://www.eyeads.co.uk' |
| 54 |
next_issue_date = 'Next issue on sale:' |
| 55 |
|
| 56 |
author = 'Private Eye' |
| 57 |
series = title = 'Private Eye Online' |
| 58 |
conversion_options = { |
| 59 |
'authors': author, |
| 60 |
'author_sort': author, |
| 61 |
'series': series, |
| 62 |
'series_index': 0, |
| 63 |
'title': title, |
| 64 |
'title_sort': title, |
| 65 |
} |
| 66 |
|
| 67 |
index_attrs_to_include = [ |
| 68 |
{'id': 'sub-nav-box'}, |
| 69 |
{'class': 'sub-nav-bar'}, |
| 70 |
] |
| 71 |
|
| 72 |
titles_to_skip = [ |
| 73 |
'Home', |
| 74 |
'more', |
| 75 |
'In This Issue', |
| 76 |
] |
| 77 |
|
| 78 |
url_to_section_name = { |
| 79 |
'news': 'News', |
| 80 |
'street-of-shame': 'Street of Shame', |
| 81 |
'hp-sauce': 'HP Sauce', |
| 82 |
'cartoons': 'Strips and Cartoons', |
| 83 |
'pms-whatsapp': "PM's Whatsapp", |
| 84 |
'mediaballs': 'Commentatorballs', |
| 85 |
'lookalikes': 'Lookalike', |
| 86 |
'crossword': 'Eye Crossword', |
| 87 |
'in-the-back': 'In the Back', |
| 88 |
'media-news': 'Media News', |
| 89 |
'columnists': 'Columnists', |
| 90 |
'rotten-boroughs': 'Rotten Boroughs', |
| 91 |
'number-crunching': 'Number Crunching', |
| 92 |
'www.eyeads.co.uk': 'Private Eye Classified Ads', |
| 93 |
} |
| 94 |
|
| 95 |
re_extract_date = re.compile(r'\d.*\d') |
| 96 |
articles = [] |
| 97 |
urls = [] |
| 98 |
|
| 99 |
def add_article(self, title, url): |
| 100 |
if url in self.urls: |
| 101 |
return |
| 102 |
known_url = url.rsplit('/', 1)[-1] |
| 103 |
if known_url and known_url in self.url_to_section_name: |
| 104 |
title = self.url_to_section_name[known_url] |
| 105 |
if not title: |
| 106 |
return |
| 107 |
self.articles.append({ |
| 108 |
'title': title, |
| 109 |
'url': url, |
| 110 |
}) |
| 111 |
self.urls.append(url) |
| 112 |
self.log('Adding:', title, '(', url, ')') |
| 113 |
|
| 114 |
def get_cover_url(self): |
| 115 |
soup = self.index_to_soup(self.current_issue) |
| 116 |
|
| 117 |
for img in soup.findAll('img', {'class': 'issue-cover'}): |
| 118 |
src = img['src'] |
| 119 |
if src.endswith('_big.jpg'): |
| 120 |
file_name = src.rsplit('/', 1)[1] |
| 121 |
if file_name is None: |
| 122 |
file_name = src |
| 123 |
try: |
| 124 |
self.conversion_options.update({'series_index': int(file_name[: -len('_big.jpg')])}) |
| 125 |
self.log('series-index:', self.conversion_options['series_index']) |
| 126 |
except TypeError, ValueError: |
| 127 |
# wrong big image |
| 128 |
continue |
| 129 |
return src |
| 130 |
return None |
| 131 |
|
| 132 |
def get_next_issue_date(self, soup): |
| 133 |
sidebar = soup.find('div', id='sections-sidebar') |
| 134 |
if sidebar is None: |
| 135 |
return None |
| 136 |
for s in sidebar.stripped_strings: |
| 137 |
match = self.re_extract_date.match(s) |
| 138 |
if match: |
| 139 |
return match.group(0) |
| 140 |
return None |
| 141 |
|
| 142 |
def parse_index(self): |
| 143 |
self.log('') |
| 144 |
self.log('Title:', self.title) |
| 145 |
self.log('Description:', self.description) |
| 146 |
self.log('Authors:', self.__author__) |
| 147 |
self.log('Copyright:', self.__copyright__) |
| 148 |
self.log('') |
| 149 |
self.log('Support:', 'If this recipe stops working email [email protected] to let them know.') |
| 150 |
self.log('') |
| 151 |
|
| 152 |
soup = self.index_to_soup(self.current_issue) |
| 153 |
|
| 154 |
try: |
| 155 |
# Get publication date - Next issue on sale date - 13 days |
| 156 |
next_issue_date = self.get_next_issue_date(soup) |
| 157 |
self.log('next_issue_date:', next_issue_date) |
| 158 |
|
| 159 |
day, month, year = next_issue_date.split(' ') |
| 160 |
# remove day suffixes e.g. 2nd |
| 161 |
day = ''.join(c for c in day if c.isdigit()) |
| 162 |
pub_date = datetime.strptime(' '.join((day, month, year)), '%d %B %Y') - timedelta(13) |
| 163 |
self.log('pub-date:', pub_date) |
| 164 |
|
| 165 |
self.conversion_options.update({'pubdate': datetime.strftime(pub_date, '%d %B %Y').lstrip('0')}) |
| 166 |
title = self.title + ' ' + datetime.strftime(pub_date, '%Y-%m-%d') |
| 167 |
self.conversion_options.update({'title': title}) |
| 168 |
self.conversion_options.update({'title_sort': title}) |
| 169 |
except TypeError, ValueError: |
| 170 |
# Bad date |
| 171 |
self.log('Cannot parse next issue date from:', next_issue_date) |
| 172 |
|
| 173 |
# Get pages from the various contents panels. |
| 174 |
# Duplicates will be eliminated automatically. |
| 175 |
|
| 176 |
# Since we are now using the first online page to generate the list of pages to include |
| 177 |
# we need to include this one: |
| 178 |
|
| 179 |
self.add_article('News', self.current_issue) |
| 180 |
|
| 181 |
for section_attrs in self.index_attrs_to_include: |
| 182 |
section = soup.find('div', attrs=section_attrs) |
| 183 |
|
| 184 |
if not section: |
| 185 |
continue |
| 186 |
|
| 187 |
for a in section.findAll('a', href=True): |
| 188 |
url = a.get('href') |
| 189 |
title = a.getText().rstrip(' »\n') |
| 190 |
if title in self.titles_to_skip: |
| 191 |
continue |
| 192 |
if not url.startswith('http'): |
| 193 |
url = self.base_url + url |
| 194 |
self.add_article(title, url) |
| 195 |
|
| 196 |
if not self.articles: |
| 197 |
raise ValueError('Private-Eye Online index of pages not found') |
| 198 |
|
| 199 |
# Add the pages with no menu |
| 200 |
self.add_article('Number crunching', self.number_crunching) |
| 201 |
|
| 202 |
# Add the About page as a final article |
| 203 |
self.add_article('About Private Eye', self.about_page) |
| 204 |
|
| 205 |
# Add classified adverts - as a fun historical record |
| 206 |
self.add_article('Classified Ads', self.classified_ads) |
| 207 |
|
| 208 |
self.log('parse_index:', self.articles) |
| 209 |
|
| 210 |
return [('Private Eye', self.articles)] |
| 211 |
|
| 212 |
def preprocess_html(self, soup): |
| 213 |
# Remove <a> tag link to crossword image |
| 214 |
for tag in soup.findAll('a', {'href': re.compile(r'/pictures/crossword/')}): |
| 215 |
self.log('Removing link to crossword image...') |
| 216 |
tag.unwrap() |
| 217 |
|
| 218 |
# Remove align tag in crossword image (so float right works) |
| 219 |
for tag in soup.findAll('img', {'src': re.compile(r'/pictures/crossword/')}): |
| 220 |
if 'align' in tag.attrs: |
| 221 |
self.log('Removing crossword image align attribute...') |
| 222 |
del tag.attrs['align'] |
| 223 |
|
| 224 |
return soup |
| 225 |
|
| 226 |
# We remove vast swathes of HTML which is not part of the articles. |
| 227 |
# Remove sibling content |
| 228 |
remove_tags_before = [ |
| 229 |
{'name': 'div', 'class': 'article'}, |
| 230 |
{'name': 'div', 'id': 'page'}, |
| 231 |
{'name': 'div', 'id': 'page-wide'}, |
| 232 |
{'name': 'div', 'id': 'content'}, |
| 233 |
] |
| 234 |
remove_tags_after = remove_tags_before.copy() |
| 235 |
remove_tags_after.extend([ |
| 236 |
{'name': 'div', 'id': 'about-covers'}, |
| 237 |
{'name': 'script', 'id': 'about-covers'}, |
| 238 |
{'name': 'a', 'attrs': {'href': 'https://shop.private-eye.co.uk'}}, |
| 239 |
# {'name': 'img', 'attrs': {'src': re.compile(r'/grfx/logos/')}}, |
| 240 |
]) |
| 241 |
# Remove non-sibling content |
| 242 |
remove_tags = [ |
| 243 |
# Top |
| 244 |
{'name': 'div', 'attrs': {'id': 'header-wide'}}, |
| 245 |
{'name': 'div', 'attrs': {'id': 'top-bar-sticky'}}, |
| 246 |
{'name': 'div', 'attrs': {'id': 'nav-box-mobile'}}, |
| 247 |
{'name': 'div', 'attrs': {'id': 'nav-box-sections-mobile'}}, |
| 248 |
{'name': 'div', 'attrs': {'id': 'sub-nav-box'}}, |
| 249 |
{'name': 'div', 'attrs': {'class': 'sub-nav-bar'}}, |
| 250 |
# Content |
| 251 |
{'name': 'div', 'attrs': {'id': 'why-subscribe'}}, |
| 252 |
{'name': 'div', 'attrs': {'id': 'issue-box'}}, |
| 253 |
{'name': 'div', 'attrs': {'id': 'sections-sidebar'}}, |
| 254 |
# Spacers |
| 255 |
{'name': 'div', 'attrs': {'class': 'gap'}}, |
| 256 |
# {'name': 'div', 'attrs': {'class': 'gap-small'}}, |
| 257 |
{'name': 'div', 'attrs': {'class': 'gap-bigger'}}, |
| 258 |
{'name': 'div', 'attrs': {'class': 'gap-biggest'}}, |
| 259 |
# Bottom |
| 260 |
{'name': 'div', 'attrs': {'id': 'home-content'}}, |
| 261 |
{'name': 'div', 'attrs': {'id': 'home-shop-content-wide'}}, |
| 262 |
{'name': 'div', 'attrs': {'id': 'page-94-strip-wide'}}, |
| 263 |
{'name': 'div', 'attrs': {'id': 'footer-wide'}}, |
| 264 |
{'name': 'div', 'attrs': {'id': 'about-covers'}}, |
| 265 |
# {'name': 'div', 'attrs': {'id': 'follow-buttons'}}, |
| 266 |
# {'name': 'div', 'attrs': {'id': 'sidebar'}}, |
| 267 |
{'name': 'a', ' attrs': {'href': 'https://www.private-eye.co.uk/shop'}}, |
| 268 |
{'name': 'a', ' attrs': {'href': 'https://shop.private-eye.co.uk'}}, |
| 269 |
{'name': 'a', ' attrs': {'href': 'https://shop.private-eye.co.uk/'}}, |
| 270 |
{'name': 'link', 'attrs': {'href': re.compile(r'/javastyle/lightbox/')}}, |
| 271 |
{'name': 'link', 'attrs': {'href': re.compile(r'/javastyle/news_ticker/')}}, |
| 272 |
{'name': 'link', 'attrs': {'href': re.compile(r'/javastyle/media-queries-')}}, |
| 273 |
{'name': 'img', 'attrs': {'src': re.compile(r'/grfx/logos/gnitty.gif')}}, |
| 274 |
{'name': 'img', 'attrs': {'src': re.compile(r'/grfx/stuff/subscribe_here_gnitty.gif')}}, |
| 275 |
{'name': 'iframe'}, |
| 276 |
{'name': 'script'}, |
| 277 |
{'name': 'style'}, |
| 278 |
# Classified site |
| 279 |
{'name': 'a', 'attrs': {'class': 'skip-link'}}, |
| 280 |
{'name': 'header', 'attrs': {'id': 'masthead'}}, |
| 281 |
{'name': 'div', 'attrs': {'class': 'placeAdIntro'}}, |
| 282 |
{'name': 'img', 'attrs': {'class': 'cat-image'}}, |
| 283 |
{'name': 'img', 'attrs': {'class': 'catArrow'}}, |
| 284 |
{'name': 'div', 'attrs': {'class': 'ad-contact'}}, |
| 285 |
{'name': 'div', 'attrs': {'id': re.compile(r'cookie-law-')}}, |
| 286 |
{'name': 'div', 'attrs': {'class': re.compile(r'cli-modal')}}, |
| 287 |
{'name': 'footer', 'attrs': {'class': 'site-footer'}}, |
| 288 |
] |
| 289 |
|
| 290 |
preprocess_regexps = [ |
| 291 |
# Convert headers to h1, strapline to h4 |
| 292 |
( |
| 293 |
re.compile( |
| 294 |
r'<span class="headline(?:-new)?">(.*?)</span>\s*(?:<br/?>\s*)*((?:<span class="text">(.*?)</span>)|(?:<font .*?>(.*?)</font>))?', |
| 295 |
re.DOTALL | re.IGNORECASE, |
| 296 |
), |
| 297 |
lambda match: '<h1 class="myheadline">' + match[1] + '</h1>' + (('<h4 class="mystrapline">' + match[2] + '</h4>') if match[2] else ''), |
| 298 |
), |
| 299 |
# Remove broken links for this issue |
| 300 |
(re.compile(r'<a href="/issue-.*?".*?>(.*?)</a>', re.DOTALL | re.IGNORECASE), lambda match: match[1]), |
| 301 |
] |
| 302 |
|
| 303 |
# The following extra css is to tweak the formatting of various elements of various article pages. |
| 304 |
# Unfortunately, there are a variety of different pages styles, hence the extended tweak list. |
| 305 |
# Some of these mimic the actual layout.css which does not seem to make it across into the calibre |
| 306 |
# ebook without duplicating it as extra css. |
| 307 |
# However some is new css to tweak output when part of an ebook. |
| 308 |
extra_css = ' \n '.join([ |
| 309 |
'.article {font-family: Merriweather, Georgia, serif; font-size: 1em}', |
| 310 |
'.myheadline {font-family: "Source Sans Pro", Arial, Helvetica, sans-serif; font-size: 2em;}', |
| 311 |
'.mystrapline {font-family: "Source Sans Pro", Arial, Helvetica, sans-serif; font-size: 1.2em;}', |
| 312 |
'#content img {float: right; width: 45%; minimum-width:350px;}', |
| 313 |
'#content img.office {float: right; width: 45%; maximum-width:390px; margin-left: 15px;}', |
| 314 |
'#content img.cartoon-left {float: left; margin-right: 15px; margin-bottom: 15px;}', |
| 315 |
'#content img.cartoon-right {float: none; margin-bottom: 15px;}', |
| 316 |
'#content img.strip {float: none; width: 100%;}', |
| 317 |
'#content img:first-child {float: none;}', |
| 318 |
'#content img.gnitty-right {float: none; width: 160px;}', |
| 319 |
'#content #story > div[align=right] > img:first-child {float: none; width: 15px;}', |
| 320 |
'#content #story > img:first-child {float: none; height: 100px; width: none; minimum-width: none;}', |
| 321 |
'#content #block-sections div.divider {height: 1px; background-color: #eee; margin: 10px 0px; clear: both;}', |
| 322 |
'#content #block-sections img {float: none; width: none;}', |
| 323 |
'#content #block-sections img.lookalike {float: none; width: 100%;}', |
| 324 |
'#content #block-sections img.photo-right {float: right; width: 25%; min-width:120px; margin-left: 15px;}', |
| 325 |
'#content #block-sections > p:last-child > img:first-child {float: none; width: 120px;}', |
| 326 |
'#content #block-sections > p:last-child > img:nth-child(2) {float: none; width: 120px;}', |
| 327 |
'#content #block-sections img.crossword {float: right; width: 40%; margin-left: 15px; min-width: 350px;}', |
| 328 |
'#content #article-caption-box {float: right; background: #222222; display: block; width: 40%; min-width: 250px; font-size: 90%; margin-left: 15px;}', |
| 329 |
'#content #article-caption-box img {float: none; width: 100%; max-width: none;}', |
| 330 |
'#caption-box {color: #ffffff; text-align: center; padding: 5px 20px 15px 20px;}', |
| 331 |
'#whatsapp {border-left: 5px #f45e61 solid; border-right: 5px #f45e61 solid; border-bottom: 5px #f45e61 solid; padding: 10px 20px 20px 10px;}', |
| 332 |
'#whatsapp::after {clear:both;}', |
| 333 |
'#whatsapp .whatsapp-left, .whatsapp-right {margin: 20px 0; padding: 15px; border-radius: 10px;}', |
| 334 |
'#whatsapp .whatsapp-left, .whatsapp-right {font-family: Helvetica, Arial, "sans-serif"; font-weight: 300; font-size: 18px; line-height: 24px;}', |
| 335 |
'#whatsapp .whatsapp-left {margin-right: 30%; background-color: #eeeeee;}', |
| 336 |
'#whatsapp .whatsapp-right {margin-left: 30%; background-color: #dce5ae;}', |
| 337 |
'#whatsapp .whatsapp-left img, #whatsapp .whatsapp-right img {width: 35px; margin: 0 10px; vertical-align: middle;}', |
| 338 |
'#whatsapp .whatsapp-left img.emoji, #whatsapp .whatsapp-right img.emoji {max-width: 35px; margin: 0 5px; vertical-align: middle;}', |
| 339 |
'.container .row .catTitle {color: fff; background-color: #12174a;}', |
| 340 |
]) |