| 1 |
""" |
| 2 |
Fetch The Oldie (Online Edition) |
| 3 |
""" |
| 4 |
|
| 5 |
import re |
| 6 |
from datetime import datetime |
| 7 |
|
| 8 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 9 |
|
| 10 |
|
| 11 |
class PrivateEyeRecipe(BasicNewsRecipe): |
| 12 |
## |
| 13 |
# Last Edited: 2023-08-07 |
| 14 |
# |
| 15 |
# Remark: Version 1.0 2023-08-07 |
| 16 |
# Initial version |
| 17 |
|
| 18 |
title = 'The Oldie (Online Edition)' |
| 19 |
description = ( |
| 20 |
'The Oldie has been dubbed ‘Private Eye for grown-ups’ and is read by intelligent people who are fed' |
| 21 |
' up with the formulaic nature of the celebrity-obsessed national press. The Oldie was cooked up in' |
| 22 |
' 1992 by Richard Ingrams (who previously co-founded Private Eye in 1961) as a free-thinking,' |
| 23 |
' funny magazine, a light-hearted alternative to a press obsessed with youth and celebrity.' |
| 24 |
' The editors claim that the Oldie is ageless and timeless, free of retirement advice, crammed' |
| 25 |
' with rejuvenating wit, intelligence and delight.' |
| 26 |
) |
| 27 |
publication_type = 'magazine' |
| 28 |
language = 'en_GB' |
| 29 |
encoding = 'utf-8' |
| 30 |
oldest_article = 31 |
| 31 |
max_articles_per_feed = 100 |
| 32 |
remove_javascript = True |
| 33 |
ignore_duplicate_articles = {'url'} |
| 34 |
|
| 35 |
__author__ = 'Sophist-UK' |
| 36 |
__copyright__ = '2023, Sophist-UK <[email protected]>' |
| 37 |
|
| 38 |
web_root = 'https://www.theoldie.co.uk' |
| 39 |
current_issue = web_root + '/magazine' |
| 40 |
about_pages = { |
| 41 |
'About Us': web_root + '/about-us', |
| 42 |
'Our History': web_root + '/about-us/history', |
| 43 |
} |
| 44 |
masthead_url = web_root + '/assets/images/theoldie_logo_22.png' |
| 45 |
name = 'Oldie Online' |
| 46 |
series = 'The ' + name |
| 47 |
now = datetime.now().strftime(' %Y-%m') |
| 48 |
title = series + now # noqa: PIE794 |
| 49 |
title_sort = name + now + ', The' |
| 50 |
conversion_options = { |
| 51 |
'authors': 'The Oldie', |
| 52 |
'author_sort': 'Oldie, The', |
| 53 |
'series': series, |
| 54 |
'series_index': 0, |
| 55 |
'title': title, |
| 56 |
'title_sort': title_sort, |
| 57 |
} |
| 58 |
cover_suburl = '-front-cover-' |
| 59 |
|
| 60 |
# Convert relative URLS to absolute ones i.e. /cover to https://theoldie.co.uk/cover |
| 61 |
def abs_url(self, url): |
| 62 |
return self.web_root + url if url.startswith('/') else url |
| 63 |
|
| 64 |
# Create a correctly formatted DICT entry for Calibre parse_index return |
| 65 |
def article_entry(self, title, url, author=None): |
| 66 |
article = { |
| 67 |
'title': title, |
| 68 |
'url': url, |
| 69 |
} |
| 70 |
if author: |
| 71 |
article['author'] = author |
| 72 |
return article |
| 73 |
|
| 74 |
edition_re = re.compile(r'(?:-front-cover-)(\d+)-') |
| 75 |
|
| 76 |
# Identify the cover image and extract the edition# from the url |
| 77 |
def get_cover_url(self): |
| 78 |
soup = self.index_to_soup(self.current_issue) |
| 79 |
|
| 80 |
for img in soup.findAll('img'): |
| 81 |
src = self.abs_url(img['src']) |
| 82 |
editions = self.edition_re.findall(src) |
| 83 |
if editions: |
| 84 |
try: |
| 85 |
self.conversion_options.update({'series_index': int(editions[0])}) |
| 86 |
self.log('series-index:', self.conversion_options['series_index']) |
| 87 |
except TypeError, ValueError: |
| 88 |
continue |
| 89 |
self.log('cover_url:', src) |
| 90 |
return src |
| 91 |
return None |
| 92 |
|
| 93 |
# oldie links/headings often contain the author (in one of various formats |
| 94 |
# 1. Title. By author |
| 95 |
# 2. Title by author: subtitle |
| 96 |
# 3. Title: author: subtitle |
| 97 |
title_author_re = re.compile(r'^(.*?)(?:(?: by )|(?:: ))(.*?): (.*?)$') |
| 98 |
|
| 99 |
# Separate author from title (where it is specified) |
| 100 |
def title_author(self, head): |
| 101 |
if '. By ' in head: |
| 102 |
return head.rsplit('. By ', 1) |
| 103 |
matches = self.title_author_re.findall(head) |
| 104 |
if matches and len(matches[0]) == 3: |
| 105 |
title_1, author, title_2 = matches[0] |
| 106 |
title = ': '.join((title_1, title_2)) |
| 107 |
return title, author |
| 108 |
return head, None |
| 109 |
|
| 110 |
# Return the list of articles from blocks in the content of an index/listing page |
| 111 |
def parse_content(self, soup): |
| 112 |
content_articles = [] |
| 113 |
|
| 114 |
content = soup.find('div', class_='content-wrapper') |
| 115 |
|
| 116 |
if not content: |
| 117 |
return content_articles |
| 118 |
|
| 119 |
for article in content.findAll('div', class_='listing-block'): |
| 120 |
for a in article.findAll('a', href=True): |
| 121 |
for h in a.findAll('h3'): |
| 122 |
title, author = self.title_author(h.getText()) |
| 123 |
content_articles.append( |
| 124 |
self.article_entry( |
| 125 |
title=title, |
| 126 |
url=self.abs_url(a.get('href')), |
| 127 |
author=author, |
| 128 |
) |
| 129 |
) |
| 130 |
break |
| 131 |
else: |
| 132 |
continue |
| 133 |
break |
| 134 |
|
| 135 |
return content_articles |
| 136 |
|
| 137 |
def parse_index(self): |
| 138 |
# The set of pages to be used in the online edition are: |
| 139 |
# 1. The list of articles in the body of the magazine index page |
| 140 |
# 2. The contents / pages linked to by each of the links in the #categories menu |
| 141 |
# 3. The div.only-in-the-magazine contents in the magazine index page |
| 142 |
# 4. The about pages |
| 143 |
# Obviously repeated content is de-duplicated by Calibre |
| 144 |
|
| 145 |
self.log('masthead_url:', self.masthead_url) |
| 146 |
soup = self.index_to_soup(self.current_issue) |
| 147 |
|
| 148 |
# 1. The list of articles in the body of the magazine index page |
| 149 |
articles = self.parse_content(soup) |
| 150 |
|
| 151 |
# 2. The contents / pages linked to by each of the links in the #categories menu |
| 152 |
categories = soup.find('nav', class_='categories') |
| 153 |
for li in categories.findAll('li'): |
| 154 |
a = li.find('a', href=True) |
| 155 |
href = self.abs_url(a.get('href')) |
| 156 |
self.log('Checking page for sub-index:', href) |
| 157 |
content = self.parse_content(self.index_to_soup(href)) |
| 158 |
if content: |
| 159 |
self.log('Subpages found:', href, len(content)) |
| 160 |
articles.extend(content) |
| 161 |
else: |
| 162 |
title, author = self.title_author(a.getText()) |
| 163 |
articles.append( |
| 164 |
self.article_entry( |
| 165 |
title=title, |
| 166 |
url=self.abs_url(a.get('href')), |
| 167 |
author=author, |
| 168 |
) |
| 169 |
) |
| 170 |
|
| 171 |
if not articles: |
| 172 |
raise ValueError('The Oldie Online index of pages not found') |
| 173 |
|
| 174 |
# 3. The div.only-in-the-magazine contents in the magazine index page |
| 175 |
articles.append({ |
| 176 |
'title': 'In the full issue…', |
| 177 |
'url': self.current_issue, |
| 178 |
}) |
| 179 |
|
| 180 |
pages = [('In this issue…', articles)] |
| 181 |
self.log('n this issue…', articles) |
| 182 |
|
| 183 |
# 4. The about pages |
| 184 |
abouts = [] |
| 185 |
for title, url in self.about_pages.items(): |
| 186 |
abouts.append({ |
| 187 |
'title': title, |
| 188 |
'url': url, |
| 189 |
}) |
| 190 |
|
| 191 |
if abouts: |
| 192 |
pages.append(('About The Oldie', abouts)) |
| 193 |
self.log('About The Oldie', abouts) |
| 194 |
|
| 195 |
return pages |
| 196 |
|
| 197 |
def preprocess_html(self, soup): |
| 198 |
for h in soup.findAll('h1'): |
| 199 |
title, author = self.title_author(h.getText()) |
| 200 |
self.log('Replacing h3 "', h.getText(), '" with "', title, '"') |
| 201 |
h.string = title |
| 202 |
|
| 203 |
return soup |
| 204 |
|
| 205 |
# Remove features not wanted and tweak HTML |
| 206 |
preprocess_regexps = [ |
| 207 |
# Remove big blank spaces |
| 208 |
(re.compile(r'<p>\s*<br\/?>\s*</p>', re.DOTALL | re.IGNORECASE), lambda match: ''), |
| 209 |
# Local fix for paragraph HTML issues join paragraphs that do not end in a full-stop. |
| 210 |
( |
| 211 |
re.compile(r'(?<=[^\.\s])\s*</p>\s*<p>', re.DOTALL | re.IGNORECASE), |
| 212 |
lambda match: ' ', # space |
| 213 |
), |
| 214 |
] |
| 215 |
|
| 216 |
# We remove vast swathes of HTML which is not part of the articles. |
| 217 |
remove_tags_before = [ |
| 218 |
{'name': 'div', 'class': 'container'}, |
| 219 |
{'name': 'div', 'class': 'content-wrapper'}, |
| 220 |
{'name': 'div', 'class': 'only-in-the-magazine'}, |
| 221 |
] |
| 222 |
remove_tags_after = [ |
| 223 |
{'name': 'div', 'class': 'container'}, |
| 224 |
{'name': 'div', 'class': 'content-wrapper'}, |
| 225 |
{'name': 'h2', 'string': 'Find out more about The Oldie'}, |
| 226 |
] |
| 227 |
# Remove non-sibling content |
| 228 |
remove_tags = [ |
| 229 |
{'name': 'nav', 'class': 'categories'}, |
| 230 |
{'name': 'div', 'class': 'internal-placeholders'}, |
| 231 |
{'name': 'div', 'class': 'leaderboard'}, |
| 232 |
{'name': 'div', 'class': 'share'}, |
| 233 |
{'name': 'div', 'class': 'most-popular'}, |
| 234 |
{'name': 'div', 'class': 'article-convert'}, |
| 235 |
# {'name': 'p', 'class': "article-convert"}, |
| 236 |
# {'name': 'p', 'class': "meta"}, |
| 237 |
{'name': 'hr'}, |
| 238 |
{'name': 'a', 'class': 'view-full-screen'}, |
| 239 |
{'name': 'div', 'class': 'image-counter'}, |
| 240 |
{'name': 'h2', 'string': 'Find out more about The Oldie'}, |
| 241 |
{'name': 'a', 'href': re.compile(r'^https?:\/\/issuu.com\/')}, |
| 242 |
{'name': 'img', 'src': re.compile(r'\/assets\/images\/icons\/icon-')}, |
| 243 |
] |
| 244 |
|
| 245 |
# The following extra css is to tweak the formatting of various elements of various article pages. |
| 246 |
extra_css = ' \n '.join([ |
| 247 |
'div.image-captions div.caption {text-align: center; font-weight: bold; width:750px;}', |
| 248 |
'p.article-convert {text-align: center;}', |
| 249 |
]) |