Debian Codegraph @

recipes/theoldie.recipe

raw
1 """
2 Fetch The Oldie (Online Edition)
3 """
4
5 import re
6 from datetime import datetime
7
8 from calibre.web.feeds.news import BasicNewsRecipe
9
10
11 class PrivateEyeRecipe(BasicNewsRecipe):
12 ##
13 # Last Edited: 2023-08-07
14 #
15 # Remark: Version 1.0 2023-08-07
16 # Initial version
17
18 title = 'The Oldie (Online Edition)'
19 description = (
20 'The Oldie has been dubbed ‘Private Eye for grown-ups’ and is read by intelligent people who are fed'
21 ' up with the formulaic nature of the celebrity-obsessed national press. The Oldie was cooked up in'
22 ' 1992 by Richard Ingrams (who previously co-founded Private Eye in 1961) as a free-thinking,'
23 ' funny magazine, a light-hearted alternative to a press obsessed with youth and celebrity.'
24 ' The editors claim that the Oldie is ageless and timeless, free of retirement advice, crammed'
25 ' with rejuvenating wit, intelligence and delight.'
26 )
27 publication_type = 'magazine'
28 language = 'en_GB'
29 encoding = 'utf-8'
30 oldest_article = 31
31 max_articles_per_feed = 100
32 remove_javascript = True
33 ignore_duplicate_articles = {'url'}
34
35 __author__ = 'Sophist-UK'
36 __copyright__ = '2023, Sophist-UK <[email protected]>'
37
38 web_root = 'https://www.theoldie.co.uk'
39 current_issue = web_root + '/magazine'
40 about_pages = {
41 'About Us': web_root + '/about-us',
42 'Our History': web_root + '/about-us/history',
43 }
44 masthead_url = web_root + '/assets/images/theoldie_logo_22.png'
45 name = 'Oldie Online'
46 series = 'The ' + name
47 now = datetime.now().strftime(' %Y-%m')
48 title = series + now # noqa: PIE794
49 title_sort = name + now + ', The'
50 conversion_options = {
51 'authors': 'The Oldie',
52 'author_sort': 'Oldie, The',
53 'series': series,
54 'series_index': 0,
55 'title': title,
56 'title_sort': title_sort,
57 }
58 cover_suburl = '-front-cover-'
59
60 # Convert relative URLS to absolute ones i.e. /cover to https://theoldie.co.uk/cover
61 def abs_url(self, url):
62 return self.web_root + url if url.startswith('/') else url
63
64 # Create a correctly formatted DICT entry for Calibre parse_index return
65 def article_entry(self, title, url, author=None):
66 article = {
67 'title': title,
68 'url': url,
69 }
70 if author:
71 article['author'] = author
72 return article
73
74 edition_re = re.compile(r'(?:-front-cover-)(\d+)-')
75
76 # Identify the cover image and extract the edition# from the url
77 def get_cover_url(self):
78 soup = self.index_to_soup(self.current_issue)
79
80 for img in soup.findAll('img'):
81 src = self.abs_url(img['src'])
82 editions = self.edition_re.findall(src)
83 if editions:
84 try:
85 self.conversion_options.update({'series_index': int(editions[0])})
86 self.log('series-index:', self.conversion_options['series_index'])
87 except TypeError, ValueError:
88 continue
89 self.log('cover_url:', src)
90 return src
91 return None
92
93 # oldie links/headings often contain the author (in one of various formats
94 # 1. Title. By author
95 # 2. Title by author: subtitle
96 # 3. Title: author: subtitle
97 title_author_re = re.compile(r'^(.*?)(?:(?: by )|(?:: ))(.*?): (.*?)$')
98
99 # Separate author from title (where it is specified)
100 def title_author(self, head):
101 if '. By ' in head:
102 return head.rsplit('. By ', 1)
103 matches = self.title_author_re.findall(head)
104 if matches and len(matches[0]) == 3:
105 title_1, author, title_2 = matches[0]
106 title = ': '.join((title_1, title_2))
107 return title, author
108 return head, None
109
110 # Return the list of articles from blocks in the content of an index/listing page
111 def parse_content(self, soup):
112 content_articles = []
113
114 content = soup.find('div', class_='content-wrapper')
115
116 if not content:
117 return content_articles
118
119 for article in content.findAll('div', class_='listing-block'):
120 for a in article.findAll('a', href=True):
121 for h in a.findAll('h3'):
122 title, author = self.title_author(h.getText())
123 content_articles.append(
124 self.article_entry(
125 title=title,
126 url=self.abs_url(a.get('href')),
127 author=author,
128 )
129 )
130 break
131 else:
132 continue
133 break
134
135 return content_articles
136
137 def parse_index(self):
138 # The set of pages to be used in the online edition are:
139 # 1. The list of articles in the body of the magazine index page
140 # 2. The contents / pages linked to by each of the links in the #categories menu
141 # 3. The div.only-in-the-magazine contents in the magazine index page
142 # 4. The about pages
143 # Obviously repeated content is de-duplicated by Calibre
144
145 self.log('masthead_url:', self.masthead_url)
146 soup = self.index_to_soup(self.current_issue)
147
148 # 1. The list of articles in the body of the magazine index page
149 articles = self.parse_content(soup)
150
151 # 2. The contents / pages linked to by each of the links in the #categories menu
152 categories = soup.find('nav', class_='categories')
153 for li in categories.findAll('li'):
154 a = li.find('a', href=True)
155 href = self.abs_url(a.get('href'))
156 self.log('Checking page for sub-index:', href)
157 content = self.parse_content(self.index_to_soup(href))
158 if content:
159 self.log('Subpages found:', href, len(content))
160 articles.extend(content)
161 else:
162 title, author = self.title_author(a.getText())
163 articles.append(
164 self.article_entry(
165 title=title,
166 url=self.abs_url(a.get('href')),
167 author=author,
168 )
169 )
170
171 if not articles:
172 raise ValueError('The Oldie Online index of pages not found')
173
174 # 3. The div.only-in-the-magazine contents in the magazine index page
175 articles.append({
176 'title': 'In the full issue…',
177 'url': self.current_issue,
178 })
179
180 pages = [('In this issue…', articles)]
181 self.log('n this issue…', articles)
182
183 # 4. The about pages
184 abouts = []
185 for title, url in self.about_pages.items():
186 abouts.append({
187 'title': title,
188 'url': url,
189 })
190
191 if abouts:
192 pages.append(('About The Oldie', abouts))
193 self.log('About The Oldie', abouts)
194
195 return pages
196
197 def preprocess_html(self, soup):
198 for h in soup.findAll('h1'):
199 title, author = self.title_author(h.getText())
200 self.log('Replacing h3 "', h.getText(), '" with "', title, '"')
201 h.string = title
202
203 return soup
204
205 # Remove features not wanted and tweak HTML
206 preprocess_regexps = [
207 # Remove big blank spaces
208 (re.compile(r'<p>\s*<br\/?>\s*</p>', re.DOTALL | re.IGNORECASE), lambda match: ''),
209 # Local fix for paragraph HTML issues join paragraphs that do not end in a full-stop.
210 (
211 re.compile(r'(?<=[^\.\s])\s*</p>\s*<p>', re.DOTALL | re.IGNORECASE),
212 lambda match: ' ', # space
213 ),
214 ]
215
216 # We remove vast swathes of HTML which is not part of the articles.
217 remove_tags_before = [
218 {'name': 'div', 'class': 'container'},
219 {'name': 'div', 'class': 'content-wrapper'},
220 {'name': 'div', 'class': 'only-in-the-magazine'},
221 ]
222 remove_tags_after = [
223 {'name': 'div', 'class': 'container'},
224 {'name': 'div', 'class': 'content-wrapper'},
225 {'name': 'h2', 'string': 'Find out more about The Oldie'},
226 ]
227 # Remove non-sibling content
228 remove_tags = [
229 {'name': 'nav', 'class': 'categories'},
230 {'name': 'div', 'class': 'internal-placeholders'},
231 {'name': 'div', 'class': 'leaderboard'},
232 {'name': 'div', 'class': 'share'},
233 {'name': 'div', 'class': 'most-popular'},
234 {'name': 'div', 'class': 'article-convert'},
235 # {'name': 'p', 'class': "article-convert"},
236 # {'name': 'p', 'class': "meta"},
237 {'name': 'hr'},
238 {'name': 'a', 'class': 'view-full-screen'},
239 {'name': 'div', 'class': 'image-counter'},
240 {'name': 'h2', 'string': 'Find out more about The Oldie'},
241 {'name': 'a', 'href': re.compile(r'^https?:\/\/issuu.com\/')},
242 {'name': 'img', 'src': re.compile(r'\/assets\/images\/icons\/icon-')},
243 ]
244
245 # The following extra css is to tweak the formatting of various elements of various article pages.
246 extra_css = ' \n '.join([
247 'div.image-captions div.caption {text-align: center; font-weight: bold; width:750px;}',
248 'p.article-convert {text-align: center;}',
249 ])