Debian Codegraph @

recipes/fokus.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 from datetime import datetime
4
5 from calibre.web.feeds.news import BasicNewsRecipe
6
7
8 class NoArticles(Exception):
9 pass
10
11
12 class Fokus(BasicNewsRecipe):
13 title = 'Fokus'
14 main_url = 'https://www.fokus.se'
15 description = "The current week's edition of Swedish current-affairs magazine 'Fokus'"
16 encoding = 'utf-8'
17 __author__ = 'Henrik Holm (https://github.com/h-holm)'
18 language = 'sv'
19 ignore_duplicate_articles = {'title', 'url'}
20 masthead_url = 'https://cdn.fokus.se/app/uploads/fokus/2022/05/12214931/fokus-logo.svg'
21 no_stylesheets = True
22 compress_news_images = True
23 needs_subscription = 'optional'
24 use_embedded_content = False
25 remove_empty_feeds = True
26 scale_news_images_to_device = True
27 scale_news_images = (800, 600)
28 delay = 3 # Avoid throttling by the server.
29
30 # 1. Center and reduce the size of images and image captions.
31 # 2. Make the lead text italic.
32 # 3. Make the article metadata text gray and small.
33 extra_css = '''
34 img { display: block; margin: auto; width: 50%; height: auto; }
35 div.calibre-nuked-tag-figure { font-size: small; text-align: center; }
36 p.Single__lead, p.Longread__lead { font-style: italic; color:#202020; }
37 p.article-metadata { color: gray; font-size:small; }
38 '''
39
40 remove_tags = [
41 dict(name='div', attrs={'class': 'Paywall'}),
42 ]
43
44 keep_only_tags = [
45 dict(name='h1', class_='Single__title'), # Title of "Single" type articles.
46 dict(name='h1', class_='Longread__title'), # Title of "Longread" type articles.
47 dict(name='p', class_='Single__lead'), # Lead text of "Single" type articles.
48 dict(name='p', class_='Longread__lead'), # Lead text of "Longread" type articles.
49 dict(name='p', class_='article-metadata'), # Dynamically created by the recipe.
50 dict(name='figure', class_='Single__thumbnail'), # Image of "Single" type articles.
51 dict(name='figure', class_='Longread__thumbnail'), # Image of "Longread" type articles.
52 dict(name='div', class_='Single__content'), # Article body of "Single" type articles.
53 dict(name='div', class_='Longread__content'), # Article body of "Longread" type articles.
54 ]
55
56 def extract_cover_url(self, a_tag) -> str:
57 """Given the <a> tag of the current edition, extract the URL of the highest-resolution cover image."""
58 figure_tag = a_tag.find('figure')
59 img_tag = figure_tag.find('img')
60
61 try:
62 # The `srcset` attribute contains a comma-separated list of URLs and their resolutions.
63 cover_urls = img_tag['srcset'].split(', ')
64 cover_urls = [src.split(' ') for src in cover_urls]
65
66 # The second item of each tuple should be the resolution, e.g., '578w' or '821w'. Remove the 'w' suffix, cast
67 # to an integer and sort in descending order.
68 cover_urls = [(url, int(resolution[:-1])) for url, resolution in cover_urls]
69 cover_urls = sorted(cover_urls, key=lambda x: x[1], reverse=True)
70
71 # The first item of the sorted list is now the URL of the highest-resolution image.
72 self.cover_url = cover_urls[0][0]
73 self.log(f"Identified cover URL: '{self.cover_url}'")
74 except (KeyError, ValueError) as exc:
75 self.log.error(f'Failed to extract cover URL! Has the website format changed?\n{exc}')
76
77 return
78
79 def get_current_edition_url(self) -> str:
80 """Return the URL of the current (weekly) edition of Fokus."""
81 current_year = datetime.now().year
82 current_week = datetime.now().isocalendar().week
83 try:
84 soup = self.index_to_soup(f'{self.main_url}/vara-utgavor')
85
86 # Identify all <a> tags of class 'Issue' that have an href attribute.
87 a_tags = soup.find_all('a', class_='Issue', href=True)
88
89 # Keep only the href, and subset to only those links that contain the current year.
90 edition_links = [a_tag['href'] for a_tag in a_tags if str(current_year) in a_tag['href']]
91
92 # To successfully sort the links chronologically, first convert the data structure to a dict, wherein the
93 # key consists of only the date part of the URL and the value consists of the entire (unmodified) URL.
94 edition_links = {link.removesuffix('/').split('/')[-1]: link for link in edition_links}
95
96 # Shorten the key by removing the year prefix.
97 edition_links = {key.split('-', 1)[-1]: value for key, value in edition_links.items()}
98
99 # Most keys in the `edition_links` dict now correspond to calendar week numbers, e.g., '1', '02', '21' or
100 # '52'. Note however that editions can sometimes cover multiples weeks, e.g., '1-2', '01-03', '50-51-52',
101 # '26/27' or '50/51/52'. For entries where this is the case, split the entry into multiple entries.
102 _edition_links = {}
103 for key, value in edition_links.items():
104 # Handle both '-' and '/' as week number separators.
105 if '-' in key:
106 week_numbers = key.split('-')
107 elif '/' in key:
108 week_numbers = key.split('/')
109 else:
110 week_numbers = [key]
111 for week_number in week_numbers:
112 _edition_links[week_number] = value
113 edition_links = _edition_links
114
115 # Convert the resulting keys to integers.
116 edition_links = {int(key): value for key, value in edition_links.items()}
117
118 if current_week in edition_links:
119 # If there exists an entry that corresponds to the `current_week`, use that entry.
120 current_edition_url = edition_links[current_week]
121 else:
122 # Otherwise, use the entry that corresponds to the highest week number.
123 max_week_number = max(edition_links.keys())
124 current_edition_url = edition_links[max_week_number]
125
126 self.log(f"Identified {len(edition_links)} editions, of which the most recent is '{current_edition_url}'.")
127
128 # Now that we know the URL of the current edition, we can use it to identify the cover image. The cover
129 # image URL exists in the `src` attribute of the <img> child tag of the <figure> child tag of the <a> tag
130 # of the current edition.
131 current_edition_a_tag = soup.find('a', class_='Issue', href=current_edition_url)
132 self.extract_cover_url(current_edition_a_tag)
133 except Exception as exc:
134 self.log.error(f'Failed to identify the current edition URL: {exc}')
135 raise NoArticles(
136 f"Could not find the URL of the current edition. Either the '{self.main_url}' server is experiencing "
137 'issues, in which case you should try again later, or the website format has changed and the recipe '
138 'needs updating.'
139 ) from exc
140 return current_edition_url
141
142 def parse_article_blurb(self, article_blurb) -> dict[str, str, str, str, str] | None:
143 """Given a <article> tag of class 'Blurb', parse it into a dict.
144
145 Args:
146 article_blurb (Tag): An <article> tag hosting metadata and the URL of an article.
147
148 Returns:
149 A dict on a `{'url': str, 'title': str, 'date': str, 'category': str, 'description': str}` format.
150 """
151 if a_tag := article_blurb.find('a', href=True):
152 url = a_tag['href'].strip().rstrip('/')
153 if url.startswith('/'):
154 url = f'{self.main_url}{url}'
155
156 if title_tag := a_tag.find('h2', {'class': 'Blurb__title'}):
157 title = self.tag_to_string(title_tag).strip()
158
159 if date_tag := a_tag.find('time', {'class': 'Blurb__date'}):
160 # Results in a Swedish date format, e.g., '23 MAJ 2025'.
161 date = self.tag_to_string(date_tag).strip()
162 # Add a newline before the date to make it more readable.
163 date = f'\n{date}'
164
165 # Assign the article to its first listed category as inferred from the first <li> tag of class
166 # 'Blurb__category'. Default to 'Fokus' if no such tag is found.
167 category = 'Fokus'
168 if category_tag := a_tag.find('li', {'class': 'Blurb__category'}):
169 category = self.tag_to_string(category_tag).strip()
170
171 desc = ''
172 if desc_tag := a_tag.find('div', {'class': 'Blurb__summary'}):
173 desc = self.tag_to_string(desc_tag).strip()
174
175 return {'url': url, 'title': title, 'date': date, 'category': category, 'description': desc}
176
177 return
178
179 def get_article_blurbs(self, soup) -> dict[str, dict[str, str, str, str, str]]:
180 """Given a Fokus webpage `soup`, return a dict of unique article entries found on the page.
181
182 The key of a given entry in the output dictionary is the article URL. The corresponding value is a dictionary
183 on a `{'url': str, 'title': str, 'date': str, 'category': str, 'description': str}` format.
184
185 Args:
186 soup (BeautifulSoup): The `bs4.BeautifulSoup` soup of a Fokus webpage.
187
188 Returns:
189 dict[str, dict[str, str, str, str, str]]: A dict with article URLs as keys and 'article dicts' as values.
190 """
191
192 def _log(article) -> None:
193 """Log a digestible summary of the input `article` blurb."""
194 log_message = f"\t{article['title']} : {article['url']}"
195 if article.get('description'):
196 log_message += f" : {article['description']}"
197 self.log.debug(log_message)
198
199 # Identify all <article> tags of class 'Blurb' that have an href attribute.
200 self.log('Identifying all articles...')
201 try:
202 article_blurbs = soup.find_all('article', {'class': 'Blurb'})
203 except AttributeError:
204 article_blurbs = []
205
206 if not article_blurbs:
207 self.log.error('Failed to identify any article blurbs.')
208 return {}
209
210 parsed_blurbs = {}
211 for article_blurb in article_blurbs:
212 if article := self.parse_article_blurb(article_blurb):
213 _log(article)
214 parsed_blurbs[article['url']] = article
215
216 return parsed_blurbs
217
218 def convert_to_section_lists(
219 self,
220 articles: dict[str, dict[str, str, str, str, str]],
221 ) -> dict[str, list[dict[str, str, str, str]]]:
222 """Convert the `articles` dict of dicts to a dict of lists; each list holds the articles of a given section.
223
224 Args:
225 articles (dict[str, dict[str, str, str, str, str]]): A dict of article URLs and article dicts.
226
227 Returns:
228 dict[str, list[dict[str, str, str, str]]]: A dict on a `{section_title: list[article_dict]}` format.
229 """
230 self.log(f'Assigning each of the {len(articles)} articles to a section...')
231 section_to_articles = {}
232 for _, article_dict in articles.items():
233 section_title = article_dict['category']
234 if section_title not in section_to_articles:
235 section_to_articles[section_title] = []
236 # Remove the 'category' key from the article dict, as it is not needed in the final output.
237 article_dict.pop('category')
238 section_to_articles[section_title].append(article_dict)
239
240 return section_to_articles
241
242 def parse_index(self):
243 current_edition_url = self.get_current_edition_url()
244 if not current_edition_url:
245 raise NoArticles(
246 f"Could not find the URL of the current edition. Either the '{self.main_url}' server is experiencing "
247 'issues, in which case you should try again later, or the website format has changed and the recipe '
248 'needs updating.'
249 )
250 self.log(f'Current edition URL: {current_edition_url}')
251
252 # Identify all sections in the web version of Fokus.
253 edition_soup = self.index_to_soup(current_edition_url)
254
255 # From the section URLs and the main URL, identify all unique articles.
256 articles = self.get_article_blurbs(edition_soup)
257 if not articles:
258 raise NoArticles(
259 f"Could not find any articles. Either the '{self.main_url}' server is experiencing issues, in which "
260 'case you should try again later, or the website format has changed and the recipe needs updating.'
261 )
262
263 # Assign each identified article to a section based on its URL.
264 section_to_articles = self.convert_to_section_lists(articles)
265
266 # Convert to tuples.
267 return list(section_to_articles.items())