| 1 |
#!/usr/bin/env python |
| 2 |
# vim:fileencoding=utf-8 |
| 3 |
from datetime import datetime |
| 4 |
|
| 5 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 6 |
|
| 7 |
|
| 8 |
class NoArticles(Exception): |
| 9 |
pass |
| 10 |
|
| 11 |
|
| 12 |
class Fokus(BasicNewsRecipe): |
| 13 |
title = 'Fokus' |
| 14 |
main_url = 'https://www.fokus.se' |
| 15 |
description = "The current week's edition of Swedish current-affairs magazine 'Fokus'" |
| 16 |
encoding = 'utf-8' |
| 17 |
__author__ = 'Henrik Holm (https://github.com/h-holm)' |
| 18 |
language = 'sv' |
| 19 |
ignore_duplicate_articles = {'title', 'url'} |
| 20 |
masthead_url = 'https://cdn.fokus.se/app/uploads/fokus/2022/05/12214931/fokus-logo.svg' |
| 21 |
no_stylesheets = True |
| 22 |
compress_news_images = True |
| 23 |
needs_subscription = 'optional' |
| 24 |
use_embedded_content = False |
| 25 |
remove_empty_feeds = True |
| 26 |
scale_news_images_to_device = True |
| 27 |
scale_news_images = (800, 600) |
| 28 |
delay = 3 # Avoid throttling by the server. |
| 29 |
|
| 30 |
# 1. Center and reduce the size of images and image captions. |
| 31 |
# 2. Make the lead text italic. |
| 32 |
# 3. Make the article metadata text gray and small. |
| 33 |
extra_css = ''' |
| 34 |
img { display: block; margin: auto; width: 50%; height: auto; } |
| 35 |
div.calibre-nuked-tag-figure { font-size: small; text-align: center; } |
| 36 |
p.Single__lead, p.Longread__lead { font-style: italic; color:#202020; } |
| 37 |
p.article-metadata { color: gray; font-size:small; } |
| 38 |
''' |
| 39 |
|
| 40 |
remove_tags = [ |
| 41 |
dict(name='div', attrs={'class': 'Paywall'}), |
| 42 |
] |
| 43 |
|
| 44 |
keep_only_tags = [ |
| 45 |
dict(name='h1', class_='Single__title'), # Title of "Single" type articles. |
| 46 |
dict(name='h1', class_='Longread__title'), # Title of "Longread" type articles. |
| 47 |
dict(name='p', class_='Single__lead'), # Lead text of "Single" type articles. |
| 48 |
dict(name='p', class_='Longread__lead'), # Lead text of "Longread" type articles. |
| 49 |
dict(name='p', class_='article-metadata'), # Dynamically created by the recipe. |
| 50 |
dict(name='figure', class_='Single__thumbnail'), # Image of "Single" type articles. |
| 51 |
dict(name='figure', class_='Longread__thumbnail'), # Image of "Longread" type articles. |
| 52 |
dict(name='div', class_='Single__content'), # Article body of "Single" type articles. |
| 53 |
dict(name='div', class_='Longread__content'), # Article body of "Longread" type articles. |
| 54 |
] |
| 55 |
|
| 56 |
def extract_cover_url(self, a_tag) -> str: |
| 57 |
"""Given the <a> tag of the current edition, extract the URL of the highest-resolution cover image.""" |
| 58 |
figure_tag = a_tag.find('figure') |
| 59 |
img_tag = figure_tag.find('img') |
| 60 |
|
| 61 |
try: |
| 62 |
# The `srcset` attribute contains a comma-separated list of URLs and their resolutions. |
| 63 |
cover_urls = img_tag['srcset'].split(', ') |
| 64 |
cover_urls = [src.split(' ') for src in cover_urls] |
| 65 |
|
| 66 |
# The second item of each tuple should be the resolution, e.g., '578w' or '821w'. Remove the 'w' suffix, cast |
| 67 |
# to an integer and sort in descending order. |
| 68 |
cover_urls = [(url, int(resolution[:-1])) for url, resolution in cover_urls] |
| 69 |
cover_urls = sorted(cover_urls, key=lambda x: x[1], reverse=True) |
| 70 |
|
| 71 |
# The first item of the sorted list is now the URL of the highest-resolution image. |
| 72 |
self.cover_url = cover_urls[0][0] |
| 73 |
self.log(f"Identified cover URL: '{self.cover_url}'") |
| 74 |
except (KeyError, ValueError) as exc: |
| 75 |
self.log.error(f'Failed to extract cover URL! Has the website format changed?\n{exc}') |
| 76 |
|
| 77 |
return |
| 78 |
|
| 79 |
def get_current_edition_url(self) -> str: |
| 80 |
"""Return the URL of the current (weekly) edition of Fokus.""" |
| 81 |
current_year = datetime.now().year |
| 82 |
current_week = datetime.now().isocalendar().week |
| 83 |
try: |
| 84 |
soup = self.index_to_soup(f'{self.main_url}/vara-utgavor') |
| 85 |
|
| 86 |
# Identify all <a> tags of class 'Issue' that have an href attribute. |
| 87 |
a_tags = soup.find_all('a', class_='Issue', href=True) |
| 88 |
|
| 89 |
# Keep only the href, and subset to only those links that contain the current year. |
| 90 |
edition_links = [a_tag['href'] for a_tag in a_tags if str(current_year) in a_tag['href']] |
| 91 |
|
| 92 |
# To successfully sort the links chronologically, first convert the data structure to a dict, wherein the |
| 93 |
# key consists of only the date part of the URL and the value consists of the entire (unmodified) URL. |
| 94 |
edition_links = {link.removesuffix('/').split('/')[-1]: link for link in edition_links} |
| 95 |
|
| 96 |
# Shorten the key by removing the year prefix. |
| 97 |
edition_links = {key.split('-', 1)[-1]: value for key, value in edition_links.items()} |
| 98 |
|
| 99 |
# Most keys in the `edition_links` dict now correspond to calendar week numbers, e.g., '1', '02', '21' or |
| 100 |
# '52'. Note however that editions can sometimes cover multiples weeks, e.g., '1-2', '01-03', '50-51-52', |
| 101 |
# '26/27' or '50/51/52'. For entries where this is the case, split the entry into multiple entries. |
| 102 |
_edition_links = {} |
| 103 |
for key, value in edition_links.items(): |
| 104 |
# Handle both '-' and '/' as week number separators. |
| 105 |
if '-' in key: |
| 106 |
week_numbers = key.split('-') |
| 107 |
elif '/' in key: |
| 108 |
week_numbers = key.split('/') |
| 109 |
else: |
| 110 |
week_numbers = [key] |
| 111 |
for week_number in week_numbers: |
| 112 |
_edition_links[week_number] = value |
| 113 |
edition_links = _edition_links |
| 114 |
|
| 115 |
# Convert the resulting keys to integers. |
| 116 |
edition_links = {int(key): value for key, value in edition_links.items()} |
| 117 |
|
| 118 |
if current_week in edition_links: |
| 119 |
# If there exists an entry that corresponds to the `current_week`, use that entry. |
| 120 |
current_edition_url = edition_links[current_week] |
| 121 |
else: |
| 122 |
# Otherwise, use the entry that corresponds to the highest week number. |
| 123 |
max_week_number = max(edition_links.keys()) |
| 124 |
current_edition_url = edition_links[max_week_number] |
| 125 |
|
| 126 |
self.log(f"Identified {len(edition_links)} editions, of which the most recent is '{current_edition_url}'.") |
| 127 |
|
| 128 |
# Now that we know the URL of the current edition, we can use it to identify the cover image. The cover |
| 129 |
# image URL exists in the `src` attribute of the <img> child tag of the <figure> child tag of the <a> tag |
| 130 |
# of the current edition. |
| 131 |
current_edition_a_tag = soup.find('a', class_='Issue', href=current_edition_url) |
| 132 |
self.extract_cover_url(current_edition_a_tag) |
| 133 |
except Exception as exc: |
| 134 |
self.log.error(f'Failed to identify the current edition URL: {exc}') |
| 135 |
raise NoArticles( |
| 136 |
f"Could not find the URL of the current edition. Either the '{self.main_url}' server is experiencing " |
| 137 |
'issues, in which case you should try again later, or the website format has changed and the recipe ' |
| 138 |
'needs updating.' |
| 139 |
) from exc |
| 140 |
return current_edition_url |
| 141 |
|
| 142 |
def parse_article_blurb(self, article_blurb) -> dict[str, str, str, str, str] | None: |
| 143 |
"""Given a <article> tag of class 'Blurb', parse it into a dict. |
| 144 |
|
| 145 |
Args: |
| 146 |
article_blurb (Tag): An <article> tag hosting metadata and the URL of an article. |
| 147 |
|
| 148 |
Returns: |
| 149 |
A dict on a `{'url': str, 'title': str, 'date': str, 'category': str, 'description': str}` format. |
| 150 |
""" |
| 151 |
if a_tag := article_blurb.find('a', href=True): |
| 152 |
url = a_tag['href'].strip().rstrip('/') |
| 153 |
if url.startswith('/'): |
| 154 |
url = f'{self.main_url}{url}' |
| 155 |
|
| 156 |
if title_tag := a_tag.find('h2', {'class': 'Blurb__title'}): |
| 157 |
title = self.tag_to_string(title_tag).strip() |
| 158 |
|
| 159 |
if date_tag := a_tag.find('time', {'class': 'Blurb__date'}): |
| 160 |
# Results in a Swedish date format, e.g., '23 MAJ 2025'. |
| 161 |
date = self.tag_to_string(date_tag).strip() |
| 162 |
# Add a newline before the date to make it more readable. |
| 163 |
date = f'\n{date}' |
| 164 |
|
| 165 |
# Assign the article to its first listed category as inferred from the first <li> tag of class |
| 166 |
# 'Blurb__category'. Default to 'Fokus' if no such tag is found. |
| 167 |
category = 'Fokus' |
| 168 |
if category_tag := a_tag.find('li', {'class': 'Blurb__category'}): |
| 169 |
category = self.tag_to_string(category_tag).strip() |
| 170 |
|
| 171 |
desc = '' |
| 172 |
if desc_tag := a_tag.find('div', {'class': 'Blurb__summary'}): |
| 173 |
desc = self.tag_to_string(desc_tag).strip() |
| 174 |
|
| 175 |
return {'url': url, 'title': title, 'date': date, 'category': category, 'description': desc} |
| 176 |
|
| 177 |
return |
| 178 |
|
| 179 |
def get_article_blurbs(self, soup) -> dict[str, dict[str, str, str, str, str]]: |
| 180 |
"""Given a Fokus webpage `soup`, return a dict of unique article entries found on the page. |
| 181 |
|
| 182 |
The key of a given entry in the output dictionary is the article URL. The corresponding value is a dictionary |
| 183 |
on a `{'url': str, 'title': str, 'date': str, 'category': str, 'description': str}` format. |
| 184 |
|
| 185 |
Args: |
| 186 |
soup (BeautifulSoup): The `bs4.BeautifulSoup` soup of a Fokus webpage. |
| 187 |
|
| 188 |
Returns: |
| 189 |
dict[str, dict[str, str, str, str, str]]: A dict with article URLs as keys and 'article dicts' as values. |
| 190 |
""" |
| 191 |
|
| 192 |
def _log(article) -> None: |
| 193 |
"""Log a digestible summary of the input `article` blurb.""" |
| 194 |
log_message = f"\t{article['title']} : {article['url']}" |
| 195 |
if article.get('description'): |
| 196 |
log_message += f" : {article['description']}" |
| 197 |
self.log.debug(log_message) |
| 198 |
|
| 199 |
# Identify all <article> tags of class 'Blurb' that have an href attribute. |
| 200 |
self.log('Identifying all articles...') |
| 201 |
try: |
| 202 |
article_blurbs = soup.find_all('article', {'class': 'Blurb'}) |
| 203 |
except AttributeError: |
| 204 |
article_blurbs = [] |
| 205 |
|
| 206 |
if not article_blurbs: |
| 207 |
self.log.error('Failed to identify any article blurbs.') |
| 208 |
return {} |
| 209 |
|
| 210 |
parsed_blurbs = {} |
| 211 |
for article_blurb in article_blurbs: |
| 212 |
if article := self.parse_article_blurb(article_blurb): |
| 213 |
_log(article) |
| 214 |
parsed_blurbs[article['url']] = article |
| 215 |
|
| 216 |
return parsed_blurbs |
| 217 |
|
| 218 |
def convert_to_section_lists( |
| 219 |
self, |
| 220 |
articles: dict[str, dict[str, str, str, str, str]], |
| 221 |
) -> dict[str, list[dict[str, str, str, str]]]: |
| 222 |
"""Convert the `articles` dict of dicts to a dict of lists; each list holds the articles of a given section. |
| 223 |
|
| 224 |
Args: |
| 225 |
articles (dict[str, dict[str, str, str, str, str]]): A dict of article URLs and article dicts. |
| 226 |
|
| 227 |
Returns: |
| 228 |
dict[str, list[dict[str, str, str, str]]]: A dict on a `{section_title: list[article_dict]}` format. |
| 229 |
""" |
| 230 |
self.log(f'Assigning each of the {len(articles)} articles to a section...') |
| 231 |
section_to_articles = {} |
| 232 |
for _, article_dict in articles.items(): |
| 233 |
section_title = article_dict['category'] |
| 234 |
if section_title not in section_to_articles: |
| 235 |
section_to_articles[section_title] = [] |
| 236 |
# Remove the 'category' key from the article dict, as it is not needed in the final output. |
| 237 |
article_dict.pop('category') |
| 238 |
section_to_articles[section_title].append(article_dict) |
| 239 |
|
| 240 |
return section_to_articles |
| 241 |
|
| 242 |
def parse_index(self): |
| 243 |
current_edition_url = self.get_current_edition_url() |
| 244 |
if not current_edition_url: |
| 245 |
raise NoArticles( |
| 246 |
f"Could not find the URL of the current edition. Either the '{self.main_url}' server is experiencing " |
| 247 |
'issues, in which case you should try again later, or the website format has changed and the recipe ' |
| 248 |
'needs updating.' |
| 249 |
) |
| 250 |
self.log(f'Current edition URL: {current_edition_url}') |
| 251 |
|
| 252 |
# Identify all sections in the web version of Fokus. |
| 253 |
edition_soup = self.index_to_soup(current_edition_url) |
| 254 |
|
| 255 |
# From the section URLs and the main URL, identify all unique articles. |
| 256 |
articles = self.get_article_blurbs(edition_soup) |
| 257 |
if not articles: |
| 258 |
raise NoArticles( |
| 259 |
f"Could not find any articles. Either the '{self.main_url}' server is experiencing issues, in which " |
| 260 |
'case you should try again later, or the website format has changed and the recipe needs updating.' |
| 261 |
) |
| 262 |
|
| 263 |
# Assign each identified article to a section based on its URL. |
| 264 |
section_to_articles = self.convert_to_section_lists(articles) |
| 265 |
|
| 266 |
# Convert to tuples. |
| 267 |
return list(section_to_articles.items()) |