| 1 |
#!/usr/bin/env python |
| 2 |
# vim:fileencoding=utf-8 |
| 3 |
# License: GPLv3 Copyright: 2026, Ben Ng <ben at tangentialtechnologies.com> |
| 4 |
""" |
| 5 |
The Golfer's Journal (https://www.golfersjournal.com) |
| 6 |
|
| 7 |
A subscriber-only quarterly. The recipe logs in with the reader's own |
| 8 |
WooCommerce account (calibre asks for the credentials when scheduling the |
| 9 |
download) and builds one e-book per printed issue from the issue's table of |
| 10 |
contents. |
| 11 |
""" |
| 12 |
|
| 13 |
import re |
| 14 |
from contextlib import closing |
| 15 |
from io import BytesIO |
| 16 |
|
| 17 |
from calibre.web.feeds.news import BasicNewsRecipe, classes |
| 18 |
|
| 19 |
|
| 20 |
class GolfersJournal(BasicNewsRecipe): |
| 21 |
title = "The Golfer's Journal" |
| 22 |
__author__ = 'Ben Ng' |
| 23 |
description = ( |
| 24 |
'A subscriber-only quarterly of golf writing, photography and art. ' |
| 25 |
'Downloads a single printed issue (default: the newest) from your ' |
| 26 |
"Golfer's Journal account. Requires an active subscription." |
| 27 |
) |
| 28 |
publisher = "The Golfer's Journal" |
| 29 |
category = 'golf, sports, magazine' |
| 30 |
language = 'en' |
| 31 |
encoding = 'utf-8' |
| 32 |
|
| 33 |
no_stylesheets = True |
| 34 |
remove_javascript = True |
| 35 |
remove_empty_feeds = True |
| 36 |
# Deliberately not compressing images: this is a photography magazine and the |
| 37 |
# site serves full-resolution originals. Let the output profile handle any |
| 38 |
# device scaling instead of recompressing here. |
| 39 |
compress_news_images = False |
| 40 |
ignore_duplicate_articles = {'url'} |
| 41 |
|
| 42 |
# Emit an EPUB 3 navigation document (nav.xhtml): the default ncx-only EPUB 2 |
| 43 |
# output leaves crengine based readers (KOReader) without a usable table of |
| 44 |
# contents for this book. Ignored for non-EPUB output formats. |
| 45 |
conversion_options = { |
| 46 |
'epub_version': '3', |
| 47 |
# Scale the cover proportionally rather than stretching it to fill the |
| 48 |
# screen, which warps these covers noticeably. |
| 49 |
'preserve_cover_aspect_ratio': True, |
| 50 |
} |
| 51 |
|
| 52 |
needs_subscription = True |
| 53 |
|
| 54 |
BASE = 'https://www.golfersjournal.com' |
| 55 |
LOGIN = BASE + '/my-account/' |
| 56 |
|
| 57 |
# Every cover is emitted at this size. Some issues publish their cover only |
| 58 |
# as a photograph of the printed magazine lying open -- back and front side |
| 59 |
# by side -- so without normalizing, the covers have wildly different |
| 60 |
# aspect ratios in the calibre grid. 1000x1214 is the aspect ratio of the |
| 61 |
# publisher's own front-cover asset (925x1123). |
| 62 |
COVER_W, COVER_H = 1000, 1214 |
| 63 |
|
| 64 |
# Download a specific issue instead of the newest one. |
| 65 |
recipe_specific_options = { |
| 66 |
'issue': { |
| 67 |
'short': 'The issue number to download', |
| 68 |
'long': 'For example, 36. Defaults to the newest issue.', |
| 69 |
} |
| 70 |
} |
| 71 |
|
| 72 |
# The whole feature -- headline, dek, byline, hero image and body -- is |
| 73 |
# wrapped in a single <article> element, so keeping it discards the site |
| 74 |
# chrome for free. |
| 75 |
keep_only_tags = [ |
| 76 |
dict(name='article'), |
| 77 |
] |
| 78 |
remove_tags = [ |
| 79 |
# For subscribers the site injects a download/share widget at the top of |
| 80 |
# every article: an ePub/Kindle/Instapaper bar and a hidden "Send to |
| 81 |
# Kindle" modal. It is all namespaced with epub-* classes and ids, so |
| 82 |
# dropping anything epub-* removes the widget and nothing else. |
| 83 |
dict(attrs={'class': lambda x: x and any((c or '').startswith('epub-') for c in (x if isinstance(x, (list, tuple)) else x.split()))}), |
| 84 |
dict(attrs={'id': lambda x: x and x.startswith('epub-')}), |
| 85 |
classes( |
| 86 |
'sign-in tgj-login-block sign-in-only paywall subscribe-block ' |
| 87 |
'sharedaddy sd-sharing-enabled jp-relatedposts related-posts ' |
| 88 |
'newsletter-signup newsletter comments-area comment-respond ' |
| 89 |
'nav-links post-navigation author-box wp-block-buttons ' |
| 90 |
'yardage-book-download entry-footer ' |
| 91 |
# the category/department link (e.g. "Adventures") above the headline |
| 92 |
'department ' |
| 93 |
# the "Light / Dark" theme toggle in the byline block |
| 94 |
'light-dark ' |
| 95 |
# audio readings ("Listen to a reading of this feature...") - the |
| 96 |
# player does not work in an e-book, so drop the whole figure |
| 97 |
'wp-block-audio ' |
| 98 |
# photo gallery chrome: the "+ Click to enlarge" hint and the |
| 99 |
# numeric slide counters shown under each gallery image |
| 100 |
'hint index' |
| 101 |
), |
| 102 |
dict(name=['script', 'style', 'noscript', 'iframe', 'form', 'button', 'svg']), |
| 103 |
dict(attrs={'role': 'dialog'}), |
| 104 |
] |
| 105 |
remove_attributes = ['style', 'width', 'height', 'srcset', 'sizes'] |
| 106 |
|
| 107 |
extra_css = ''' |
| 108 |
.articleSection, .article-section { text-transform: uppercase; font-size: 0.7em; letter-spacing: 0.1em; color: #666; } |
| 109 |
.subtitle, .dek { font-style: italic; color: #333; } |
| 110 |
.byline, .credit, figcaption, .caption, .wp-caption-text { font-size: 0.75em; color: #555; } |
| 111 |
img { display: block; margin: 0.5em auto; max-width: 100%; height: auto; } |
| 112 |
figure { margin: 0.5em 0; } |
| 113 |
blockquote { font-style: italic; color: #333; } |
| 114 |
''' |
| 115 |
|
| 116 |
def get_browser(self, *args, **kwargs): |
| 117 |
br = BasicNewsRecipe.get_browser(self, *args, **kwargs) |
| 118 |
if self.username and self.password: |
| 119 |
br.open(self.LOGIN) |
| 120 |
|
| 121 |
def is_login_form(form): |
| 122 |
names = {c.name for c in form.controls} |
| 123 |
return 'username' in names and 'woocommerce-login-nonce' in names |
| 124 |
|
| 125 |
br.select_form(predicate=is_login_form) |
| 126 |
br['username'] = self.username |
| 127 |
br['password'] = self.password |
| 128 |
res = br.submit() |
| 129 |
html = res.read().decode('utf-8', 'replace') |
| 130 |
if 'customer-logout' not in html and 'woocommerce-MyAccount' not in html: |
| 131 |
hint = '' |
| 132 |
if re.search(r'wordfence|verification code|two-factor|2fa', html, re.I): |
| 133 |
hint = ' The site asked for two-factor / Wordfence verification, which cannot be answered automatically.' |
| 134 |
raise ValueError("Failed to log in to The Golfer's Journal. Check your username and password." + hint) |
| 135 |
return br |
| 136 |
|
| 137 |
def _newest_issue(self): |
| 138 |
soup = self.index_to_soup(self.BASE + '/issues/') |
| 139 |
nums = set() |
| 140 |
for a in soup.findAll('a', href=True): |
| 141 |
m = re.search(r'/issues/no-(\d+)/', a['href']) |
| 142 |
if m: |
| 143 |
nums.add(int(m.group(1))) |
| 144 |
if not nums: |
| 145 |
raise ValueError('Could not find any issues on ' + self.BASE + '/issues/') |
| 146 |
return max(nums) |
| 147 |
|
| 148 |
def parse_index(self): |
| 149 |
opt = self.recipe_specific_options.get('issue') |
| 150 |
if opt and isinstance(opt, str) and opt.strip().isdigit(): |
| 151 |
num = int(opt.strip()) |
| 152 |
else: |
| 153 |
num = self._newest_issue() |
| 154 |
|
| 155 |
issue_url = '%s/issues/no-%d/' % (self.BASE, num) |
| 156 |
soup = self.index_to_soup(issue_url) |
| 157 |
|
| 158 |
self.title = "The Golfer's Journal No. %d" % num |
| 159 |
self.issue_number = num |
| 160 |
self._cover_candidates = self._find_cover_candidates(soup, num) |
| 161 |
|
| 162 |
toc = soup.find(attrs={'class': lambda x: x and 'toc-block' in x}) |
| 163 |
if toc is None: |
| 164 |
toc = soup.find('div', id='toc') |
| 165 |
|
| 166 |
articles = [] |
| 167 |
seen = set() |
| 168 |
containers = toc.findAll('div', attrs={'class': 'feature'}) if toc else [] |
| 169 |
for feat in containers: |
| 170 |
a = feat.find('a', href=re.compile(r'/editorial/')) |
| 171 |
if a is None: |
| 172 |
continue |
| 173 |
url = a['href'].split('#')[0].split('?')[0] |
| 174 |
if url in seen: |
| 175 |
continue |
| 176 |
seen.add(url) |
| 177 |
title = self.tag_to_string(a).strip() |
| 178 |
sub = feat.find(attrs={'class': 'subtitle'}) |
| 179 |
desc = self.tag_to_string(sub).strip() if sub else '' |
| 180 |
articles.append({'title': title, 'url': url, 'description': desc}) |
| 181 |
self.log('\t', title, url) |
| 182 |
|
| 183 |
if not articles: |
| 184 |
# Older issues (roughly No. 26 and earlier) use a different layout: |
| 185 |
# no toc-block; instead each article is an <a href> wrapping a |
| 186 |
# div.feature-story with the headline in <h1> and the dek in <h3>. |
| 187 |
for feat in soup.findAll(**classes('feature-story')): |
| 188 |
url = '' |
| 189 |
meta = feat.find('meta', attrs={'itemprop': 'url', 'content': True}) |
| 190 |
if meta is not None: |
| 191 |
url = meta['content'] |
| 192 |
else: |
| 193 |
a = feat.findParent('a', href=True) |
| 194 |
if a is not None: |
| 195 |
url = a['href'] |
| 196 |
if '/editorial/' not in url: |
| 197 |
continue |
| 198 |
url = url.split('#')[0].split('?')[0] |
| 199 |
if url in seen: |
| 200 |
continue |
| 201 |
seen.add(url) |
| 202 |
h = feat.find(['h1', 'h2']) |
| 203 |
title = self.tag_to_string(h).strip() if h else url |
| 204 |
sub = feat.find('h3') |
| 205 |
desc = self.tag_to_string(sub).strip() if sub else '' |
| 206 |
articles.append({'title': title, 'url': url, 'description': desc}) |
| 207 |
self.log('\t', title, url) |
| 208 |
|
| 209 |
if not articles and containers: |
| 210 |
# A few issues (Nos. 27-28) have a table of contents with no links |
| 211 |
# at all, just titles. The articles do exist online at the WordPress |
| 212 |
# slug of the title, so construct each URL and keep it if it resolves. |
| 213 |
br = self.browser |
| 214 |
for feat in containers: |
| 215 |
h = feat.find('h3') |
| 216 |
if h is None: |
| 217 |
continue |
| 218 |
title = self.tag_to_string(h).strip() |
| 219 |
slug = re.sub(r'[^a-z0-9]+', '-', title.lower().replace("'", '').replace('’', '')).strip('-') |
| 220 |
if not slug: |
| 221 |
continue |
| 222 |
url = '%s/editorial/%s/' % (self.BASE, slug) |
| 223 |
if url in seen: |
| 224 |
continue |
| 225 |
try: |
| 226 |
br.open(url, timeout=self.timeout).close() |
| 227 |
except Exception: |
| 228 |
self.log.warn('No online version found for: ' + title) |
| 229 |
continue |
| 230 |
seen.add(url) |
| 231 |
sub = feat.find(attrs={'class': 'subtitle'}) |
| 232 |
desc = self.tag_to_string(sub).strip() if sub else '' |
| 233 |
articles.append({'title': title, 'url': url, 'description': desc}) |
| 234 |
self.log('\t', title, url) |
| 235 |
|
| 236 |
if not articles: |
| 237 |
raise ValueError('No articles found for issue No. %d' % num) |
| 238 |
|
| 239 |
return [("The Golfer's Journal No. %d" % num, articles)] |
| 240 |
|
| 241 |
@staticmethod |
| 242 |
def _unsized(url): |
| 243 |
# WordPress appends -WIDTHxHEIGHT before the extension for the scaled |
| 244 |
# copies it generates. Strip it to get the full resolution original. |
| 245 |
return re.sub(r'-\d+x\d+(\.\w+)$', r'\1', url.strip()) |
| 246 |
|
| 247 |
def _find_cover_candidates(self, soup, num): |
| 248 |
"""URLs that might be the cover, best first. |
| 249 |
|
| 250 |
Most issues publish a front-cover-only asset somewhere on the issue |
| 251 |
page even when og:image points at a back+front spread, so look for that |
| 252 |
first and fall back to og:image (which is cropped later if need be). |
| 253 |
""" |
| 254 |
urls, seen = [], set() |
| 255 |
|
| 256 |
def add(url): |
| 257 |
if not url: |
| 258 |
return |
| 259 |
url = self._unsized(url) |
| 260 |
if url.startswith('//'): |
| 261 |
url = 'https:' + url |
| 262 |
if url.startswith('http') and url not in seen: |
| 263 |
seen.add(url) |
| 264 |
urls.append(url) |
| 265 |
|
| 266 |
# 1. Newer issue pages wrap the front cover in its own block. |
| 267 |
block = soup.find(attrs={'class': lambda x: x and 'cover-image-block' in x}) |
| 268 |
if block is not None: |
| 269 |
for img in block.findAll('img'): |
| 270 |
for attr in ('data-lazy-src', 'data-src', 'src'): |
| 271 |
val = img.get(attr) |
| 272 |
if val and not val.startswith('data:'): |
| 273 |
add(val) |
| 274 |
break |
| 275 |
|
| 276 |
# 2. Otherwise the front cover is uploaded under a predictable name, |
| 277 |
# e.g. No36-FRONT-925w-CI-Page.webp or TGJ-1-4-Cover-925w.jpg. Names |
| 278 |
# saying BACK/SIDE-BY-SIDE/SPREAD are the two-page spread, not this. |
| 279 |
front = re.compile(r'/[^/]*(?:no\.?-?%d|tgj-(?:1-)?%d)[-_.][^/]*(?:front|cover)[^/]*\.(?:jpe?g|png|webp)$' % (num, num), re.I) |
| 280 |
spread = re.compile(r'back|side-by-side|spread|placeholder', re.I) |
| 281 |
for url in sorted(set(re.findall(r'https://[^"\'\s]+?\.(?:jpe?g|png|webp)', str(soup)))): |
| 282 |
name = url.rpartition('/')[2] |
| 283 |
if front.search('/' + name) and not spread.search(name): |
| 284 |
add(url) |
| 285 |
|
| 286 |
# 3. Last resort: whatever the page advertises to social networks. |
| 287 |
og = soup.find('meta', attrs={'property': 'og:image', 'content': True}) |
| 288 |
if og is not None: |
| 289 |
self.cover_url = og['content'] |
| 290 |
add(og['content']) |
| 291 |
return urls |
| 292 |
|
| 293 |
def _crop_front_cover(self, img): |
| 294 |
"""Cut the front cover out of a photograph of a back+front spread. |
| 295 |
|
| 296 |
The two panels are equal width with the front cover on the left (the |
| 297 |
one carrying the masthead), photographed on a plain but often |
| 298 |
gradient-lit background. |
| 299 |
""" |
| 300 |
from qt.core import QImage, QRect, Qt |
| 301 |
|
| 302 |
aw = 360 |
| 303 |
small = img.scaledToWidth(aw, Qt.TransformationMode.SmoothTransformation) |
| 304 |
small = small.convertToFormat(QImage.Format.Format_RGB32) |
| 305 |
w, h = small.width(), small.height() |
| 306 |
grid = [] |
| 307 |
for y in range(h): |
| 308 |
row = [] |
| 309 |
for x in range(w): |
| 310 |
v = small.pixel(x, y) |
| 311 |
row.append(((v >> 16) & 0xFF, (v >> 8) & 0xFF, v & 0xFF)) |
| 312 |
grid.append(row) |
| 313 |
|
| 314 |
thr = 34 |
| 315 |
|
| 316 |
def differs(a, b): |
| 317 |
return abs(a[0] - b[0]) + abs(a[1] - b[1]) + abs(a[2] - b[2]) > thr |
| 318 |
|
| 319 |
# Count, per column, the pixels unlike that column's own top and bottom |
| 320 |
# pixel. Comparing within a column rather than against one global |
| 321 |
# background colour tolerates the gradient backgrounds. |
| 322 |
colmass = [sum(1 for y in range(h) if differs(grid[y][x], grid[0][x]) and differs(grid[y][x], grid[h - 1][x])) for x in range(w)] |
| 323 |
cmin = max(2, int(0.12 * h)) |
| 324 |
cols = [x for x in range(w) if colmass[x] > cmin] |
| 325 |
if not cols: |
| 326 |
self.log.warn('Could not locate the covers in the spread; using it as is') |
| 327 |
return img |
| 328 |
lo, hi = cols[0], cols[-1] |
| 329 |
span = hi - lo + 1 |
| 330 |
|
| 331 |
gaps, x = [], lo |
| 332 |
while x <= hi: |
| 333 |
if colmass[x] <= cmin: |
| 334 |
start = x |
| 335 |
while x <= hi and colmass[x] <= cmin: |
| 336 |
x += 1 |
| 337 |
if x - start >= max(2, int(0.008 * w)): |
| 338 |
gaps.append((start, x)) |
| 339 |
else: |
| 340 |
x += 1 |
| 341 |
# The panels are equal width, so the gutter sits at the middle of the |
| 342 |
# span. Pick the gap nearest to it rather than the widest one: a flat |
| 343 |
# area inside a cover can read as a wider gap than the gutter itself. |
| 344 |
best = None |
| 345 |
for start, end in gaps: |
| 346 |
off = abs(((start + end) / 2.0 - lo) / span - 0.5) |
| 347 |
if off < 0.15 and (best is None or off < best[0]): |
| 348 |
best = (off, start) |
| 349 |
x0, x1 = (lo, best[1] - 1) if best is not None else (lo, lo + span // 2) |
| 350 |
|
| 351 |
# Vertical extent measured only within the panel that was chosen. |
| 352 |
rmin = max(2, int(0.12 * (x1 - x0 + 1))) |
| 353 |
ys = [y for y in range(h) if sum(1 for x in range(x0, x1 + 1) if differs(grid[y][x], grid[y][x0]) and differs(grid[y][x], grid[y][x1])) > rmin] |
| 354 |
y0, y1 = (ys[0], ys[-1]) if ys else (0, h - 1) |
| 355 |
|
| 356 |
sx, sy = img.width() / w, img.height() / h |
| 357 |
return img.copy(QRect(int(x0 * sx), int(y0 * sy), max(1, int((x1 - x0 + 1) * sx)), max(1, int((y1 - y0 + 1) * sy)))) |
| 358 |
|
| 359 |
def _normalize_cover(self, img): |
| 360 |
from qt.core import QRect, Qt |
| 361 |
|
| 362 |
w, h = img.width(), img.height() |
| 363 |
target = self.COVER_W / self.COVER_H |
| 364 |
if w / h > target: |
| 365 |
nw = round(h * target) |
| 366 |
img = img.copy(QRect((w - nw) // 2, 0, nw, h)) |
| 367 |
else: |
| 368 |
nh = round(w / target) |
| 369 |
img = img.copy(QRect(0, (h - nh) // 2, w, nh)) |
| 370 |
return img.scaled(self.COVER_W, self.COVER_H, Qt.AspectRatioMode.IgnoreAspectRatio, Qt.TransformationMode.SmoothTransformation) |
| 371 |
|
| 372 |
def _stamp_issue_number(self, img, num): |
| 373 |
"""Overlay the issue number in a black disc in the lower right corner. |
| 374 |
|
| 375 |
The covers are photographs and carry the issue number only in small |
| 376 |
print, if at all, so this is what makes an issue identifiable in the |
| 377 |
calibre cover grid. |
| 378 |
""" |
| 379 |
from qt.core import QBrush, QColor, QFont, QFontMetrics, QImage, QPainter, Qt |
| 380 |
|
| 381 |
from calibre.ebooks.covers import init_environment |
| 382 |
|
| 383 |
init_environment() # a Qt app and the bundled fonts, needed to draw text |
| 384 |
img = img.convertToFormat(QImage.Format.Format_RGB32) |
| 385 |
w, h = img.width(), img.height() |
| 386 |
diameter = int(0.25 * w) |
| 387 |
margin = int(0.045 * w) |
| 388 |
cx, cy = w - margin - diameter // 2, h - margin - diameter // 2 |
| 389 |
|
| 390 |
text = str(num) |
| 391 |
font = QFont('Liberation Serif') |
| 392 |
font.setBold(True) |
| 393 |
font.setStyleStrategy(QFont.StyleStrategy.PreferAntialias) |
| 394 |
size = int(diameter * 0.66) |
| 395 |
font.setPixelSize(size) |
| 396 |
fm = QFontMetrics(font) |
| 397 |
while fm.horizontalAdvance(text) > int(diameter * 0.68) and size > 8: |
| 398 |
size -= 2 |
| 399 |
font.setPixelSize(size) |
| 400 |
fm = QFontMetrics(font) |
| 401 |
|
| 402 |
p = QPainter(img) |
| 403 |
try: |
| 404 |
p.setRenderHints(QPainter.RenderHint.Antialiasing | QPainter.RenderHint.TextAntialiasing) |
| 405 |
p.setPen(Qt.PenStyle.NoPen) |
| 406 |
p.setBrush(QBrush(QColor(0, 0, 0))) |
| 407 |
p.drawEllipse(cx - diameter // 2, cy - diameter // 2, diameter, diameter) |
| 408 |
p.setFont(font) |
| 409 |
p.setPen(QColor(255, 255, 255)) |
| 410 |
rect = fm.tightBoundingRect(text) |
| 411 |
p.drawText(cx - rect.width() // 2 - rect.x(), cy + rect.height() // 2, text) |
| 412 |
finally: |
| 413 |
p.end() |
| 414 |
return img |
| 415 |
|
| 416 |
def get_cover_url(self): |
| 417 |
from calibre.utils.img import image_from_data, image_to_data |
| 418 |
|
| 419 |
candidates = list(getattr(self, '_cover_candidates', ())) |
| 420 |
if not candidates and getattr(self, 'cover_url', None): |
| 421 |
candidates = [self.cover_url] |
| 422 |
front, spread = None, None |
| 423 |
for url in candidates: |
| 424 |
try: |
| 425 |
with closing(self.browser.open(url, timeout=self.timeout)) as r: |
| 426 |
img = image_from_data(r.read()) |
| 427 |
except Exception as err: |
| 428 |
self.log.warn('Could not fetch cover candidate %s: %s' % (url, err)) |
| 429 |
continue |
| 430 |
if not img.width() or not img.height(): |
| 431 |
continue |
| 432 |
if img.height() > img.width(): |
| 433 |
self.log('Using front cover image: ' + url) |
| 434 |
front = img |
| 435 |
break |
| 436 |
if spread is None: |
| 437 |
spread = (url, img) |
| 438 |
if front is None: |
| 439 |
if spread is None: |
| 440 |
self.log.warn('No usable cover image found') |
| 441 |
return None |
| 442 |
url, img = spread |
| 443 |
self.log('Only a back+front spread is published for this issue; cropping the front cover out of ' + url) |
| 444 |
front = self._crop_front_cover(img) |
| 445 |
img = self._normalize_cover(front) |
| 446 |
num = getattr(self, 'issue_number', None) |
| 447 |
if num is not None: |
| 448 |
img = self._stamp_issue_number(img, num) |
| 449 |
# _download_cover() reads file-like objects, which lets the cover be |
| 450 |
# handed over already cropped and stamped. |
| 451 |
return BytesIO(image_to_data(img, fmt='JPEG', compression_quality=92)) |
| 452 |
|
| 453 |
# Every article carries its credits in a div.byline near the headline, but |
| 454 |
# the markup varies a lot between issues: |
| 455 |
# <address class="author">Words by <span itemprop="author">Tom Coyne</span></address> |
| 456 |
# <address itemprop="author">Words by Caleb Hannan, photos curated by Chris Reynolds</address> |
| 457 |
# <address itemprop="author">Photos and captions by Ryan Young</address> |
| 458 |
# so the credit line is parsed into "<role> by <names>" clauses and the |
| 459 |
# writing credit is preferred over the photography one. Photo essays have |
| 460 |
# only the latter, and a few features (and the odd "Photos courtesy of ..." |
| 461 |
# archive piece) have no attributable person at all. |
| 462 |
_ROLE_BY = re.compile( |
| 463 |
r'\b(words|word|story|stories|text|writing|written|essay|interview|interviews|reporting|as told to|' |
| 464 |
r'photo|photos|photograph|photographs|photography|image|images|caption|captions|' |
| 465 |
r'illustration|illustrations|artwork|art|film|video)\b[^,;]{0,40}?\bby\b[:\s]*', |
| 466 |
re.I, |
| 467 |
) |
| 468 |
_WRITING_ROLE = re.compile(r'word|story|stories|text|writing|written|essay|interview|reporting|as told to', re.I) |
| 469 |
# Credits that name an archive, agency or the magazine itself rather than a person. |
| 470 |
_NOT_A_PERSON = re.compile( |
| 471 |
r'courtesy|unless otherwise|getty|archive|associated press|staff|' |
| 472 |
r'golfer.s journal|tgj|usga|lpga|pga|r&a', |
| 473 |
re.I, |
| 474 |
) |
| 475 |
|
| 476 |
@classmethod |
| 477 |
def _clean_byline_names(cls, text): |
| 478 |
"""A displayable person (or two) from the names half of a credit clause.""" |
| 479 |
text = re.sub(r'\s+', ' ', text or '').strip(' .,;:&/-·') |
| 480 |
if not text or len(text) > 60 or any(ch.isdigit() for ch in text): |
| 481 |
return '' |
| 482 |
if cls._NOT_A_PERSON.search(text): |
| 483 |
return '' |
| 484 |
names = [n.strip(' .,;:&/-') for n in re.split(r',|\s+&\s+|\s+and\s+', text)] |
| 485 |
names = [n for n in names if n and not cls._NOT_A_PERSON.search(n)] |
| 486 |
if not names: |
| 487 |
return '' |
| 488 |
# Keep the credit short enough to read in a TOC list on a small screen. |
| 489 |
return ' & '.join(names[:2]) |
| 490 |
|
| 491 |
@classmethod |
| 492 |
def _author_from_credit(cls, text): |
| 493 |
text = text or '' |
| 494 |
matches = list(cls._ROLE_BY.finditer(text)) |
| 495 |
if not matches: |
| 496 |
# No named role, so either a bare name ("Tom Coyne") or a one-off |
| 497 |
# credit such as "Excerpted from The Authentic Swing by Steven |
| 498 |
# Pressfield", where the name follows the last "by". |
| 499 |
parts = re.split(r'\bby\b', text, flags=re.I) |
| 500 |
return cls._clean_byline_names(parts[-1] if len(parts) > 1 else text) |
| 501 |
clauses = [] |
| 502 |
for i, m in enumerate(matches): |
| 503 |
end = matches[i + 1].start() if i + 1 < len(matches) else len(text) |
| 504 |
clauses.append((m.group(1), text[m.end() : end])) |
| 505 |
for want_writer in (True, False): |
| 506 |
for role, names in clauses: |
| 507 |
if want_writer and not cls._WRITING_ROLE.search(role): |
| 508 |
continue |
| 509 |
name = cls._clean_byline_names(names) |
| 510 |
if name: |
| 511 |
return name |
| 512 |
return '' |
| 513 |
|
| 514 |
def _article_author(self, soup): |
| 515 |
block = soup.find(attrs={'class': lambda x: x and 'byline' in (x if isinstance(x, (list, tuple)) else x.split())}) |
| 516 |
if block is None: |
| 517 |
return '' |
| 518 |
# itemprop="author" is sometimes the writer's name alone and sometimes |
| 519 |
# the whole credit line, so parse whatever it holds either way. |
| 520 |
node = block.find(attrs={'itemprop': 'author'}) or block |
| 521 |
return self._author_from_credit(self.tag_to_string(node)) or (self._author_from_credit(self.tag_to_string(block)) if node is not block else '') |
| 522 |
|
| 523 |
def populate_article_metadata(self, article, soup, first): |
| 524 |
# Put the writer in the table of contents ("Title - Author"): the |
| 525 |
# features are titled evocatively rather than descriptively, so the |
| 526 |
# byline is the most useful thing to identify them by. calibre builds |
| 527 |
# both the NCX and the EPUB 3 nav from these titles, so this lands in |
| 528 |
# both. The article page itself is left untouched. |
| 529 |
if not first: |
| 530 |
return |
| 531 |
author = self._article_author(soup) |
| 532 |
if not author: |
| 533 |
self.log('\tNo byline found for: ' + (article.title or '')) |
| 534 |
return |
| 535 |
article.author = author |
| 536 |
if article.title and ' · ' not in article.title: |
| 537 |
article.title = '%s · %s' % (article.title, author) |
| 538 |
|
| 539 |
def postprocess_book(self, oeb, opts, log): |
| 540 |
# Remove the inline navigation bars ('| Next | Section menu | ... |') |
| 541 |
# that calibre adds to the top and bottom of every article. The book |
| 542 |
# has a proper navigation document, so they are just visual clutter in |
| 543 |
# a photography-heavy magazine. |
| 544 |
for item in oeb.spine: |
| 545 |
root = getattr(item, 'data', None) |
| 546 |
if root is None or not hasattr(root, 'xpath'): |
| 547 |
continue |
| 548 |
for div in root.xpath('//*[contains(@class, "calibre_navbar")]'): |
| 549 |
div.getparent().remove(div) |
| 550 |
|
| 551 |
@staticmethod |
| 552 |
def _best_srcset(srcset): |
| 553 |
# srcset is either "url 1x, url 2x" or "url 640w, url 2560w, ...". |
| 554 |
# Return the URL with the highest resolution (largest w/x descriptor). |
| 555 |
best_url, best_score = None, -1.0 |
| 556 |
for cand in srcset.split(','): |
| 557 |
parts = cand.split() |
| 558 |
if not parts: |
| 559 |
continue |
| 560 |
url, score = parts[0], 1.0 |
| 561 |
if len(parts) > 1: |
| 562 |
d = parts[1].strip().lower() |
| 563 |
try: |
| 564 |
score = float(d[:-1]) if d[-1:] in ('w', 'x') else float(d) |
| 565 |
except ValueError: |
| 566 |
score = 1.0 |
| 567 |
if score > best_score: |
| 568 |
best_score, best_url = score, url |
| 569 |
return best_url |
| 570 |
|
| 571 |
def preprocess_html(self, soup): |
| 572 |
# A paywalled copy means the login did not take effect; say so in the |
| 573 |
# log rather than silently producing a stub article. |
| 574 |
if soup.find(string=re.compile(r'Sign in to read this feature')): |
| 575 |
self.log.warn('Article appears paywalled - login may have failed') |
| 576 |
# Unwrap <picture>: crengine (KOReader) support for it is spotty and can |
| 577 |
# mis-size the contained image. Replace each <picture> with its <img>. |
| 578 |
for pic in soup.findAll('picture'): |
| 579 |
img = pic.find('img') |
| 580 |
if img is not None: |
| 581 |
pic.replaceWith(img) |
| 582 |
for img in soup.findAll('img'): |
| 583 |
# Prefer the highest-resolution source: src holds a downscaled image |
| 584 |
# (e.g. ...-1280x855.webp) while srcset lists the full-size original. |
| 585 |
# Use the original so the photographs aren't needlessly shrunk. |
| 586 |
srcset = img.get('srcset') or img.get('data-srcset') |
| 587 |
if srcset: |
| 588 |
best = self._best_srcset(srcset) |
| 589 |
if best: |
| 590 |
img['src'] = best |
| 591 |
continue |
| 592 |
for attr in ('data-lazy-src', 'data-src'): |
| 593 |
if img.get(attr): |
| 594 |
img['src'] = img[attr] |
| 595 |
break |
| 596 |
return soup |