| 1 |
#!/usr/bin/env python |
| 2 |
|
| 3 |
import re |
| 4 |
from collections import defaultdict |
| 5 |
from datetime import date, timedelta |
| 6 |
|
| 7 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 8 |
|
| 9 |
DIR_COLLECTIONS = [ |
| 10 |
['world'], |
| 11 |
['nation'], |
| 12 |
['politics'], |
| 13 |
['opinion', 'op-ed', 'opinion-la', 'editorials', 'readersreact', 'topoftheticket', 'endorsements'], |
| 14 |
['local', 'lanow', 'california', 'crime', 'abcarian', 'education', 'weather'], |
| 15 |
['business', 'hollywood', 'technology'], |
| 16 |
['sports'], |
| 17 |
['entertainment', 'movies', 'music', 'tv', 'arts', 'gossip', 'envelope'], |
| 18 |
['books'], |
| 19 |
['food', 'jonathon-gold', 'dailydish'], |
| 20 |
['health'], |
| 21 |
['style', 'laaffairs', 'pets'], |
| 22 |
['science', 'sciencenow'], |
| 23 |
['home'], |
| 24 |
['travel'], |
| 25 |
['fashion'], |
| 26 |
] |
| 27 |
|
| 28 |
|
| 29 |
def classes(classes): |
| 30 |
q = frozenset(classes.split(' ')) |
| 31 |
return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)}) |
| 32 |
|
| 33 |
|
| 34 |
def absurl(url): |
| 35 |
if url.startswith('/'): |
| 36 |
url = 'https://www.latimes.com' + url |
| 37 |
return url |
| 38 |
|
| 39 |
|
| 40 |
def what_section(url): |
| 41 |
parts = url.split('/') |
| 42 |
return parts[-4].capitalize() |
| 43 |
|
| 44 |
|
| 45 |
class LATimes(BasicNewsRecipe): |
| 46 |
title = 'Los Angeles Times' |
| 47 |
__author__ = 'Jose Ortiz' |
| 48 |
description = 'The Los Angeles Times is a leading source of news on Southern California, entertainment, movies, television, music, politics, business, health, technology, travel, sports, environment, economics, autos, jobs, real estate and other topics affecting California' # noqa: E501 |
| 49 |
category = 'news, politics, USA, Los Angeles, world' |
| 50 |
oldest_article = 1 |
| 51 |
max_articles_per_feed = 200 |
| 52 |
no_stylesheets = True |
| 53 |
encoding = 'utf8' |
| 54 |
use_embedded_content = False |
| 55 |
compress_news_images = True |
| 56 |
compress_news_images_auto_size = 5 |
| 57 |
language = 'en_US' |
| 58 |
remove_empty_feeds = True |
| 59 |
ignore_duplicate_articles = {'url'} |
| 60 |
publication_type = 'newspaper' |
| 61 |
cover_url = 'http://www.latimes.com/includes/sectionfronts/A1.pdf' |
| 62 |
|
| 63 |
keep_only_tags = [ |
| 64 |
classes('headline page-lead-media authors published-date page-article-container'), |
| 65 |
dict(attrs={'data-element': 'story-body'}), |
| 66 |
] |
| 67 |
|
| 68 |
remove_tags = [classes('google-dfp-ad-wrapper enhancement')] |
| 69 |
|
| 70 |
def parse_index(self): |
| 71 |
index = 'https://www.latimes.com/' |
| 72 |
# Constrain the date path-segment to today+yesterday so the section |
| 73 |
# indexes don't pull in arbitrary older articles. Same idea as |
| 74 |
# chicago_tribune.recipe's inline tdy/yest filter (just expressed as |
| 75 |
# a regex alternation since latimes uses re.compile). |
| 76 |
tdy = date.today().strftime('%Y-%m-%d') |
| 77 |
yest = (date.today() - timedelta(days=1)).strftime('%Y-%m-%d') |
| 78 |
pat = r'^https://www\.latimes\.com/[^/]+?/story/(' + tdy + '|' + yest + r')/\S+' |
| 79 |
articles = self.find_articles(index, pat) |
| 80 |
for collection in DIR_COLLECTIONS: |
| 81 |
if self.test: |
| 82 |
continue |
| 83 |
topdir = collection.pop(0) |
| 84 |
collection_index = index + topdir + '/' |
| 85 |
articles += self.find_articles(collection_index, pat) |
| 86 |
for subdir in collection: |
| 87 |
sub_index = collection_index + subdir + '/' |
| 88 |
articles += self.find_articles(sub_index, pat) |
| 89 |
|
| 90 |
feeds = defaultdict(list) |
| 91 |
for article in articles: |
| 92 |
section = what_section(article['url']) |
| 93 |
feeds[section].append(article) |
| 94 |
|
| 95 |
return [(k, feeds[k]) for k in sorted(feeds)] |
| 96 |
|
| 97 |
def preprocess_html(self, soup): |
| 98 |
for img in soup.findAll('img', attrs={'data-src': True}): |
| 99 |
if img.findParent('a', href='http://www.latimes.com/opinion/la-letter-to-the-editor-htmlstory.html') is img.parent and img['data-src'].endswith( |
| 100 |
'/la-letter-to-the-editor' |
| 101 |
): |
| 102 |
img.parent.extract() |
| 103 |
else: |
| 104 |
img['src'] = img['data-src'] |
| 105 |
# Strip <picture> wrappers and viewport sizing hints. <source srcset> |
| 106 |
# URLs are remote and never load in an EPUB (no network at read |
| 107 |
# time), and sizes="100vw" on the <img> tells the reader to render |
| 108 |
# at full viewport width — combined with the image's natural aspect |
| 109 |
# ratio that overflows pages for portrait photos. Drop <source>, |
| 110 |
# unwrap <picture> so the <img> rises to its parent, and remove the |
| 111 |
# sizes/fetchpriority hints so the reader respects the image's |
| 112 |
# embedded dimensions. |
| 113 |
for src in soup.findAll('source'): |
| 114 |
src.decompose() |
| 115 |
for picture in soup.findAll('picture'): |
| 116 |
picture.unwrap() |
| 117 |
for img in soup.findAll('img'): |
| 118 |
for attr in ('sizes', 'fetchpriority'): |
| 119 |
if attr in img.attrs: |
| 120 |
del img[attr] |
| 121 |
# Drop the Twitter "Follow" link from each byline. The anchor wraps |
| 122 |
# an SVG <use> sprite reference that points at a <shape> in a |
| 123 |
# sibling <svg> elsewhere in the source document; that cross- |
| 124 |
# document reference can't resolve in the converted EPUB, so the |
| 125 |
# anchor renders as an empty icon slot followed by the orphan word |
| 126 |
# "Follow" next to the byline. The "Staff Writer" <span> sibling |
| 127 |
# is not a descendant of the anchor, so the rubric stays in place. |
| 128 |
for a in soup.findAll('a', attrs={'data-social-trigger': 'enhancedByline'}): |
| 129 |
a.decompose() |
| 130 |
return soup |
| 131 |
|
| 132 |
def find_articles(self, index, pattern): |
| 133 |
self.log('Downloading and parsing index: ', index) |
| 134 |
self.log('Pattern: ', pattern) |
| 135 |
try: |
| 136 |
soup = self.index_to_soup(index) |
| 137 |
except Exception: |
| 138 |
self.log('Failed to download ', index) |
| 139 |
return [] |
| 140 |
if soup.main is not None: |
| 141 |
alinks = soup.main.findAll('a', {'href': re.compile(pattern)}) |
| 142 |
else: |
| 143 |
alinks = soup.findAll('a', {'href': re.compile(pattern)}) |
| 144 |
alinks = [a for a in alinks if len(a.contents) == 1 and a.find(text=True, recursive=False)] |
| 145 |
articles = [{'title': self.tag_to_string(a), 'url': absurl(a['href'])} for a in alinks] |
| 146 |
# Populate `description` from each article page's |
| 147 |
cache = getattr(self, '_og_desc_cache', None) |
| 148 |
if cache is None: |
| 149 |
cache = self._og_desc_cache = {} |
| 150 |
for art in articles: |
| 151 |
url = art.get('url') |
| 152 |
if not url: |
| 153 |
continue |
| 154 |
if url not in cache: |
| 155 |
cache[url] = self._fetch_og_description(url) |
| 156 |
desc = cache[url] |
| 157 |
if desc and len(desc) >= 20: |
| 158 |
art['description'] = desc |
| 159 |
self.log('Found: ', len(articles), ' articles.\n') |
| 160 |
return articles |
| 161 |
|
| 162 |
def _fetch_og_description(self, url): |
| 163 |
try: |
| 164 |
soup = self.index_to_soup(url) |
| 165 |
except Exception as e: |
| 166 |
self.log.warn('LAT og:description fetch failed for', url, ':', e) |
| 167 |
return None |
| 168 |
meta = soup.find('meta', attrs={'property': 'og:description'}) |
| 169 |
if meta and meta.get('content'): |
| 170 |
return meta['content'].strip() or None |
| 171 |
return None |