Debian Codegraph @

recipes/latimes.recipe

raw
1 #!/usr/bin/env python
2
3 import re
4 from collections import defaultdict
5 from datetime import date, timedelta
6
7 from calibre.web.feeds.news import BasicNewsRecipe
8
9 DIR_COLLECTIONS = [
10 ['world'],
11 ['nation'],
12 ['politics'],
13 ['opinion', 'op-ed', 'opinion-la', 'editorials', 'readersreact', 'topoftheticket', 'endorsements'],
14 ['local', 'lanow', 'california', 'crime', 'abcarian', 'education', 'weather'],
15 ['business', 'hollywood', 'technology'],
16 ['sports'],
17 ['entertainment', 'movies', 'music', 'tv', 'arts', 'gossip', 'envelope'],
18 ['books'],
19 ['food', 'jonathon-gold', 'dailydish'],
20 ['health'],
21 ['style', 'laaffairs', 'pets'],
22 ['science', 'sciencenow'],
23 ['home'],
24 ['travel'],
25 ['fashion'],
26 ]
27
28
29 def classes(classes):
30 q = frozenset(classes.split(' '))
31 return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)})
32
33
34 def absurl(url):
35 if url.startswith('/'):
36 url = 'https://www.latimes.com' + url
37 return url
38
39
40 def what_section(url):
41 parts = url.split('/')
42 return parts[-4].capitalize()
43
44
45 class LATimes(BasicNewsRecipe):
46 title = 'Los Angeles Times'
47 __author__ = 'Jose Ortiz'
48 description = 'The Los Angeles Times is a leading source of news on Southern California, entertainment, movies, television, music, politics, business, health, technology, travel, sports, environment, economics, autos, jobs, real estate and other topics affecting California' # noqa: E501
49 category = 'news, politics, USA, Los Angeles, world'
50 oldest_article = 1
51 max_articles_per_feed = 200
52 no_stylesheets = True
53 encoding = 'utf8'
54 use_embedded_content = False
55 compress_news_images = True
56 compress_news_images_auto_size = 5
57 language = 'en_US'
58 remove_empty_feeds = True
59 ignore_duplicate_articles = {'url'}
60 publication_type = 'newspaper'
61 cover_url = 'http://www.latimes.com/includes/sectionfronts/A1.pdf'
62
63 keep_only_tags = [
64 classes('headline page-lead-media authors published-date page-article-container'),
65 dict(attrs={'data-element': 'story-body'}),
66 ]
67
68 remove_tags = [classes('google-dfp-ad-wrapper enhancement')]
69
70 def parse_index(self):
71 index = 'https://www.latimes.com/'
72 # Constrain the date path-segment to today+yesterday so the section
73 # indexes don't pull in arbitrary older articles. Same idea as
74 # chicago_tribune.recipe's inline tdy/yest filter (just expressed as
75 # a regex alternation since latimes uses re.compile).
76 tdy = date.today().strftime('%Y-%m-%d')
77 yest = (date.today() - timedelta(days=1)).strftime('%Y-%m-%d')
78 pat = r'^https://www\.latimes\.com/[^/]+?/story/(' + tdy + '|' + yest + r')/\S+'
79 articles = self.find_articles(index, pat)
80 for collection in DIR_COLLECTIONS:
81 if self.test:
82 continue
83 topdir = collection.pop(0)
84 collection_index = index + topdir + '/'
85 articles += self.find_articles(collection_index, pat)
86 for subdir in collection:
87 sub_index = collection_index + subdir + '/'
88 articles += self.find_articles(sub_index, pat)
89
90 feeds = defaultdict(list)
91 for article in articles:
92 section = what_section(article['url'])
93 feeds[section].append(article)
94
95 return [(k, feeds[k]) for k in sorted(feeds)]
96
97 def preprocess_html(self, soup):
98 for img in soup.findAll('img', attrs={'data-src': True}):
99 if img.findParent('a', href='http://www.latimes.com/opinion/la-letter-to-the-editor-htmlstory.html') is img.parent and img['data-src'].endswith(
100 '/la-letter-to-the-editor'
101 ):
102 img.parent.extract()
103 else:
104 img['src'] = img['data-src']
105 # Strip <picture> wrappers and viewport sizing hints. <source srcset>
106 # URLs are remote and never load in an EPUB (no network at read
107 # time), and sizes="100vw" on the <img> tells the reader to render
108 # at full viewport width — combined with the image's natural aspect
109 # ratio that overflows pages for portrait photos. Drop <source>,
110 # unwrap <picture> so the <img> rises to its parent, and remove the
111 # sizes/fetchpriority hints so the reader respects the image's
112 # embedded dimensions.
113 for src in soup.findAll('source'):
114 src.decompose()
115 for picture in soup.findAll('picture'):
116 picture.unwrap()
117 for img in soup.findAll('img'):
118 for attr in ('sizes', 'fetchpriority'):
119 if attr in img.attrs:
120 del img[attr]
121 # Drop the Twitter "Follow" link from each byline. The anchor wraps
122 # an SVG <use> sprite reference that points at a <shape> in a
123 # sibling <svg> elsewhere in the source document; that cross-
124 # document reference can't resolve in the converted EPUB, so the
125 # anchor renders as an empty icon slot followed by the orphan word
126 # "Follow" next to the byline. The "Staff Writer" <span> sibling
127 # is not a descendant of the anchor, so the rubric stays in place.
128 for a in soup.findAll('a', attrs={'data-social-trigger': 'enhancedByline'}):
129 a.decompose()
130 return soup
131
132 def find_articles(self, index, pattern):
133 self.log('Downloading and parsing index: ', index)
134 self.log('Pattern: ', pattern)
135 try:
136 soup = self.index_to_soup(index)
137 except Exception:
138 self.log('Failed to download ', index)
139 return []
140 if soup.main is not None:
141 alinks = soup.main.findAll('a', {'href': re.compile(pattern)})
142 else:
143 alinks = soup.findAll('a', {'href': re.compile(pattern)})
144 alinks = [a for a in alinks if len(a.contents) == 1 and a.find(text=True, recursive=False)]
145 articles = [{'title': self.tag_to_string(a), 'url': absurl(a['href'])} for a in alinks]
146 # Populate `description` from each article page's
147 cache = getattr(self, '_og_desc_cache', None)
148 if cache is None:
149 cache = self._og_desc_cache = {}
150 for art in articles:
151 url = art.get('url')
152 if not url:
153 continue
154 if url not in cache:
155 cache[url] = self._fetch_og_description(url)
156 desc = cache[url]
157 if desc and len(desc) >= 20:
158 art['description'] = desc
159 self.log('Found: ', len(articles), ' articles.\n')
160 return articles
161
162 def _fetch_og_description(self, url):
163 try:
164 soup = self.index_to_soup(url)
165 except Exception as e:
166 self.log.warn('LAT og:description fetch failed for', url, ':', e)
167 return None
168 meta = soup.find('meta', attrs={'property': 'og:description'})
169 if meta and meta.get('content'):
170 return meta['content'].strip() or None
171 return None