Debian Codegraph @

recipes/ap.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 """
4 https://apnews.com
5 """
6
7 from datetime import datetime, timezone
8
9 from calibre.web.feeds.news import BasicNewsRecipe, classes
10
11
12 class AssociatedPress(BasicNewsRecipe):
13 title = 'Associated Press'
14 description = (
15 'Read the latest headlines, breaking news, and videos at APNews.com, the definitive '
16 'source for independent journalism from every corner of the globe. Articles from Front Page.'
17 )
18 __author__ = 'unkn0wn'
19 language = 'en'
20 encoding = 'utf-8'
21 no_stylesheets = True
22 remove_javascript = True
23 ignore_duplicate_articles = {'url'}
24 remove_empty_feeds = False
25 remove_attributes = ['style', 'height', 'width']
26 simultaneous_downloads = 1
27 oldest_article = 1 # days; AP publishes constantly
28 cover_url = 'https://upload.wikimedia.org/wikipedia/commons/thumb/0/0c/Associated_Press_logo_2012.svg/662px-Associated_Press_logo_2012.svg.png'
29
30 keep_only_tags = [
31 classes('StoryPage-lede-content Page-lead Page-byline-info RichTextStoryBody'),
32 ]
33 remove_tags = [
34 classes('displayNone Advertisement HTMLModuleEnhancement AudioEnhancement'),
35 dict(
36 name=[
37 'source',
38 'button',
39 'svg',
40 'template',
41 'video',
42 'astro-island',
43 'iframe',
44 'document',
45 ]
46 ),
47 dict(attrs={'data-parsely-title': 'Related Stories'}),
48 ]
49
50 extra_css = '''
51 .Page-byline-info, .Page-breadcrumbs, .CarouselSlide-info, .Figure-caption { font-size:small; }
52 img {display:block; margin:0 auto;}
53 em { color: #202020; }
54 '''
55
56 def parse_index(self):
57 feeds = []
58 soup = self.index_to_soup('https://apnews.com')
59 for a in soup.findAll(
60 'a',
61 attrs={'href': lambda x: x and x.startswith('https://apnews.com/article/')},
62 ):
63 url = a['href']
64 title = self.tag_to_string(a)
65 pub = self._fetch_published_time(url)
66 if pub is None:
67 self.log.warn('Skipping (no published_time):', url)
68 continue
69 self.log(title, '\n\t', url, '\n\t', pub.isoformat())
70 feeds.append({
71 'title': title,
72 'url': url,
73 'date': pub.strftime('%Y-%m-%d %H:%M:%S'),
74 'timestamp': pub.timestamp(),
75 })
76 return [('Articles', feeds)]
77
78 def _fetch_published_time(self, url):
79 # AP has no RSS feeds, so parse_index can't filter by date without
80 # fetching each candidate article. Read <meta property="article:
81 # published_time"> from the article HTML so oldest_article applies.
82 cache = getattr(self, '_pub_time_cache', None)
83 if cache is None:
84 cache = self._pub_time_cache = {}
85 if url in cache:
86 return cache[url]
87 try:
88 soup = self.index_to_soup(url)
89 except Exception as e:
90 self.log.warn('AP published_time fetch failed for', url, ':', e)
91 cache[url] = None
92 return None
93 meta = soup.find('meta', attrs={'property': 'article:published_time'})
94 if not meta or not meta.get('content'):
95 cache[url] = None
96 return None
97 try:
98 pub = datetime.fromisoformat(meta['content'])
99 except Exception as e:
100 self.log.warn('AP bad published_time on', url, ':', e)
101 cache[url] = None
102 return None
103 if pub.tzinfo is None:
104 pub = pub.replace(tzinfo=timezone.utc)
105 cache[url] = pub
106 return pub
107
108 def preprocess_html(self, soup):
109 for v in soup.findAll('bsp-jw-player', attrs={'poster': True}):
110 v.name = 'img'
111 v.attrs = {'src': v.get('poster', '')}
112 for st in soup.findAll(**classes('CarouselSlide-infoDescription Figure-caption')):
113 if p := st.find('p'):
114 p.name = 'span'
115 for h in soup.findAll(['h2', 'h3']):
116 h.name = 'h4'
117 for img in soup.findAll('img', attrs={'srcset': True}):
118 img['src'] = img['srcset'].split()[0]
119 for img_ in soup.findAll('img', attrs={'data-flickity-lazyload-srcset': True, 'srcset': False}):
120 img_['src'] = img_['data-flickity-lazyload-srcset'].split()[0]
121 return soup