Debian Codegraph @

recipes/boston.com.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 # License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
4
5 from __future__ import absolute_import, division, print_function, unicode_literals
6
7 import json
8 import pprint
9 from datetime import timedelta
10
11 from calibre.utils.date import utcnow
12 from calibre.utils.iso8601 import parse_iso8601
13 from calibre.web.feeds.recipes import BasicNewsRecipe
14
15 oldest_article = 1 # days, includes articles that were published no more than the specified number of days ago
16
17
18 def classes(classes):
19 q = frozenset(classes.split(' '))
20 return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)})
21
22
23 def class_as_string(x):
24 if isinstance(x, (list, tuple)):
25 x = ' '.join(x)
26 return x
27
28
29 def class_startswith(*prefixes):
30
31 def q(x):
32 if x:
33 x = class_as_string(x)
34 for prefix in prefixes:
35 if x.startswith(prefix):
36 return True
37 return False
38
39 return dict(attrs={'class': q})
40
41
42 # From: https://www3.bostonglobe.com/lifestyle/comics?arc404=true
43 comics_to_fetch = {
44 'ADAM@HOME': 'ad',
45 'ARLO & JANIS': 'aj',
46 # "CUL DE SAC": 'cds',
47 # "CURTIS": 'kfcrt',
48 'DILBERT': 'dt',
49 'DOONESBURY': 'db',
50 'DUSTIN': 'kfdus',
51 'F MINUS': 'fm',
52 'FOR BETTER OR WORSE': 'fb',
53 # "GET FUZZY": 'gz',
54 # "MOTHER GOOSE & GRIMM": 'tmmgg',
55 # "JUMPSTART": 'jt',
56 'MONTY': 'mt',
57 # "POOCH CAFE",
58 'RHYMES WITH ORANGE': 'kfrwo',
59 # "ROSE IS ROSE": 'rr',
60 # "ZIPPY THE PINHEAD": 'kfzpy',
61 'ZITS': 'kfzt',
62 }
63
64
65 def extract_json(raw_html):
66 idx = raw_html.find('Fusion.contentCache={')
67 close_idx = raw_html.find('</script>', idx)
68 raw = raw_html[idx:close_idx]
69 raw = raw[raw.find('{') :]
70 idx = raw.find(';Fusion')
71 raw = raw[:idx]
72 # open('/t/raw.html', 'w').write(raw)
73 data = json.loads(raw, strict=False)
74 # open('/t/raw.json', 'w').write(json.dumps(data, indent=2))
75 return data
76
77
78 def absolutize_url(url):
79 if url.startswith('//'):
80 return 'https:' + url
81 if url.startswith('/'):
82 url = 'https://www.bostonglobe.com' + url
83 return url
84
85
86 def parse_section(raw_html):
87 data = extract_json(raw_html)['content-feed']
88 now = utcnow()
89 cutoff_date = now - timedelta(days=oldest_article)
90
91 def text(e):
92 if not e:
93 return ''
94 return e.get('basic') or e.get('native', '')
95
96 for group in data.values():
97 for elem in group['data']['content_elements']:
98 date = parse_iso8601(elem['publish_date'])
99 if date < cutoff_date:
100 continue
101 title = text(elem['headlines'])
102 description = text(elem.get('description'))
103 try:
104 url = absolutize_url(elem['canonical_url'])
105 except KeyError:
106 continue
107 yield {'title': title, 'url': url, 'description': description, 'date': ' ' + str(date.date())}
108
109
110 def main():
111 for sec in 'metro world'.split():
112 for item in parse_section(open('/t/{}.html'.format(sec)).read()):
113 print(item)
114
115
116 # if __name__ == '__main__':
117 # main()
118
119
120 class BostonGlobeSubscription(BasicNewsRecipe):
121 title = 'Boston Globe'
122 __author__ = 'Kovid Goyal'
123 description = 'The Boston Globe'
124 language = 'en_US'
125 timefmt = ' [%a, %d %b, %Y]'
126 keep_only_tags = [
127 class_startswith('headline |', 'subheader |', 'byline |', 'image |', 'lead |', 'body |', 'comic-debug'),
128 ]
129 remove_tags = [classes('inline-newsletter ad skip-nav article-footer sharebar arc_ad'), dict(id='continue_button'), dict(name=['meta', 'link'])]
130 remove_tags_after = dict(attrs={'class': lambda x: x and x.startswith('body |')})
131 remove_attributes = ['style']
132 no_stylesheets = True
133 scale_news_images = 1600, 1200
134 ignore_duplicate_articles = {'url'}
135 # simultaneous_downloads = 1
136
137 def image_url_processor(self, baseurl, url):
138 return absolutize_url(url)
139
140 def parse_index(self):
141 feeds = []
142 for sec in 'metro sports nation world business opinion lifestyle arts'.split():
143 articles = list(parse_section(self.index_to_soup(absolutize_url('/' + sec), raw=True).decode('utf-8')))
144 if articles:
145 self.log(sec.capitalize())
146 self.log(pprint.pformat(articles))
147 feeds.append((sec.capitalize(), articles))
148 if self.test:
149 del articles[self.test[1] :]
150 if len(feeds) >= self.test[0]:
151 break
152
153 articles = []
154 for title, slug in comics_to_fetch.items():
155 articles.append({'title': title, 'url': 'https://www.bostonglobe.com/games-comics/comics/{}/'.format(slug)})
156 if articles:
157 feeds.append(('Comics', articles))
158 return feeds
159
160 def preprocess_raw_html(self, raw_html, url):
161 soup = self.index_to_soup(raw_html)
162 meta = soup.find(attrs={'name': 'description'}, content=True)
163 if meta is not None and meta['content'].startswith('Comics: '):
164 meta = soup.find(property='og:image', content=True)
165 img_url = 'https://cloudfront-us-east-1.images.arcpublishing.com/bostonglobe/' + meta['content'].split('/')[-1]
166 title = self.tag_to_string(soup.find('title'))
167 raw_html = '<html><body><h1 class="headline |">{}</h1><div class="image |"><img src="{}"></div></body></html>'.format(title, img_url)
168 return raw_html
169
170 def preprocess_html(self, soup):
171 for img in soup.findAll('img'):
172 fs = img.get('data-src')
173 if fs:
174 img['src'] = fs
175 return soup