| 1 |
#!/usr/bin/env python |
| 2 |
# vim:fileencoding=utf-8 |
| 3 |
# License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net> |
| 4 |
|
| 5 |
from __future__ import absolute_import, division, print_function, unicode_literals |
| 6 |
|
| 7 |
import json |
| 8 |
import pprint |
| 9 |
from datetime import timedelta |
| 10 |
|
| 11 |
from calibre.utils.date import utcnow |
| 12 |
from calibre.utils.iso8601 import parse_iso8601 |
| 13 |
from calibre.web.feeds.recipes import BasicNewsRecipe |
| 14 |
|
| 15 |
oldest_article = 1 # days, includes articles that were published no more than the specified number of days ago |
| 16 |
|
| 17 |
|
| 18 |
def classes(classes): |
| 19 |
q = frozenset(classes.split(' ')) |
| 20 |
return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)}) |
| 21 |
|
| 22 |
|
| 23 |
def class_as_string(x): |
| 24 |
if isinstance(x, (list, tuple)): |
| 25 |
x = ' '.join(x) |
| 26 |
return x |
| 27 |
|
| 28 |
|
| 29 |
def class_startswith(*prefixes): |
| 30 |
|
| 31 |
def q(x): |
| 32 |
if x: |
| 33 |
x = class_as_string(x) |
| 34 |
for prefix in prefixes: |
| 35 |
if x.startswith(prefix): |
| 36 |
return True |
| 37 |
return False |
| 38 |
|
| 39 |
return dict(attrs={'class': q}) |
| 40 |
|
| 41 |
|
| 42 |
# From: https://www3.bostonglobe.com/lifestyle/comics?arc404=true |
| 43 |
comics_to_fetch = { |
| 44 |
'ADAM@HOME': 'ad', |
| 45 |
'ARLO & JANIS': 'aj', |
| 46 |
# "CUL DE SAC": 'cds', |
| 47 |
# "CURTIS": 'kfcrt', |
| 48 |
'DILBERT': 'dt', |
| 49 |
'DOONESBURY': 'db', |
| 50 |
'DUSTIN': 'kfdus', |
| 51 |
'F MINUS': 'fm', |
| 52 |
'FOR BETTER OR WORSE': 'fb', |
| 53 |
# "GET FUZZY": 'gz', |
| 54 |
# "MOTHER GOOSE & GRIMM": 'tmmgg', |
| 55 |
# "JUMPSTART": 'jt', |
| 56 |
'MONTY': 'mt', |
| 57 |
# "POOCH CAFE", |
| 58 |
'RHYMES WITH ORANGE': 'kfrwo', |
| 59 |
# "ROSE IS ROSE": 'rr', |
| 60 |
# "ZIPPY THE PINHEAD": 'kfzpy', |
| 61 |
'ZITS': 'kfzt', |
| 62 |
} |
| 63 |
|
| 64 |
|
| 65 |
def extract_json(raw_html): |
| 66 |
idx = raw_html.find('Fusion.contentCache={') |
| 67 |
close_idx = raw_html.find('</script>', idx) |
| 68 |
raw = raw_html[idx:close_idx] |
| 69 |
raw = raw[raw.find('{') :] |
| 70 |
idx = raw.find(';Fusion') |
| 71 |
raw = raw[:idx] |
| 72 |
# open('/t/raw.html', 'w').write(raw) |
| 73 |
data = json.loads(raw, strict=False) |
| 74 |
# open('/t/raw.json', 'w').write(json.dumps(data, indent=2)) |
| 75 |
return data |
| 76 |
|
| 77 |
|
| 78 |
def absolutize_url(url): |
| 79 |
if url.startswith('//'): |
| 80 |
return 'https:' + url |
| 81 |
if url.startswith('/'): |
| 82 |
url = 'https://www.bostonglobe.com' + url |
| 83 |
return url |
| 84 |
|
| 85 |
|
| 86 |
def parse_section(raw_html): |
| 87 |
data = extract_json(raw_html)['content-feed'] |
| 88 |
now = utcnow() |
| 89 |
cutoff_date = now - timedelta(days=oldest_article) |
| 90 |
|
| 91 |
def text(e): |
| 92 |
if not e: |
| 93 |
return '' |
| 94 |
return e.get('basic') or e.get('native', '') |
| 95 |
|
| 96 |
for group in data.values(): |
| 97 |
for elem in group['data']['content_elements']: |
| 98 |
date = parse_iso8601(elem['publish_date']) |
| 99 |
if date < cutoff_date: |
| 100 |
continue |
| 101 |
title = text(elem['headlines']) |
| 102 |
description = text(elem.get('description')) |
| 103 |
try: |
| 104 |
url = absolutize_url(elem['canonical_url']) |
| 105 |
except KeyError: |
| 106 |
continue |
| 107 |
yield {'title': title, 'url': url, 'description': description, 'date': ' ' + str(date.date())} |
| 108 |
|
| 109 |
|
| 110 |
def main(): |
| 111 |
for sec in 'metro world'.split(): |
| 112 |
for item in parse_section(open('/t/{}.html'.format(sec)).read()): |
| 113 |
print(item) |
| 114 |
|
| 115 |
|
| 116 |
# if __name__ == '__main__': |
| 117 |
# main() |
| 118 |
|
| 119 |
|
| 120 |
class BostonGlobeSubscription(BasicNewsRecipe): |
| 121 |
title = 'Boston Globe' |
| 122 |
__author__ = 'Kovid Goyal' |
| 123 |
description = 'The Boston Globe' |
| 124 |
language = 'en_US' |
| 125 |
timefmt = ' [%a, %d %b, %Y]' |
| 126 |
keep_only_tags = [ |
| 127 |
class_startswith('headline |', 'subheader |', 'byline |', 'image |', 'lead |', 'body |', 'comic-debug'), |
| 128 |
] |
| 129 |
remove_tags = [classes('inline-newsletter ad skip-nav article-footer sharebar arc_ad'), dict(id='continue_button'), dict(name=['meta', 'link'])] |
| 130 |
remove_tags_after = dict(attrs={'class': lambda x: x and x.startswith('body |')}) |
| 131 |
remove_attributes = ['style'] |
| 132 |
no_stylesheets = True |
| 133 |
scale_news_images = 1600, 1200 |
| 134 |
ignore_duplicate_articles = {'url'} |
| 135 |
# simultaneous_downloads = 1 |
| 136 |
|
| 137 |
def image_url_processor(self, baseurl, url): |
| 138 |
return absolutize_url(url) |
| 139 |
|
| 140 |
def parse_index(self): |
| 141 |
feeds = [] |
| 142 |
for sec in 'metro sports nation world business opinion lifestyle arts'.split(): |
| 143 |
articles = list(parse_section(self.index_to_soup(absolutize_url('/' + sec), raw=True).decode('utf-8'))) |
| 144 |
if articles: |
| 145 |
self.log(sec.capitalize()) |
| 146 |
self.log(pprint.pformat(articles)) |
| 147 |
feeds.append((sec.capitalize(), articles)) |
| 148 |
if self.test: |
| 149 |
del articles[self.test[1] :] |
| 150 |
if len(feeds) >= self.test[0]: |
| 151 |
break |
| 152 |
|
| 153 |
articles = [] |
| 154 |
for title, slug in comics_to_fetch.items(): |
| 155 |
articles.append({'title': title, 'url': 'https://www.bostonglobe.com/games-comics/comics/{}/'.format(slug)}) |
| 156 |
if articles: |
| 157 |
feeds.append(('Comics', articles)) |
| 158 |
return feeds |
| 159 |
|
| 160 |
def preprocess_raw_html(self, raw_html, url): |
| 161 |
soup = self.index_to_soup(raw_html) |
| 162 |
meta = soup.find(attrs={'name': 'description'}, content=True) |
| 163 |
if meta is not None and meta['content'].startswith('Comics: '): |
| 164 |
meta = soup.find(property='og:image', content=True) |
| 165 |
img_url = 'https://cloudfront-us-east-1.images.arcpublishing.com/bostonglobe/' + meta['content'].split('/')[-1] |
| 166 |
title = self.tag_to_string(soup.find('title')) |
| 167 |
raw_html = '<html><body><h1 class="headline |">{}</h1><div class="image |"><img src="{}"></div></body></html>'.format(title, img_url) |
| 168 |
return raw_html |
| 169 |
|
| 170 |
def preprocess_html(self, soup): |
| 171 |
for img in soup.findAll('img'): |
| 172 |
fs = img.get('data-src') |
| 173 |
if fs: |
| 174 |
img['src'] = fs |
| 175 |
return soup |