| 1 |
#!/usr/bin/env python |
| 2 |
# vim:fileencoding=utf-8 |
| 3 |
# License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net> |
| 4 |
from collections import defaultdict |
| 5 |
from datetime import date |
| 6 |
|
| 7 |
from calibre.web.feeds.news import BasicNewsRecipe, classes |
| 8 |
|
| 9 |
|
| 10 |
def class_as_string(x): |
| 11 |
if isinstance(x, (list, tuple)): |
| 12 |
x = ' '.join(x) |
| 13 |
return x |
| 14 |
|
| 15 |
|
| 16 |
def class_startswith(*prefixes): |
| 17 |
|
| 18 |
def q(x): |
| 19 |
if x: |
| 20 |
x = class_as_string(x) |
| 21 |
for prefix in prefixes: |
| 22 |
if x.startswith(prefix): |
| 23 |
return True |
| 24 |
return False |
| 25 |
|
| 26 |
return dict(attrs={'class': q}) |
| 27 |
|
| 28 |
|
| 29 |
def absolutize_url(url): |
| 30 |
if url.startswith('//'): |
| 31 |
return 'https:' + url |
| 32 |
if url.startswith('/'): |
| 33 |
url = 'https://www.bostonglobe.com' + url |
| 34 |
return url |
| 35 |
|
| 36 |
|
| 37 |
class BostonGlobePrint(BasicNewsRecipe): |
| 38 |
title = 'Boston Globe | Print Edition' |
| 39 |
__author__ = 'Kovid Goyal, unkn0wn' |
| 40 |
description = "The Boston Globe - Today's Paper" |
| 41 |
language = 'en_US' |
| 42 |
|
| 43 |
keep_only_tags = [ |
| 44 |
class_startswith('headline |', 'subheader |', 'byline |', 'image |', 'lead |', 'body |', 'comic-debug'), |
| 45 |
] |
| 46 |
remove_tags = [classes('inline-newsletter ad skip-nav article-footer sharebar arc_ad'), dict(id='continue_button'), dict(name=['meta', 'link'])] |
| 47 |
remove_tags_after = dict(attrs={'class': lambda x: x and x.startswith('body |')}) |
| 48 |
remove_attributes = ['style', 'height', 'width'] |
| 49 |
no_stylesheets = True |
| 50 |
scale_news_images = 1600, 1200 |
| 51 |
ignore_duplicate_articles = {'url'} |
| 52 |
# simultaneous_downloads = 1 |
| 53 |
|
| 54 |
def image_url_processor(self, baseurl, url): |
| 55 |
return absolutize_url(url) |
| 56 |
|
| 57 |
def get_cover_url(self): |
| 58 |
cover = ( |
| 59 |
'https://img.kiosko.net/' |
| 60 |
+ str(date.today().year) |
| 61 |
+ '/' |
| 62 |
+ date.today().strftime('%m') |
| 63 |
+ '/' |
| 64 |
+ date.today().strftime('%d') |
| 65 |
+ '/us/boston_globe.750.jpg' |
| 66 |
) |
| 67 |
br = BasicNewsRecipe.get_browser(self, verify_ssl_certificates=False) |
| 68 |
try: |
| 69 |
br.open(cover) |
| 70 |
except Exception: |
| 71 |
index = 'https://en.kiosko.net/us/np/boston_globe.html' |
| 72 |
soup = self.index_to_soup(index) |
| 73 |
for image in soup.findAll('img', src=True): |
| 74 |
if image['src'].endswith('750.jpg'): |
| 75 |
return 'https:' + image['src'] |
| 76 |
self.log('\nCover unavailable') |
| 77 |
cover = None |
| 78 |
return cover |
| 79 |
|
| 80 |
def parse_index(self): |
| 81 |
|
| 82 |
soup = self.index_to_soup('https://www.bostonglobe.com/todays-paper/') |
| 83 |
if timefmt := soup.find(**classes('todays-date')): |
| 84 |
self.timefmt = ' [' + self.tag_to_string(timefmt) + ']' |
| 85 |
|
| 86 |
feeds_dict = defaultdict(list) |
| 87 |
|
| 88 |
for div in soup.findAll('section', attrs={'id': ['sp-top-main', 'sp-middle-main']}): |
| 89 |
for a in div.findAll('a', href=lambda x: x and x.startswith('/' + str(date.today().year) + '/')): |
| 90 |
section = 'Front Page' |
| 91 |
if bar := a.findParent(**classes('container')).find_previous_sibling(**classes('title_bar')): |
| 92 |
section = self.tag_to_string(bar) |
| 93 |
url = absolutize_url(a['href']) |
| 94 |
title = self.tag_to_string(a.find('h2')) |
| 95 |
desc = '' |
| 96 |
if d := a.find(**classes('deck')): |
| 97 |
desc = self.tag_to_string(d) |
| 98 |
|
| 99 |
self.log(section, '\n\t', title, '\n\t', desc, '\n\t\t', url) |
| 100 |
feeds_dict[section].append({'title': title, 'url': url, 'description': desc}) |
| 101 |
return list(feeds_dict.items()) |
| 102 |
|
| 103 |
def preprocess_raw_html(self, raw_html, url): |
| 104 |
soup = self.index_to_soup(raw_html) |
| 105 |
meta = soup.find(attrs={'name': 'description'}, content=True) |
| 106 |
if meta is not None and meta['content'].startswith('Comics: '): |
| 107 |
meta = soup.find(property='og:image', content=True) |
| 108 |
img_url = 'https://cloudfront-us-east-1.images.arcpublishing.com/bostonglobe/' + meta['content'].split('/')[-1] |
| 109 |
title = self.tag_to_string(soup.find('title')) |
| 110 |
raw_html = '<html><body><h1 class="headline |">{}</h1><div class="image |"><img src="{}"></div></body></html>'.format(title, img_url) |
| 111 |
return raw_html |
| 112 |
|
| 113 |
def preprocess_html(self, soup): |
| 114 |
for img in soup.findAll('img'): |
| 115 |
fs = img.get('data-src') |
| 116 |
if fs: |
| 117 |
img['src'] = fs |
| 118 |
return soup |