Debian Codegraph @

recipes/boston_globe_print_edition.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 # License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
4 from collections import defaultdict
5 from datetime import date
6
7 from calibre.web.feeds.news import BasicNewsRecipe, classes
8
9
10 def class_as_string(x):
11 if isinstance(x, (list, tuple)):
12 x = ' '.join(x)
13 return x
14
15
16 def class_startswith(*prefixes):
17
18 def q(x):
19 if x:
20 x = class_as_string(x)
21 for prefix in prefixes:
22 if x.startswith(prefix):
23 return True
24 return False
25
26 return dict(attrs={'class': q})
27
28
29 def absolutize_url(url):
30 if url.startswith('//'):
31 return 'https:' + url
32 if url.startswith('/'):
33 url = 'https://www.bostonglobe.com' + url
34 return url
35
36
37 class BostonGlobePrint(BasicNewsRecipe):
38 title = 'Boston Globe | Print Edition'
39 __author__ = 'Kovid Goyal, unkn0wn'
40 description = "The Boston Globe - Today's Paper"
41 language = 'en_US'
42
43 keep_only_tags = [
44 class_startswith('headline |', 'subheader |', 'byline |', 'image |', 'lead |', 'body |', 'comic-debug'),
45 ]
46 remove_tags = [classes('inline-newsletter ad skip-nav article-footer sharebar arc_ad'), dict(id='continue_button'), dict(name=['meta', 'link'])]
47 remove_tags_after = dict(attrs={'class': lambda x: x and x.startswith('body |')})
48 remove_attributes = ['style', 'height', 'width']
49 no_stylesheets = True
50 scale_news_images = 1600, 1200
51 ignore_duplicate_articles = {'url'}
52 # simultaneous_downloads = 1
53
54 def image_url_processor(self, baseurl, url):
55 return absolutize_url(url)
56
57 def get_cover_url(self):
58 cover = (
59 'https://img.kiosko.net/'
60 + str(date.today().year)
61 + '/'
62 + date.today().strftime('%m')
63 + '/'
64 + date.today().strftime('%d')
65 + '/us/boston_globe.750.jpg'
66 )
67 br = BasicNewsRecipe.get_browser(self, verify_ssl_certificates=False)
68 try:
69 br.open(cover)
70 except Exception:
71 index = 'https://en.kiosko.net/us/np/boston_globe.html'
72 soup = self.index_to_soup(index)
73 for image in soup.findAll('img', src=True):
74 if image['src'].endswith('750.jpg'):
75 return 'https:' + image['src']
76 self.log('\nCover unavailable')
77 cover = None
78 return cover
79
80 def parse_index(self):
81
82 soup = self.index_to_soup('https://www.bostonglobe.com/todays-paper/')
83 if timefmt := soup.find(**classes('todays-date')):
84 self.timefmt = ' [' + self.tag_to_string(timefmt) + ']'
85
86 feeds_dict = defaultdict(list)
87
88 for div in soup.findAll('section', attrs={'id': ['sp-top-main', 'sp-middle-main']}):
89 for a in div.findAll('a', href=lambda x: x and x.startswith('/' + str(date.today().year) + '/')):
90 section = 'Front Page'
91 if bar := a.findParent(**classes('container')).find_previous_sibling(**classes('title_bar')):
92 section = self.tag_to_string(bar)
93 url = absolutize_url(a['href'])
94 title = self.tag_to_string(a.find('h2'))
95 desc = ''
96 if d := a.find(**classes('deck')):
97 desc = self.tag_to_string(d)
98
99 self.log(section, '\n\t', title, '\n\t', desc, '\n\t\t', url)
100 feeds_dict[section].append({'title': title, 'url': url, 'description': desc})
101 return list(feeds_dict.items())
102
103 def preprocess_raw_html(self, raw_html, url):
104 soup = self.index_to_soup(raw_html)
105 meta = soup.find(attrs={'name': 'description'}, content=True)
106 if meta is not None and meta['content'].startswith('Comics: '):
107 meta = soup.find(property='og:image', content=True)
108 img_url = 'https://cloudfront-us-east-1.images.arcpublishing.com/bostonglobe/' + meta['content'].split('/')[-1]
109 title = self.tag_to_string(soup.find('title'))
110 raw_html = '<html><body><h1 class="headline |">{}</h1><div class="image |"><img src="{}"></div></body></html>'.format(title, img_url)
111 return raw_html
112
113 def preprocess_html(self, soup):
114 for img in soup.findAll('img'):
115 fs = img.get('data-src')
116 if fs:
117 img['src'] = fs
118 return soup