Debian Codegraph @

recipes/guardian.recipe

raw
1 #!/usr/bin/env python
2 __license__ = 'GPL v3'
3 __copyright__ = '2008, Kovid Goyal [email protected]'
4 __docformat__ = 'restructuredtext en'
5
6 '''
7 www.guardian.co.uk
8 '''
9 from datetime import date
10
11 from calibre import random_user_agent
12 from calibre.web.feeds.news import BasicNewsRecipe
13
14
15 def classes(classes):
16 q = frozenset(classes.split(' '))
17 return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)})
18
19
20 class Guardian(BasicNewsRecipe):
21 title = 'The Guardian and The Observer'
22 is_observer = False
23 base_url = 'https://www.theguardian.com/uk'
24 if date.today().weekday() == 6:
25 is_observer = True
26 base_url = 'https://www.theguardian.com/observer'
27
28 __author__ = 'Kovid Goyal'
29 language = 'en_GB'
30
31 oldest_article = 7
32 max_articles_per_feed = 100
33 remove_javascript = True
34 encoding = 'utf-8'
35 remove_empty_feeds = True
36 no_stylesheets = True
37 remove_attributes = ['style', 'width', 'height']
38 ignore_duplicate_articles = {'title', 'url'}
39
40 timefmt = ' [%a, %d %b %Y]'
41
42 remove_tags = [
43 dict(attrs={'class': lambda x: x and '--twitter' in x}),
44 dict(attrs={'class': lambda x: x and 'submeta' in x.split()}),
45 dict(name='gu-island'),
46 dict(attrs={'data-component': ['share', 'social', 'nav', 'nav2', 'topbar']}),
47 dict(attrs={'data-link-name': 'block share'}),
48 dict(attrs={'data-print-layout': 'hide'}),
49 dict(attrs={'data-spacefinder-type': 'model.dotcomrendering.pageElements.NewsletterSignupBlockElement'}),
50 dict(id=['dfp-ad--survey', 'sub-nav-root', 'the-caption', 'bannerandheader']),
51 {'for': 'the-checkbox'},
52 dict(href=['#maincontent', '#navigation']),
53 dict(role=['navigation', 'button']),
54 dict(attrs={'class': lambda x: x and 'inline-expand-image' in x}),
55 dict(name='a', attrs={'aria-label': lambda x: x and 'Share On' in x}),
56 dict(name='a', attrs={'class': lambda x: x and 'social__action js-social__action--top' in x}),
57 dict(name='div', attrs={'id': 'share-count-root'}),
58 dict(attrs={'class': lambda x: x and 'modern-visible' in x.split()}),
59 classes('badge-slot reveal-caption__checkbox mobile-only element-rich-link'),
60 dict(name=['link', 'meta', 'style', 'svg', 'input', 'source', 'noscript', 'button']),
61 dict(name='img', src=lambda x: x and 'https://sb.scorecardresearch.com/' in x),
62 ]
63 remove_tags_after = [
64 classes('content__article-body js-bottom-marker article-body-commercial-selector'),
65 ]
66
67 extra_css = '''
68 img {
69 max-width: 100% !important;
70 max-height: 100% !important;
71 }
72
73 a span {
74 color: #E05E02;
75 }
76
77 figcaption span {
78 font-size: 0.5em;
79 color: #6B6B6B;
80 }
81 '''
82
83 def get_browser(self, *a, **kw):
84 # This site returns images in JPEG-XR format if the user agent is IE
85 if not hasattr(self, 'non_ie_ua'):
86 try:
87 self.non_ie_ua = random_user_agent(allow_ie=False)
88 except TypeError:
89 self.non_ie_ua = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.111 Safari/537.36'
90 kw['user_agent'] = self.non_ie_ua
91 br = BasicNewsRecipe.get_browser(self, *a, **kw)
92 return br
93
94 def parse_section(self, section_url):
95 soup = self.index_to_soup(section_url)
96 for section in soup.findAll('section'):
97 articles = []
98 title = self.tag_to_string(section.find('h2'))
99 if not title:
100 continue
101 self.log('Found section:', title)
102 for li in section.findAll('li'):
103 a = li.find('a', attrs={'href': True, 'aria-label': True})
104 if a:
105 url = a['href']
106 if url.startswith('/'):
107 url = self.base_url.rpartition('/')[0] + url
108 self.log('\t', a['aria-label'], url)
109 articles.append({'title': a['aria-label'], 'url': url})
110 if articles:
111 yield title, articles
112
113 def parse_index(self):
114 # return [('Test', [{'url':
115 # 'https://www.theguardian.com/environment/2025/nov/07/if-theres-a-free-alternative-ill-eat-healthily-how-sweden-devised-brilliant-school-meals',
116 # 'title': 'test'}])]
117 feeds = list(self.parse_section(self.base_url))
118 feeds += list(self.parse_section('https://www.theguardian.com/uk/sport'))
119 return feeds
120
121 def preprocess_html(self, soup):
122 for table in soup.findAll('table'):
123 if len(table.findAll('tr')) > 20:
124 table.decompose()
125 for dateline in soup.findAll(attrs={'data-gu-name': 'dateline'}):
126 for s in dateline.findAll('summary'):
127 s.extract()
128 dateline.name = 'div'
129 return soup