| 1 |
#!/usr/bin/env python |
| 2 |
__license__ = 'GPL v3' |
| 3 |
__copyright__ = '2008, Kovid Goyal [email protected]' |
| 4 |
__docformat__ = 'restructuredtext en' |
| 5 |
|
| 6 |
''' |
| 7 |
www.guardian.co.uk |
| 8 |
''' |
| 9 |
from datetime import date |
| 10 |
|
| 11 |
from calibre import random_user_agent |
| 12 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 13 |
|
| 14 |
|
| 15 |
def classes(classes): |
| 16 |
q = frozenset(classes.split(' ')) |
| 17 |
return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)}) |
| 18 |
|
| 19 |
|
| 20 |
class Guardian(BasicNewsRecipe): |
| 21 |
title = 'The Guardian and The Observer' |
| 22 |
is_observer = False |
| 23 |
base_url = 'https://www.theguardian.com/uk' |
| 24 |
if date.today().weekday() == 6: |
| 25 |
is_observer = True |
| 26 |
base_url = 'https://www.theguardian.com/observer' |
| 27 |
|
| 28 |
__author__ = 'Kovid Goyal' |
| 29 |
language = 'en_GB' |
| 30 |
|
| 31 |
oldest_article = 7 |
| 32 |
max_articles_per_feed = 100 |
| 33 |
remove_javascript = True |
| 34 |
encoding = 'utf-8' |
| 35 |
remove_empty_feeds = True |
| 36 |
no_stylesheets = True |
| 37 |
remove_attributes = ['style', 'width', 'height'] |
| 38 |
ignore_duplicate_articles = {'title', 'url'} |
| 39 |
|
| 40 |
timefmt = ' [%a, %d %b %Y]' |
| 41 |
|
| 42 |
remove_tags = [ |
| 43 |
dict(attrs={'class': lambda x: x and '--twitter' in x}), |
| 44 |
dict(attrs={'class': lambda x: x and 'submeta' in x.split()}), |
| 45 |
dict(name='gu-island'), |
| 46 |
dict(attrs={'data-component': ['share', 'social', 'nav', 'nav2', 'topbar']}), |
| 47 |
dict(attrs={'data-link-name': 'block share'}), |
| 48 |
dict(attrs={'data-print-layout': 'hide'}), |
| 49 |
dict(attrs={'data-spacefinder-type': 'model.dotcomrendering.pageElements.NewsletterSignupBlockElement'}), |
| 50 |
dict(id=['dfp-ad--survey', 'sub-nav-root', 'the-caption', 'bannerandheader']), |
| 51 |
{'for': 'the-checkbox'}, |
| 52 |
dict(href=['#maincontent', '#navigation']), |
| 53 |
dict(role=['navigation', 'button']), |
| 54 |
dict(attrs={'class': lambda x: x and 'inline-expand-image' in x}), |
| 55 |
dict(name='a', attrs={'aria-label': lambda x: x and 'Share On' in x}), |
| 56 |
dict(name='a', attrs={'class': lambda x: x and 'social__action js-social__action--top' in x}), |
| 57 |
dict(name='div', attrs={'id': 'share-count-root'}), |
| 58 |
dict(attrs={'class': lambda x: x and 'modern-visible' in x.split()}), |
| 59 |
classes('badge-slot reveal-caption__checkbox mobile-only element-rich-link'), |
| 60 |
dict(name=['link', 'meta', 'style', 'svg', 'input', 'source', 'noscript', 'button']), |
| 61 |
dict(name='img', src=lambda x: x and 'https://sb.scorecardresearch.com/' in x), |
| 62 |
] |
| 63 |
remove_tags_after = [ |
| 64 |
classes('content__article-body js-bottom-marker article-body-commercial-selector'), |
| 65 |
] |
| 66 |
|
| 67 |
extra_css = ''' |
| 68 |
img { |
| 69 |
max-width: 100% !important; |
| 70 |
max-height: 100% !important; |
| 71 |
} |
| 72 |
|
| 73 |
a span { |
| 74 |
color: #E05E02; |
| 75 |
} |
| 76 |
|
| 77 |
figcaption span { |
| 78 |
font-size: 0.5em; |
| 79 |
color: #6B6B6B; |
| 80 |
} |
| 81 |
''' |
| 82 |
|
| 83 |
def get_browser(self, *a, **kw): |
| 84 |
# This site returns images in JPEG-XR format if the user agent is IE |
| 85 |
if not hasattr(self, 'non_ie_ua'): |
| 86 |
try: |
| 87 |
self.non_ie_ua = random_user_agent(allow_ie=False) |
| 88 |
except TypeError: |
| 89 |
self.non_ie_ua = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.111 Safari/537.36' |
| 90 |
kw['user_agent'] = self.non_ie_ua |
| 91 |
br = BasicNewsRecipe.get_browser(self, *a, **kw) |
| 92 |
return br |
| 93 |
|
| 94 |
def parse_section(self, section_url): |
| 95 |
soup = self.index_to_soup(section_url) |
| 96 |
for section in soup.findAll('section'): |
| 97 |
articles = [] |
| 98 |
title = self.tag_to_string(section.find('h2')) |
| 99 |
if not title: |
| 100 |
continue |
| 101 |
self.log('Found section:', title) |
| 102 |
for li in section.findAll('li'): |
| 103 |
a = li.find('a', attrs={'href': True, 'aria-label': True}) |
| 104 |
if a: |
| 105 |
url = a['href'] |
| 106 |
if url.startswith('/'): |
| 107 |
url = self.base_url.rpartition('/')[0] + url |
| 108 |
self.log('\t', a['aria-label'], url) |
| 109 |
articles.append({'title': a['aria-label'], 'url': url}) |
| 110 |
if articles: |
| 111 |
yield title, articles |
| 112 |
|
| 113 |
def parse_index(self): |
| 114 |
# return [('Test', [{'url': |
| 115 |
# 'https://www.theguardian.com/environment/2025/nov/07/if-theres-a-free-alternative-ill-eat-healthily-how-sweden-devised-brilliant-school-meals', |
| 116 |
# 'title': 'test'}])] |
| 117 |
feeds = list(self.parse_section(self.base_url)) |
| 118 |
feeds += list(self.parse_section('https://www.theguardian.com/uk/sport')) |
| 119 |
return feeds |
| 120 |
|
| 121 |
def preprocess_html(self, soup): |
| 122 |
for table in soup.findAll('table'): |
| 123 |
if len(table.findAll('tr')) > 20: |
| 124 |
table.decompose() |
| 125 |
for dateline in soup.findAll(attrs={'data-gu-name': 'dateline'}): |
| 126 |
for s in dateline.findAll('summary'): |
| 127 |
s.extract() |
| 128 |
dateline.name = 'div' |
| 129 |
return soup |