Debian Codegraph @

recipes/endgadget.recipe

raw
1 #!/usr/bin/env python
2
3 __license__ = 'GPL v3'
4 __copyright__ = 'Copyright 2011 Starson17'
5 '''
6 engadget.com
7 '''
8
9 from calibre.web.feeds.news import BasicNewsRecipe
10
11
12 def classes(classes):
13 q = frozenset(classes.split(' '))
14 return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)})
15
16
17 class Engadget(BasicNewsRecipe):
18 title = 'Engadget'
19 __author__ = 'Starson17, modified by epubli'
20 __version__ = 'v2.2'
21 __date__ = '2024-09-15'
22 description = 'Tech news'
23 language = 'en'
24 oldest_article = 7
25 max_articles_per_feed = 100
26 no_stylesheets = True
27 use_embedded_content = False
28 remove_javascript = True
29 remove_empty_feeds = True
30 compress_news_images = True
31 scale_news_images_to_device = True
32 cover_url = 'https://upload.wikimedia.org/wikipedia/commons/b/bb/Engadget-logo.svg'
33
34 keep_only_tags = [classes('caas-content-wrapper caas-title-wrapper'), dict(name='figure')]
35 remove_tags = [
36 dict(name='div', attrs={'class': 'caas-content-byline-wrapper'}),
37 dict(name='div', attrs={'data-component': 'ArticleAuthorInfo'}),
38 classes(
39 'commerce-module caas-header caas-prestige-bottom-share caas-share-buttons caas-da'
40 ' caas-3p-blocked commerce-disclaimer notification-upsell-push'
41 ' article-slideshow athena-button email-form'
42 ),
43 ]
44
45 feeds = [('Posts', 'https://www.engadget.com/rss.xml')]
46
47 def parse_feeds(self):
48 # Call parent's method.
49 feeds = BasicNewsRecipe.parse_feeds(self)
50 # Loop through all feeds.
51 for feed in feeds:
52 # Loop through all articles in feed.
53 for article in feed.articles[:]:
54 # Remove articles with '...' in the url.
55 if '/deals/' in article.url:
56 print('Removing:', article.title)
57 feed.articles.remove(article)
58 # Remove articles with '...' in the title.
59 elif 'best tech deals' in article.title:
60 print('Removing:', article.title)
61 feed.articles.remove(article)
62 elif 'Podcast' in article.title:
63 print('Removing:', article.title)
64 feed.articles.remove(article)
65 elif 'The Morning After' in article.title:
66 print('Removing:', article.title)
67 feed.articles.remove(article)
68 return feeds
69
70 def preprocess_html(self, soup):
71 for attr in 'data-src data-src-mobile'.split():
72 for img in soup.findAll('img'):
73 try:
74 ds = img[attr].split()[0]
75 del img[attr]
76 except KeyError:
77 continue
78 if ds:
79 index = ds.rfind('https://')
80 if index != -1:
81 imgsrc = ds[index:]
82 img['src'] = imgsrc
83 for divs in soup.findAll('div'):
84 try:
85 if divs['style'].split()[0].startswith('padding'):
86 print('Removing padding')
87 del divs['style']
88 except KeyError:
89 continue
90 # Reorder the "title" and "content" elements
91 title_div = soup.find('div', {'class': 'caas-title-wrapper'})
92 content_div = soup.find('div', {'class': 'caas-content-wrapper'})
93 if title_div and content_div:
94 soup.body.clear()
95 soup.body.append(title_div)
96 soup.body.append(content_div)
97 return soup