Debian Codegraph @

recipes/smith.recipe

raw
1 from calibre.web.feeds.news import BasicNewsRecipe, classes
2
3 CATEGORIES = {
4 'smart-news': 'Smart News',
5 'history': 'History',
6 'science-nature': 'Science',
7 'innovation': 'Innovation',
8 'arts-culture': 'Arts & Culture',
9 'travel': 'Travel',
10 'smithsonian-institution': 'At the Smithsonian',
11 }
12
13
14 class Smithsonian(BasicNewsRecipe):
15 title = 'Smithsonian Magazine'
16 __author__ = 'Kovid Goyal'
17
18 description = 'This magazine chronicles the arts, environment, sciences and popular culture of the times. It is edited for modern, well-rounded individuals with diverse, general interests. With your order, you become a National Associate Member of the Smithsonian. Membership benefits include your subscription to Smithsonian magazine, a personalized membership card, discounts from the Smithsonian catalog, and more.' # noqa: E501
19 language = 'en'
20 category = 'news'
21 encoding = 'UTF-8'
22 keep_only_tags = [classes('article-header'), classes('articleLeft')]
23 remove_tags = [
24 dict(name='source'),
25 classes(
26 'hidden-phone hidden-tablet hidden-desktop slideshow-nav associated-container share-control'
27 ' widget-article-pixel tag-list recommended-videos comments widget-related-articles in-article-newsletter'
28 ),
29 ]
30 no_javascript = True
31 no_stylesheets = True
32
33 def parse_section(self, url):
34 soup = self.index_to_soup(url)
35 seen = set()
36 for article in soup.findAll(attrs={'class': 'content-list-item'}):
37 div = article.find(attrs={'class': 'content-list-text'})
38 if div is None:
39 continue
40 a = div.find('a')
41 if a is None:
42 continue
43 title = self.tag_to_string(a)
44 if title in seen:
45 continue
46 seen.add(title)
47 article_url = a['href']
48 if not article_url.startswith('http'):
49 article_url = 'https://www.smithsonianmag.com/' + article_url.lstrip('/')
50 if '/tag/' in article_url:
51 continue
52 desc = ''
53 p = div.find(attrs={'class': 'dek'})
54 if p is not None:
55 desc = self.tag_to_string(p)
56 self.log('\t' + title)
57 yield {'title': title, 'url': article_url, 'description': desc}
58
59 def parse_index(self):
60 ans = []
61 for slug, title in CATEGORIES.items():
62 url = 'https://www.smithsonianmag.com/category/' + slug + '/'
63 self.log('Parsing section:', title, 'at:', url)
64 articles = list(self.parse_section(url))
65 if articles:
66 ans.append((title, articles))
67 if self.test and len(ans) >= self.test[0]:
68 break
69 return ans