Debian Codegraph @

recipes/nymag.recipe

raw
1 #!/usr/bin/env python
2
3 __license__ = 'GPL v3'
4 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
5 '''
6 theatlantic.com
7 '''
8 from calibre.web.feeds.news import BasicNewsRecipe
9
10
11 def classes(classes):
12 q = frozenset(classes.split(' '))
13 return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)})
14
15
16 class NewYorkMagazine(BasicNewsRecipe):
17 title = 'New York Magazine'
18 __author__ = 'Kovid Goyal'
19 description = 'Food, culture, arts and entertainment in New York'
20 language = 'en_US'
21 no_stylesheets = True
22 remove_javascript = True
23 encoding = 'utf-8'
24 extra_css = '''
25 .nym-image-figcaption,
26 .bylines, .rubric,
27 .clay-paragraph_prologue,
28 .secondary-area-caption-credits {
29 font-size: small;
30 }
31 '''
32 keep_only_tags = [dict(name='article', attrs={'class': lambda x: x and 'article' in x.split()})]
33 remove_tags = [
34 dict(name=['svg', 'iframe']),
35 classes('related-stories start-discussion newsletter-flex-text package-toc comments-link tags related secondary-area author-photo error-pop-up'),
36 dict(id=['minibrowserbox', 'article-related', 'article-tools']),
37 ]
38 remove_attributes = ['style', 'height', 'width', 'srcset']
39
40 recipe_specific_options = {
41 'date': {
42 'short': 'The date of the edition to download (YYYY-MM-DD format)',
43 'long': 'For example, 2024-07-01',
44 }
45 }
46
47 def nymag_get_index(self):
48 issue_url = 'https://nymag.com/maglinks/nym-home-05'
49 d = self.recipe_specific_options.get('date')
50 if d and isinstance(d, str):
51 issue_url = 'https://nymag.com/magazine/toc/' + d + '.html'
52 return self.index_to_soup(issue_url)
53
54 def parse_index(self):
55 soup = self.nymag_get_index()
56 cdiv = soup.find(**classes('magazine-toc-cover-image-wrap'))
57 if cdiv is not None:
58 for source in cdiv.findAll('source', srcset=True):
59 self.cover_url = source['srcset'].split()[0]
60 self.log('Cover:', self.cover_url)
61 break
62 feeds = []
63 if cover_art := soup.find(**classes('magazine-toc-cover-text')):
64 a = cover_art.find('a', **classes('headline-link'))
65 c_url = a['href']
66 c_title = self.tag_to_string(a.find(**classes('magazine-toc-cover-headline'))).strip()
67 c_desc = self.tag_to_string(a.find(**classes('magazine-toc-cover-teaser'))).strip()
68 self.log('Cover Story', '\n\t', c_title, c_url)
69 feeds.append((
70 'Cover Story',
71 [{'title': c_title, 'url': c_url, 'description': c_desc}],
72 ))
73 for div in soup.findAll(attrs={'data-editable': 'settingTitle'}):
74 section = self.tag_to_string(div).strip().capitalize()
75 articles = []
76 self.log(section)
77 ul = div.findNextSibling('ul')
78 for li in ul.findAll('li'):
79 a = li.find(href=True, **classes('article-link'))
80 if a is None:
81 continue
82 url = a['href']
83 h3 = li.find('h3')
84 title = self.tag_to_string(h3)
85 desc = ''
86 teaser = h3.findNextSibling(**classes('teaser'))
87 if teaser is not None:
88 desc = self.tag_to_string(teaser)
89 self.log('\t', title, url)
90 articles.append({'title': title, 'url': url, 'description': desc})
91 if articles:
92 feeds.append((section, articles))
93
94 return feeds
95
96 def preprocess_html(self, soup):
97 if lede := soup.findAll('div', attrs={'class': lambda x: x and 'lede-image-wrapper' in x.split()}):
98 if len(lede) > 1:
99 lede[1].extract()
100 for img in soup.findAll('img', attrs={'data-src': True}):
101 img['src'] = img['data-src']
102 for h2 in soup.findAll(['h2', 'h3']):
103 h2.name = 'h4'
104 return soup