Debian Codegraph @

recipes/nyt_tmag.recipe

raw
1 #!/usr/bin/env python
2
3 from calibre.web.feeds.news import BasicNewsRecipe
4
5 use_wayback_machine = False
6
7
8 class NytMag(BasicNewsRecipe):
9 title = 'NYT T Magazine'
10 __author__ = 'unkn0wn'
11 description = 'The latest from The New York Times Style Magazine.'
12 oldest_article = 30 # days
13 encoding = 'utf-8'
14 use_embedded_content = False
15 language = 'en_US'
16 remove_empty_feeds = True
17 resolve_internal_links = True
18 ignore_duplicate_articles = {'title', 'url'}
19 masthead_url = 'https://static01.nytimes.com/newsgraphics/2015-12-23-masthead-2016/b15c3d81d3d7b59065fff9a3f3afe85aa2e2dff5/_assets/nyt-logo.png'
20 browser_type = 'webengine'
21
22 def get_cover_url(self):
23 soup = self.index_to_soup('https://www.nytimes.com/section/t-magazine')
24 issue = soup.find(attrs={'class': 'issue-promo'})
25 return issue.a.img['src']
26
27 feeds = [
28 'https://rss.nytimes.com/services/xml/rss/nyt/tmagazine.xml',
29 'https://rss.nytimes.com/services/xml/rss/nyt/FashionandStyle.xml',
30 ]
31
32 recipe_specific_options = {
33 'days': {
34 'short': 'Oldest article to download from this news source. In days ',
35 'long': 'For example, 0.5, gives you articles from the past 12 hours',
36 'default': str(oldest_article),
37 },
38 'comp': {'short': 'Compress News Images?', 'long': 'enter yes', 'default': 'no'},
39 'rev': {
40 'short': 'Reverse the order of articles in each feed?',
41 'long': 'enter yes',
42 'default': 'no',
43 },
44 'res': {
45 'short': ('For hi-res images, select a resolution from the following\noptions: popup, jumbo, mobileMasterAt3x, superJumbo'),
46 'long': (
47 'This is useful for non e-ink devices, and for a lower file size\nthan '
48 'the default, use mediumThreeByTwo440, mediumThreeByTwo225, articleInline.'
49 ),
50 },
51 }
52
53 def __init__(self, *args, **kwargs):
54 BasicNewsRecipe.__init__(self, *args, **kwargs)
55 d = self.recipe_specific_options.get('days')
56 if d and isinstance(d, str):
57 self.oldest_article = float(d)
58 r = self.recipe_specific_options.get('rev')
59 if r and isinstance(r, str):
60 if r.lower() == 'yes':
61 self.reverse_article_order = True
62 c = self.recipe_specific_options.get('comp')
63 if c and isinstance(c, str):
64 if c.lower() == 'yes':
65 self.compress_news_images = True
66
67 extra_css = '''
68 .byl, .time { font-size:small; color:#202020; }
69 .cap { font-size:small; text-align:center; }
70 .cred { font-style:italic; font-size:small; }
71 em, blockquote { color: #202020; }
72 .sc { font-variant: small-caps; }
73 .lbl { font-size:small; color:#404040; }
74 img { display:block; margin:0 auto; }
75 '''
76
77 @property
78 def nyt_parser(self):
79 ans = getattr(self, '_nyt_parser', None)
80 if ans is None:
81 from calibre.live import load_module
82
83 self._nyt_parser = ans = load_module('calibre.web.site_parsers.nytimes')
84 return ans
85
86 def get_nyt_page(self, url, skip_wayback=False):
87 if use_wayback_machine and not skip_wayback:
88 from calibre import browser
89
90 return self.nyt_parser.download_url(url, browser())
91 return self.index_to_soup(url, raw=True)
92
93 articles_are_obfuscated = use_wayback_machine
94
95 if use_wayback_machine:
96
97 def get_obfuscated_article(self, url):
98 from calibre.ptempfile import PersistentTemporaryFile
99
100 with PersistentTemporaryFile() as tf:
101 tf.write(self.get_nyt_page(url))
102 return tf.name
103
104 def preprocess_raw_html(self, raw_html, url):
105 return self.nyt_parser.extract_html(self.index_to_soup(raw_html), url)
106
107 def get_browser(self, *args, **kwargs):
108 # kwargs['user_agent'] = 'User-Agent: Mozilla/5.0 (compatible; archive.org_bot; Wayback Machine Live Record; +http://archive.org/details/archive.org_bot)'
109 br = BasicNewsRecipe.get_browser(self, *args, **kwargs)
110 return br
111
112 def preprocess_html(self, soup):
113 w = self.recipe_specific_options.get('res')
114 if w and isinstance(w, str):
115 res = '-' + w
116 for img in soup.findAll('img', attrs={'src': True}):
117 if '-article' in img['src']:
118 ext = img['src'].split('?')[0].split('.')[-1]
119 img['src'] = img['src'].rsplit('-article', 1)[0] + res + '.' + ext
120 for c in soup.findAll('div', attrs={'class': 'cap'}):
121 for p in c.findAll(['p', 'div']):
122 p.name = 'span'
123 return soup