Debian Codegraph @

recipes/nytfeeds.recipe

raw
1 #!/usr/bin/env python
2 import re
3
4 from calibre.web.feeds.news import BasicNewsRecipe
5
6 use_wayback_machine = False
7
8
9 class NytFeeds(BasicNewsRecipe):
10 title = 'NYT News'
11 __author__ = 'unkn0wn'
12 description = (
13 'The New York Times is dedicated to helping people understand the world through '
14 'on-the-ground, expert and deeply reported independent journalism. Feeds based recipe.'
15 )
16 oldest_article = 1
17 encoding = 'utf-8'
18 use_embedded_content = False
19 language = 'en_US'
20 remove_empty_feeds = True
21 resolve_internal_links = True
22 ignore_duplicate_articles = {'title', 'url'}
23 masthead_url = 'https://static01.nytimes.com/newsgraphics/2015-12-23-masthead-2016/b15c3d81d3d7b59065fff9a3f3afe85aa2e2dff5/_assets/nyt-logo.png'
24
25 # https://www.nytimes.com/rss
26 # https://developer.nytimes.com/docs/rss-api/1/overview
27 feeds = [
28 # to filter out all opinions from other sections first
29 'https://rss.nytimes.com/services/xml/rss/nyt/Opinion.xml',
30 'https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml',
31 'https://rss.nytimes.com/services/xml/rss/nyt/World.xml',
32 'https://rss.nytimes.com/services/xml/rss/nyt/US.xml',
33 'https://rss.nytimes.com/services/xml/rss/nyt/Business.xml',
34 'https://rss.nytimes.com/services/xml/rss/nyt/YourMoney.xml',
35 'https://rss.nytimes.com/services/xml/rss/nyt/Science.xml',
36 'https://rss.nytimes.com/services/xml/rss/nyt/Climate.xml',
37 'https://rss.nytimes.com/services/xml/rss/nyt/Health.xml',
38 'https://rss.nytimes.com/services/xml/rss/nyt/Arts.xml',
39 'https://www.nytimes.com/services/xml/rss/nyt/Travel.xml',
40 'https://rss.nytimes.com/services/xml/rss/nyt/Sports.xml',
41 ]
42
43 def get_cover_url(self):
44 soup = self.index_to_soup('https://www.frontpages.com/us-newspapers/')
45 for img in soup.findAll('img', src=True):
46 if 'new-york-times' in img['src']:
47 return 'https://www.frontpages.com' + img['src']
48
49 recipe_specific_options = {
50 'days': {
51 'short': 'Oldest article to download from this news source. In days ',
52 'long': 'For example, 0.5, gives you articles from the past 12 hours',
53 'default': str(oldest_article),
54 },
55 'comp': {'short': 'Compress News Images?', 'long': 'enter yes', 'default': 'no'},
56 'rev': {
57 'short': 'Reverse the order of articles in each feed?',
58 'long': 'enter yes',
59 'default': 'no',
60 },
61 'res': {
62 'short': ('For hi-res images, select a resolution from the following\noptions: popup, jumbo, mobileMasterAt3x, superJumbo'),
63 'long': (
64 'This is useful for non e-ink devices, and for a lower file size\nthan '
65 'the default, use mediumThreeByTwo440, mediumThreeByTwo225, articleInline.'
66 ),
67 },
68 'cont': {
69 'short': 'Get all content?',
70 'long': 'enter yes. Includes all articles from all feeds... magazines, books etc.',
71 'default': 'no',
72 },
73 }
74
75 def __init__(self, *args, **kwargs):
76 BasicNewsRecipe.__init__(self, *args, **kwargs)
77 d = self.recipe_specific_options.get('days')
78 if d and isinstance(d, str):
79 self.oldest_article = float(d)
80 r = self.recipe_specific_options.get('rev')
81 if r and isinstance(r, str):
82 if r.lower() == 'yes':
83 self.reverse_article_order = True
84 c = self.recipe_specific_options.get('comp')
85 if c and isinstance(c, str):
86 if c.lower() == 'yes':
87 self.compress_news_images = True
88 a = self.recipe_specific_options.get('cont')
89 if a and isinstance(a, str):
90 if a.lower() == 'yes':
91 self.feeds.extend([
92 'https://rss.nytimes.com/services/xml/rss/nyt/Technology.xml',
93 'https://rss.nytimes.com/services/xml/rss/nyt/FashionandStyle.xml',
94 'https://rss.nytimes.com/services/xml/rss/nyt/tmagazine.xml',
95 'https://rss.nytimes.com/services/xml/rss/nyt/books.xml',
96 'https://www.nytimes.com/services/xml/rss/nyt/Travel.xml',
97 'https://rss.nytimes.com/services/xml/rss/nyt/well.xml',
98 'https://rss.nytimes.com/services/xml/rss/nyt/Magazine.xml',
99 'http://nytimes.com/timeswire/feeds/',
100 ])
101
102 extra_css = '''
103 .byl, .time { font-size:small; color:#202020; }
104 .cap { font-size:small; text-align:center; }
105 .cred { font-style:italic; font-size:small; }
106 em, blockquote { color: #202020; }
107 .sc { font-variant: small-caps; }
108 .lbl { font-size:small; color:#404040; }
109 img { display:block; margin:0 auto; }
110 '''
111
112 @property
113 def nyt_parser(self):
114 ans = getattr(self, '_nyt_parser', None)
115 if ans is None:
116 from calibre.live import load_module
117
118 self._nyt_parser = ans = load_module('calibre.web.site_parsers.nytimes')
119 return ans
120
121 def get_nyt_page(self, url, skip_wayback=False):
122 if use_wayback_machine and not skip_wayback:
123 from calibre import browser
124
125 return self.nyt_parser.download_url(url, browser())
126 return self.index_to_soup(url, raw=True)
127
128 articles_are_obfuscated = use_wayback_machine
129
130 if use_wayback_machine:
131
132 def get_obfuscated_article(self, url):
133 from calibre.ptempfile import PersistentTemporaryFile
134
135 with PersistentTemporaryFile() as tf:
136 tf.write(self.get_nyt_page(url))
137 return tf.name
138
139 def preprocess_raw_html(self, raw_html, url):
140 return self.nyt_parser.extract_html(self.index_to_soup(raw_html), url)
141
142 def get_browser(self, *args, **kwargs):
143 kwargs['user_agent'] = 'User-Agent: Mozilla/5.0 (compatible; archive.org_bot; Wayback Machine Live Record; +http://archive.org/details/archive.org_bot)'
144 br = BasicNewsRecipe.get_browser(self, *args, **kwargs)
145 return br
146
147 def preprocess_html(self, soup):
148 w = self.recipe_specific_options.get('res')
149 if w and isinstance(w, str):
150 res = '-' + w
151 for img in soup.findAll('img', attrs={'src': True}):
152 if '-article' in img['src']:
153 ext = img['src'].split('?')[0].split('.')[-1]
154 img['src'] = img['src'].rsplit('-article', 1)[0] + res + '.' + ext
155 for c in soup.findAll('div', attrs={'class': 'cap'}):
156 for p in c.findAll(['p', 'div']):
157 p.name = 'span'
158 return soup
159
160 def get_article_url(self, article):
161 url = BasicNewsRecipe.get_article_url(self, article)
162 # you can remove '|/espanol/' from code below to include spanish articles.
163 if not re.search(r'/video/|/live/|/athletic/|/espanol/|/card/', url):
164 return url
165 self.log('\tSkipped URL: ', url)
166 return None