Debian Codegraph @

recipes/al_monitor.recipe

raw
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3 from __future__ import print_function, unicode_literals
4
5 __license__ = 'GPL v3'
6 __copyright__ = '2014, spswerling'
7 '''
8 http://www.al-monitor.com/
9 '''
10 import datetime
11 import inspect
12 import re
13 import string
14
15 from calibre.ebooks.BeautifulSoup import BeautifulSoup
16 from calibre.web.feeds.news import BasicNewsRecipe
17
18
19 class AlMonitor(BasicNewsRecipe):
20 title = 'Al Monitor'
21 __author__ = 'spswerling'
22 description = 'a The Pulse of the Middle East'
23 no_stylesheets = True
24 encoding = 'utf-8'
25 category = 'news'
26 language = 'en'
27 publication_type = 'newspaper'
28 cover_img_url = 'http://www.al-monitor.com/modules/almcontent/a-img/elements/logo.png'
29 masthead_url = cover_img_url
30 remove_empty_feeds = True
31
32 # on kindle, images can make things kind of fat. Slim them down.
33 recursions = 0
34 compress_news_images = True
35 compress_news_images_max_size = 7
36 scale_news_images = (150, 200) # (kindle touch: 600x800)
37 useHighResImages = False
38 oldest_article = 1.5
39 max_articles_per_section = 15
40
41 sections = [
42 ('egypt', 'http://www.al-monitor.com/pulse/egypt-pulse'),
43 ('gulf', 'http://www.al-monitor.com/pulse/gulf-pulse'),
44 ('iran', 'http://www.al-monitor.com/pulse/iran-pulse'),
45 ('iraq', 'http://www.al-monitor.com/pulse/iraq-pulse'),
46 ('israel', 'http://www.al-monitor.com/pulse/israel-pulse'),
47 ('lebanon', 'http://www.al-monitor.com/pulse/lebanon-pulse'),
48 ('palistine', 'http://www.al-monitor.com/pulse/palistine-pulse'),
49 ('syria', 'http://www.al-monitor.com/pulse/syria-pulse'),
50 ('turkey', 'http://www.al-monitor.com/pulse/turkey-pulse'),
51 ]
52
53 # util for creating remove_tags and keep_tags style regex matchers
54 def tag_matcher(elt, attr, rgx_str):
55 return dict(name=elt, attrs={attr: re.compile(rgx_str, re.IGNORECASE)})
56
57 remove_tags = [
58 dict(
59 attrs={
60 'id': [
61 'header',
62 'pulsebanner',
63 'relatedarticles',
64 'sidecolumn',
65 'disqus',
66 'footer',
67 'footer2',
68 'footer3',
69 'mobile-extras',
70 ]
71 }
72 ),
73 tag_matcher('hr', 'id', 'spacer'),
74 tag_matcher('a', 'title', 'print this article'),
75 tag_matcher('div', 'class', 'extras'),
76 tag_matcher('div', 'class', '^clear$'),
77 tag_matcher('div', 'class', '^overlay$'),
78 tag_matcher('div', 'class', 'shareTag'),
79 ]
80
81 articles = {}
82 urls_done = []
83
84 def parse_index(self):
85 for section in self.sections:
86 self.parse_section(section[0], section[1])
87 ans = []
88 for k in self.articles:
89 ans.append((string.capwords(k), self.articles[k]))
90 return ans
91
92 def parse_section(self, section, url):
93
94 self.articles[section] = []
95
96 try:
97 self._p('process section ' + section + ', url: ' + url)
98 soup = self.index_to_soup(url)
99 except Exception:
100 self._p('Unable to spider section')
101 return []
102
103 self._p('Got section. Processing links.')
104
105 for link in soup.findAll('a', href=True):
106 href = link.get('href')
107 text = self.text(link)
108 if text and ('pulse/originals' in href):
109 self.process_link(section, link)
110
111 def process_link(self, section, link):
112 title = self.text(link)
113 if len(title) > 120:
114 title = title[0:120] + '...'
115 href = link.get('href')
116 if not href:
117 self._p('BAD HREF: ' + str(link))
118 return
119 self.queue_article_link(section, href, title)
120
121 def queue_article_link(self, section, url, title):
122 full_url = self.abs_url(url)
123 if full_url in self.urls_done:
124 self._p('Skip (already Qd): ' + ' - '.join([section, title, url]))
125 return
126
127 self._p('Q: ' + ' - '.join([section, title, url]))
128 self.urls_done.append(full_url)
129 if len(self.articles[section]) >= self.max_articles_per_section:
130 return
131 self.articles[section].append(dict(title=title, url=full_url, date='', description='', author='', content=''))
132
133 def preprocess_raw_html(self, raw_html, url):
134 reason_to_skip = self.should_skip_article(BeautifulSoup(raw_html))
135 if reason_to_skip:
136 self._p('Skipping article: ' + reason_to_skip + ', ' + url)
137 # Next line will show up as an error in the logs, but ignore, see
138 # http://www.mobileread.com/forums/showthread.php?p=2931136
139 return None
140 else:
141 return super(self.__class__, self).preprocess_raw_html(raw_html, url)
142
143 def populate_article_metadata(self, article, soup, first):
144 summary_node = soup.find('div', {'id': 'summary'})
145 if summary_node:
146 summary = self.text(summary_node)
147 self._p('Summary: ' + summary)
148 article.text_summary = summary
149 else:
150 self._p('No summary')
151
152 def should_skip_article(self, soup):
153 date = self.scrape_article_date(soup)
154 if not date:
155 return False
156
157 age = (datetime.datetime.now() - date).days
158 if age > self.oldest_article:
159 return 'too old'
160 return False
161
162 def scrape_article_date(self, soup):
163 for span in soup.findAll('span'):
164 txt = self.text(span)
165 rgx = re.compile(r'Posted ([a-zA-Z]+ \d\d?, \d\d\d\d).*')
166 hit = rgx.match(txt)
167 if hit:
168 return self.date_from_string(txt)
169
170 return None
171
172 def date_from_string(self, datestring):
173 try:
174 # eg: Posted September 17, 2014
175 dt = datetime.datetime.strptime(datestring, 'Posted %B %d, %Y')
176 except Exception:
177 dt = None
178
179 if dt:
180 self._p('From string "' + datestring + '", datetime: ' + str(dt))
181 else:
182 self._p('Could not get datetime from ' + datestring)
183
184 return dt
185
186 def abs_url(self, url):
187 if 'www.al-monitor.com' in url:
188 abs_url = url
189 elif url[0] == '/':
190 abs_url = 'http://www.al-monitor.com' + url
191 else:
192 self._p('Not sure how to make abs_url: ' + url)
193 raise ValueError('Not sure how to make abs_url: ' + url)
194
195 if '#' in abs_url:
196 abs_url = ''.join(abs_url.split('#')[0:-1])
197
198 return abs_url
199
200 def text(self, n):
201 return self.tag_to_string(n).strip()
202
203 def _dbg_soup_node(self, node):
204 return (
205 ' cls: '
206 + str(node.get('class')).strip()
207 + ' id: '
208 + str(node.get('id')).strip()
209 + ' role: '
210 + str(node.get('role')).strip()
211 + ' txt: '
212 + self.text(node)
213 )
214
215 def _p(self, msg):
216 curframe = inspect.currentframe()
217 calframe = inspect.getouterframes(curframe, 2)
218 calname = calframe[1][3].upper()
219 print('[' + calname + '] ' + msg[0:100])