| 1 |
#!/usr/bin/env python |
| 2 |
# -*- coding: utf-8 -*- |
| 3 |
from __future__ import print_function, unicode_literals |
| 4 |
|
| 5 |
__license__ = 'GPL v3' |
| 6 |
__copyright__ = '2014, spswerling' |
| 7 |
''' |
| 8 |
http://www.al-monitor.com/ |
| 9 |
''' |
| 10 |
import datetime |
| 11 |
import inspect |
| 12 |
import re |
| 13 |
import string |
| 14 |
|
| 15 |
from calibre.ebooks.BeautifulSoup import BeautifulSoup |
| 16 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 17 |
|
| 18 |
|
| 19 |
class AlMonitor(BasicNewsRecipe): |
| 20 |
title = 'Al Monitor' |
| 21 |
__author__ = 'spswerling' |
| 22 |
description = 'a The Pulse of the Middle East' |
| 23 |
no_stylesheets = True |
| 24 |
encoding = 'utf-8' |
| 25 |
category = 'news' |
| 26 |
language = 'en' |
| 27 |
publication_type = 'newspaper' |
| 28 |
cover_img_url = 'http://www.al-monitor.com/modules/almcontent/a-img/elements/logo.png' |
| 29 |
masthead_url = cover_img_url |
| 30 |
remove_empty_feeds = True |
| 31 |
|
| 32 |
# on kindle, images can make things kind of fat. Slim them down. |
| 33 |
recursions = 0 |
| 34 |
compress_news_images = True |
| 35 |
compress_news_images_max_size = 7 |
| 36 |
scale_news_images = (150, 200) # (kindle touch: 600x800) |
| 37 |
useHighResImages = False |
| 38 |
oldest_article = 1.5 |
| 39 |
max_articles_per_section = 15 |
| 40 |
|
| 41 |
sections = [ |
| 42 |
('egypt', 'http://www.al-monitor.com/pulse/egypt-pulse'), |
| 43 |
('gulf', 'http://www.al-monitor.com/pulse/gulf-pulse'), |
| 44 |
('iran', 'http://www.al-monitor.com/pulse/iran-pulse'), |
| 45 |
('iraq', 'http://www.al-monitor.com/pulse/iraq-pulse'), |
| 46 |
('israel', 'http://www.al-monitor.com/pulse/israel-pulse'), |
| 47 |
('lebanon', 'http://www.al-monitor.com/pulse/lebanon-pulse'), |
| 48 |
('palistine', 'http://www.al-monitor.com/pulse/palistine-pulse'), |
| 49 |
('syria', 'http://www.al-monitor.com/pulse/syria-pulse'), |
| 50 |
('turkey', 'http://www.al-monitor.com/pulse/turkey-pulse'), |
| 51 |
] |
| 52 |
|
| 53 |
# util for creating remove_tags and keep_tags style regex matchers |
| 54 |
def tag_matcher(elt, attr, rgx_str): |
| 55 |
return dict(name=elt, attrs={attr: re.compile(rgx_str, re.IGNORECASE)}) |
| 56 |
|
| 57 |
remove_tags = [ |
| 58 |
dict( |
| 59 |
attrs={ |
| 60 |
'id': [ |
| 61 |
'header', |
| 62 |
'pulsebanner', |
| 63 |
'relatedarticles', |
| 64 |
'sidecolumn', |
| 65 |
'disqus', |
| 66 |
'footer', |
| 67 |
'footer2', |
| 68 |
'footer3', |
| 69 |
'mobile-extras', |
| 70 |
] |
| 71 |
} |
| 72 |
), |
| 73 |
tag_matcher('hr', 'id', 'spacer'), |
| 74 |
tag_matcher('a', 'title', 'print this article'), |
| 75 |
tag_matcher('div', 'class', 'extras'), |
| 76 |
tag_matcher('div', 'class', '^clear$'), |
| 77 |
tag_matcher('div', 'class', '^overlay$'), |
| 78 |
tag_matcher('div', 'class', 'shareTag'), |
| 79 |
] |
| 80 |
|
| 81 |
articles = {} |
| 82 |
urls_done = [] |
| 83 |
|
| 84 |
def parse_index(self): |
| 85 |
for section in self.sections: |
| 86 |
self.parse_section(section[0], section[1]) |
| 87 |
ans = [] |
| 88 |
for k in self.articles: |
| 89 |
ans.append((string.capwords(k), self.articles[k])) |
| 90 |
return ans |
| 91 |
|
| 92 |
def parse_section(self, section, url): |
| 93 |
|
| 94 |
self.articles[section] = [] |
| 95 |
|
| 96 |
try: |
| 97 |
self._p('process section ' + section + ', url: ' + url) |
| 98 |
soup = self.index_to_soup(url) |
| 99 |
except Exception: |
| 100 |
self._p('Unable to spider section') |
| 101 |
return [] |
| 102 |
|
| 103 |
self._p('Got section. Processing links.') |
| 104 |
|
| 105 |
for link in soup.findAll('a', href=True): |
| 106 |
href = link.get('href') |
| 107 |
text = self.text(link) |
| 108 |
if text and ('pulse/originals' in href): |
| 109 |
self.process_link(section, link) |
| 110 |
|
| 111 |
def process_link(self, section, link): |
| 112 |
title = self.text(link) |
| 113 |
if len(title) > 120: |
| 114 |
title = title[0:120] + '...' |
| 115 |
href = link.get('href') |
| 116 |
if not href: |
| 117 |
self._p('BAD HREF: ' + str(link)) |
| 118 |
return |
| 119 |
self.queue_article_link(section, href, title) |
| 120 |
|
| 121 |
def queue_article_link(self, section, url, title): |
| 122 |
full_url = self.abs_url(url) |
| 123 |
if full_url in self.urls_done: |
| 124 |
self._p('Skip (already Qd): ' + ' - '.join([section, title, url])) |
| 125 |
return |
| 126 |
|
| 127 |
self._p('Q: ' + ' - '.join([section, title, url])) |
| 128 |
self.urls_done.append(full_url) |
| 129 |
if len(self.articles[section]) >= self.max_articles_per_section: |
| 130 |
return |
| 131 |
self.articles[section].append(dict(title=title, url=full_url, date='', description='', author='', content='')) |
| 132 |
|
| 133 |
def preprocess_raw_html(self, raw_html, url): |
| 134 |
reason_to_skip = self.should_skip_article(BeautifulSoup(raw_html)) |
| 135 |
if reason_to_skip: |
| 136 |
self._p('Skipping article: ' + reason_to_skip + ', ' + url) |
| 137 |
# Next line will show up as an error in the logs, but ignore, see |
| 138 |
# http://www.mobileread.com/forums/showthread.php?p=2931136 |
| 139 |
return None |
| 140 |
else: |
| 141 |
return super(self.__class__, self).preprocess_raw_html(raw_html, url) |
| 142 |
|
| 143 |
def populate_article_metadata(self, article, soup, first): |
| 144 |
summary_node = soup.find('div', {'id': 'summary'}) |
| 145 |
if summary_node: |
| 146 |
summary = self.text(summary_node) |
| 147 |
self._p('Summary: ' + summary) |
| 148 |
article.text_summary = summary |
| 149 |
else: |
| 150 |
self._p('No summary') |
| 151 |
|
| 152 |
def should_skip_article(self, soup): |
| 153 |
date = self.scrape_article_date(soup) |
| 154 |
if not date: |
| 155 |
return False |
| 156 |
|
| 157 |
age = (datetime.datetime.now() - date).days |
| 158 |
if age > self.oldest_article: |
| 159 |
return 'too old' |
| 160 |
return False |
| 161 |
|
| 162 |
def scrape_article_date(self, soup): |
| 163 |
for span in soup.findAll('span'): |
| 164 |
txt = self.text(span) |
| 165 |
rgx = re.compile(r'Posted ([a-zA-Z]+ \d\d?, \d\d\d\d).*') |
| 166 |
hit = rgx.match(txt) |
| 167 |
if hit: |
| 168 |
return self.date_from_string(txt) |
| 169 |
|
| 170 |
return None |
| 171 |
|
| 172 |
def date_from_string(self, datestring): |
| 173 |
try: |
| 174 |
# eg: Posted September 17, 2014 |
| 175 |
dt = datetime.datetime.strptime(datestring, 'Posted %B %d, %Y') |
| 176 |
except Exception: |
| 177 |
dt = None |
| 178 |
|
| 179 |
if dt: |
| 180 |
self._p('From string "' + datestring + '", datetime: ' + str(dt)) |
| 181 |
else: |
| 182 |
self._p('Could not get datetime from ' + datestring) |
| 183 |
|
| 184 |
return dt |
| 185 |
|
| 186 |
def abs_url(self, url): |
| 187 |
if 'www.al-monitor.com' in url: |
| 188 |
abs_url = url |
| 189 |
elif url[0] == '/': |
| 190 |
abs_url = 'http://www.al-monitor.com' + url |
| 191 |
else: |
| 192 |
self._p('Not sure how to make abs_url: ' + url) |
| 193 |
raise ValueError('Not sure how to make abs_url: ' + url) |
| 194 |
|
| 195 |
if '#' in abs_url: |
| 196 |
abs_url = ''.join(abs_url.split('#')[0:-1]) |
| 197 |
|
| 198 |
return abs_url |
| 199 |
|
| 200 |
def text(self, n): |
| 201 |
return self.tag_to_string(n).strip() |
| 202 |
|
| 203 |
def _dbg_soup_node(self, node): |
| 204 |
return ( |
| 205 |
' cls: ' |
| 206 |
+ str(node.get('class')).strip() |
| 207 |
+ ' id: ' |
| 208 |
+ str(node.get('id')).strip() |
| 209 |
+ ' role: ' |
| 210 |
+ str(node.get('role')).strip() |
| 211 |
+ ' txt: ' |
| 212 |
+ self.text(node) |
| 213 |
) |
| 214 |
|
| 215 |
def _p(self, msg): |
| 216 |
curframe = inspect.currentframe() |
| 217 |
calframe = inspect.getouterframes(curframe, 2) |
| 218 |
calname = calframe[1][3].upper() |
| 219 |
print('[' + calname + '] ' + msg[0:100]) |