| 1 |
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai |
| 2 |
from __future__ import unicode_literals |
| 3 |
|
| 4 |
__license__ = 'GPL v3' |
| 5 |
__copyright__ = '2012, Onur Gungor [email protected]' |
| 6 |
__docformat__ = 'restructuredtext en' |
| 7 |
|
| 8 |
''' |
| 9 |
www.sol.org.tr |
| 10 |
''' |
| 11 |
|
| 12 |
import datetime |
| 13 |
import re |
| 14 |
|
| 15 |
from calibre.web.feeds.recipes import BasicNewsRecipe |
| 16 |
|
| 17 |
|
| 18 |
class SolHaberRecipe(BasicNewsRecipe): |
| 19 |
title = 'soL Haber' |
| 20 |
oldest_article = 7 |
| 21 |
max_articles_per_feed = 100 |
| 22 |
|
| 23 |
language = 'tr' |
| 24 |
__author__ = 'Onur Güngör' |
| 25 |
description = "Hayata soL'dan bakın.." |
| 26 |
publisher = 'soL Haber' |
| 27 |
tags = 'news, haberler, siyaset, türkiye, turkey, politics' |
| 28 |
|
| 29 |
conversion_options = {'comment': description, 'tags': tags, 'publisher': publisher, 'language': language} |
| 30 |
|
| 31 |
category_dict = { |
| 32 |
'sonuncu-kavga': 'Sonuncu Kavga', |
| 33 |
'devlet-ve-siyaset': 'Devlet ve Siyaset', |
| 34 |
'ekonomi': 'Ekonomi', |
| 35 |
'enternasyonal-gundem': 'Enternasyonel Gündem', |
| 36 |
'kent-gundemleri': 'Kent Gündemleri', |
| 37 |
'kultur-sanat': 'Kültür Sanat', |
| 38 |
'dunyadan': 'Dünyadan', |
| 39 |
'serbest-kursu': 'Serbest Kürsü', |
| 40 |
'medya': 'Medya', |
| 41 |
'liseliler': 'Liseliler', |
| 42 |
'yazarlar': 'Köşe Yazıları', |
| 43 |
} |
| 44 |
|
| 45 |
end_date = datetime.date.today().isoformat() |
| 46 |
start_date = (datetime.date.today() - datetime.timedelta(days=1)).isoformat() |
| 47 |
|
| 48 |
section_tuples = [ |
| 49 |
[ |
| 50 |
'Köşe Yazıları', |
| 51 |
'http://haber.sol.org.tr/arsiv?icerik=kose_yazisi&tarih%%5Bmin%%5D%%5Bdate%%5D=%s&tarih%%5Bmax%%5D%%5Bdate%%5D=%s' % (start_date, end_date), |
| 52 |
], # noqa: E501 |
| 53 |
['Haberler', 'http://haber.sol.org.tr/arsiv?icerik=haber&tarih%%5Bmin%%5D%%5Bdate%%5D=%s&tarih%%5Bmax%%5D%%5Bdate%%5D=%s' % (start_date, end_date)], |
| 54 |
['soL postal', 'http://haber.sol.org.tr/arsiv?icerik=postal&tarih%%5Bmin%%5D%%5Bdate%%5D=%s&tarih%%5Bmax%%5D%%5Bdate%%5D=%s' % (start_date, end_date)], |
| 55 |
[ |
| 56 |
'Bizim Amerika', |
| 57 |
'http://haber.sol.org.tr/arsiv?icerik=bizim_amerika&tarih%%5Bmin%%5D%%5Bdate%%5D=%s&tarih%%5Bmax%%5D%%5Bdate%%5D=%s' % (start_date, end_date), |
| 58 |
], |
| 59 |
] # noqa: E501 |
| 60 |
|
| 61 |
# Disable stylesheets from site. |
| 62 |
no_stylesheets = True |
| 63 |
|
| 64 |
cover_margins = (20, 20, '#ffffff') |
| 65 |
|
| 66 |
storybody_reg_exp = r'^\s*(haber|kose)\s*$' |
| 67 |
|
| 68 |
comments_reg_exp = r'^\s*makale-elestiri\s*$' |
| 69 |
|
| 70 |
remove_tags = [dict(name='div', attrs={'class': re.compile(comments_reg_exp, re.IGNORECASE)})] |
| 71 |
|
| 72 |
keep_only_tags = [dict(name='div', attrs={'class': re.compile(storybody_reg_exp, re.IGNORECASE)})] |
| 73 |
|
| 74 |
def get_masthead_title(self): |
| 75 |
return self.title + '(' + self.end_date + ')' |
| 76 |
|
| 77 |
def parse_index(self): |
| 78 |
|
| 79 |
result = [] |
| 80 |
articles_dict = {} |
| 81 |
|
| 82 |
author_regexp = re.compile(r'^http://.*?/yazarlar/(.*?)/.*$') |
| 83 |
category_regexp = re.compile(r'^http://.*?/(.+?)/.*$') |
| 84 |
|
| 85 |
for section_tuple in self.section_tuples: |
| 86 |
section_title = section_tuple[0] |
| 87 |
section_index_url = section_tuple[1] |
| 88 |
|
| 89 |
self.log('Bölüm:', section_title, 'URL:', section_index_url) |
| 90 |
|
| 91 |
soup = self.index_to_soup(section_index_url) |
| 92 |
|
| 93 |
logo = soup.find('div', id='logo').find('img', src=True) |
| 94 |
if logo is not None: |
| 95 |
self.cover_url = logo['src'] |
| 96 |
if self.cover_url.startswith('/'): |
| 97 |
self.cover_url = 'http://haber.sol.org.tr' + self.cover_url |
| 98 |
|
| 99 |
view_content = soup.find('div', id='ana-icerik').find('div', attrs={'class': 'view-content'}) |
| 100 |
if view_content is None: |
| 101 |
break |
| 102 |
rows = view_content.find('tbody').findAll('tr') |
| 103 |
|
| 104 |
self.log('Row sayısı', len(rows)) |
| 105 |
for row in rows: |
| 106 |
cells = row.findAll('td') |
| 107 |
|
| 108 |
a = cells[1].find('a', href=True) |
| 109 |
|
| 110 |
url = a['href'] |
| 111 |
title = self.tag_to_string(a) |
| 112 |
|
| 113 |
if url.startswith('/'): |
| 114 |
url = 'http://haber.sol.org.tr' + url |
| 115 |
|
| 116 |
category = section_title |
| 117 |
category_match_result = category_regexp.match(url) |
| 118 |
if category_match_result: |
| 119 |
category = category_match_result.group(1) |
| 120 |
|
| 121 |
date = self.tag_to_string(cells[2]) |
| 122 |
|
| 123 |
author = 'soL haber' |
| 124 |
|
| 125 |
author_match_result = author_regexp.match(url) |
| 126 |
if author_match_result: |
| 127 |
author = author_match_result.group(1) |
| 128 |
|
| 129 |
self.log('\tFound article:', title, 'at', url, 'published at ', date, 'by', author) |
| 130 |
article = {'title': title, 'url': url, 'description': None, 'date': date, 'author': author} |
| 131 |
if category in articles_dict: |
| 132 |
articles_dict[category].append(article) |
| 133 |
else: |
| 134 |
articles_dict[category] = [article] |
| 135 |
|
| 136 |
for category, entrys in articles_dict.items(): |
| 137 |
if category in self.category_dict: |
| 138 |
result.append((self.category_dict[category], entrys)) |
| 139 |
else: |
| 140 |
result.append((category, entrys)) |
| 141 |
|
| 142 |
return result |