Debian Codegraph @

recipes/sol_haber.recipe

raw
1 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
2 from __future__ import unicode_literals
3
4 __license__ = 'GPL v3'
5 __copyright__ = '2012, Onur Gungor [email protected]'
6 __docformat__ = 'restructuredtext en'
7
8 '''
9 www.sol.org.tr
10 '''
11
12 import datetime
13 import re
14
15 from calibre.web.feeds.recipes import BasicNewsRecipe
16
17
18 class SolHaberRecipe(BasicNewsRecipe):
19 title = 'soL Haber'
20 oldest_article = 7
21 max_articles_per_feed = 100
22
23 language = 'tr'
24 __author__ = 'Onur Güngör'
25 description = "Hayata soL'dan bakın.."
26 publisher = 'soL Haber'
27 tags = 'news, haberler, siyaset, türkiye, turkey, politics'
28
29 conversion_options = {'comment': description, 'tags': tags, 'publisher': publisher, 'language': language}
30
31 category_dict = {
32 'sonuncu-kavga': 'Sonuncu Kavga',
33 'devlet-ve-siyaset': 'Devlet ve Siyaset',
34 'ekonomi': 'Ekonomi',
35 'enternasyonal-gundem': 'Enternasyonel Gündem',
36 'kent-gundemleri': 'Kent Gündemleri',
37 'kultur-sanat': 'Kültür Sanat',
38 'dunyadan': 'Dünyadan',
39 'serbest-kursu': 'Serbest Kürsü',
40 'medya': 'Medya',
41 'liseliler': 'Liseliler',
42 'yazarlar': 'Köşe Yazıları',
43 }
44
45 end_date = datetime.date.today().isoformat()
46 start_date = (datetime.date.today() - datetime.timedelta(days=1)).isoformat()
47
48 section_tuples = [
49 [
50 'Köşe Yazıları',
51 'http://haber.sol.org.tr/arsiv?icerik=kose_yazisi&tarih%%5Bmin%%5D%%5Bdate%%5D=%s&tarih%%5Bmax%%5D%%5Bdate%%5D=%s' % (start_date, end_date),
52 ], # noqa: E501
53 ['Haberler', 'http://haber.sol.org.tr/arsiv?icerik=haber&tarih%%5Bmin%%5D%%5Bdate%%5D=%s&tarih%%5Bmax%%5D%%5Bdate%%5D=%s' % (start_date, end_date)],
54 ['soL postal', 'http://haber.sol.org.tr/arsiv?icerik=postal&tarih%%5Bmin%%5D%%5Bdate%%5D=%s&tarih%%5Bmax%%5D%%5Bdate%%5D=%s' % (start_date, end_date)],
55 [
56 'Bizim Amerika',
57 'http://haber.sol.org.tr/arsiv?icerik=bizim_amerika&tarih%%5Bmin%%5D%%5Bdate%%5D=%s&tarih%%5Bmax%%5D%%5Bdate%%5D=%s' % (start_date, end_date),
58 ],
59 ] # noqa: E501
60
61 # Disable stylesheets from site.
62 no_stylesheets = True
63
64 cover_margins = (20, 20, '#ffffff')
65
66 storybody_reg_exp = r'^\s*(haber|kose)\s*$'
67
68 comments_reg_exp = r'^\s*makale-elestiri\s*$'
69
70 remove_tags = [dict(name='div', attrs={'class': re.compile(comments_reg_exp, re.IGNORECASE)})]
71
72 keep_only_tags = [dict(name='div', attrs={'class': re.compile(storybody_reg_exp, re.IGNORECASE)})]
73
74 def get_masthead_title(self):
75 return self.title + '(' + self.end_date + ')'
76
77 def parse_index(self):
78
79 result = []
80 articles_dict = {}
81
82 author_regexp = re.compile(r'^http://.*?/yazarlar/(.*?)/.*$')
83 category_regexp = re.compile(r'^http://.*?/(.+?)/.*$')
84
85 for section_tuple in self.section_tuples:
86 section_title = section_tuple[0]
87 section_index_url = section_tuple[1]
88
89 self.log('Bölüm:', section_title, 'URL:', section_index_url)
90
91 soup = self.index_to_soup(section_index_url)
92
93 logo = soup.find('div', id='logo').find('img', src=True)
94 if logo is not None:
95 self.cover_url = logo['src']
96 if self.cover_url.startswith('/'):
97 self.cover_url = 'http://haber.sol.org.tr' + self.cover_url
98
99 view_content = soup.find('div', id='ana-icerik').find('div', attrs={'class': 'view-content'})
100 if view_content is None:
101 break
102 rows = view_content.find('tbody').findAll('tr')
103
104 self.log('Row sayısı', len(rows))
105 for row in rows:
106 cells = row.findAll('td')
107
108 a = cells[1].find('a', href=True)
109
110 url = a['href']
111 title = self.tag_to_string(a)
112
113 if url.startswith('/'):
114 url = 'http://haber.sol.org.tr' + url
115
116 category = section_title
117 category_match_result = category_regexp.match(url)
118 if category_match_result:
119 category = category_match_result.group(1)
120
121 date = self.tag_to_string(cells[2])
122
123 author = 'soL haber'
124
125 author_match_result = author_regexp.match(url)
126 if author_match_result:
127 author = author_match_result.group(1)
128
129 self.log('\tFound article:', title, 'at', url, 'published at ', date, 'by', author)
130 article = {'title': title, 'url': url, 'description': None, 'date': date, 'author': author}
131 if category in articles_dict:
132 articles_dict[category].append(article)
133 else:
134 articles_dict[category] = [article]
135
136 for category, entrys in articles_dict.items():
137 if category in self.category_dict:
138 result.append((self.category_dict[category], entrys))
139 else:
140 result.append((category, entrys))
141
142 return result