Debian Codegraph @

recipes/esenja.recipe

raw
1 #!/usr/bin/env python
2
3 __license__ = 'GPL v3'
4 __copyright__ = '2010, matek09, [email protected]'
5
6 import re
7
8 from calibre.ebooks.BeautifulSoup import BeautifulSoup, Comment
9 from calibre.web.feeds.news import BasicNewsRecipe
10
11
12 class Esensja(BasicNewsRecipe):
13 title = 'Esensja'
14 __author__ = 'matek09 & fenuks'
15 description = 'Magazyn kultury popularnej'
16 encoding = 'utf-8'
17 no_stylesheets = True
18 language = 'pl'
19 remove_javascript = True
20 masthead_url = 'http://esensja.pl/img/wrss.gif'
21 oldest_article = 1
22 URL = 'http://esensja.pl'
23 HREF = '0'
24 remove_attributes = ['style', 'bgcolor', 'alt', 'color']
25 keep_only_tags = [
26 dict(attrs={'class': 'sekcja'}),
27 ]
28 # keep_only_tags.append(dict(name = 'div', attrs = {'class' : 'article'})
29 remove_tags_after = dict(id='tekst')
30
31 remove_tags = [
32 dict(name='img', attrs={'src': ['../../../2000/01/img/tab_top.gif', '../../../2000/01/img/tab_bot.gif']}),
33 dict(name='div', attrs={'class': 't-title2 nextpage'}),
34 # dict(attrs={'rel':'lightbox[galeria]'})
35 dict(attrs={'class': ['tekst_koniec', 'ref', 'wykop']}),
36 dict(attrs={'itemprop': ['copyrightHolder', 'publisher']}),
37 dict(id='komentarze'),
38 ]
39
40 extra_css = '''
41 .t-title {font-size: x-large; font-weight: bold; text-align: left}
42 .t-author {font-size: x-small; text-align: left}
43 .t-title2 {font-size: x-small; font-style: italic; text-align: left}
44 .text {font-size: small; text-align: left}
45 .annot-ref {font-style: italic; text-align: left}
46 '''
47
48 preprocess_regexps = [(re.compile(r'alt="[^"]*"'), lambda match: ''), (re.compile(r'(title|alt)="[^"]*?"', re.DOTALL), lambda match: '')]
49
50 def parse_index(self):
51 soup = self.index_to_soup('http://www.esensja.pl/magazyn/')
52 a = soup.find('a', attrs={'href': re.compile(r'.*/index.html')})
53 year = a['href'].split('/')[0]
54 month = a['href'].split('/')[1]
55 self.HREF = 'http://www.esensja.pl/magazyn/' + year + '/' + month + '/iso/'
56 soup = self.index_to_soup(self.HREF + '01.html')
57 self.cover_url = 'http://www.esensja.pl/magazyn/' + year + '/' + month + '/img/ilustr/cover_b.jpg'
58 feeds = []
59 chapter = ''
60 subchapter = ''
61 articles = []
62 intro = soup.find('div', attrs={'class': 'n-title'})
63 '''
64 introduction = {'title' : self.tag_to_string(intro.a),
65 'url' : self.HREF + intro.a['href'],
66 'date' : '',
67 'description' : ''}
68 chapter = 'Wprowadzenie'
69 articles.append(introduction)
70 '''
71
72 for tag in intro.findAllNext(attrs={'class': ['chapter', 'subchapter', 'n-title']}):
73 if tag.name in 'td':
74 if len(articles) > 0:
75 section = chapter
76 if len(subchapter) > 0:
77 section += ' - ' + subchapter
78 feeds.append((section, articles))
79 articles = []
80 if ''.join(tag['class']) == 'chapter':
81 chapter = self.tag_to_string(tag).capitalize()
82 subchapter = ''
83 else:
84 subchapter = self.tag_to_string(tag)
85 subchapter = self.tag_to_string(tag)
86 continue
87
88 finalurl = tag.a['href']
89 if not finalurl.startswith('http'):
90 finalurl = self.HREF + finalurl
91 articles.append({'title': self.tag_to_string(tag.a), 'url': finalurl, 'date': '', 'description': ''})
92
93 a = self.index_to_soup(finalurl)
94 i = 1
95
96 while True:
97 div = a.find('div', attrs={'class': 't-title2 nextpage'})
98 if div is not None:
99 link = div.a['href']
100 if not link.startswith('http'):
101 link = self.HREF + link
102 a = self.index_to_soup(link)
103 articles.append({'title': self.tag_to_string(tag.a) + ' c. d. ' + str(i), 'url': link, 'date': '', 'description': ''})
104 i = i + 1
105 else:
106 break
107
108 return feeds
109
110 def append_page(self, soup, appendtag):
111 r = appendtag.find(attrs={'class': 'wiecej_xxx'})
112 if r:
113 nr = r.findAll(attrs={'class': 'tn-link'})[-1]
114 try:
115 nr = int(nr.a.string)
116 except Exception:
117 return
118 baseurl = soup.find(attrs={'property': 'og:url'})['content'] + '&strona={0}'
119 for number in range(2, nr + 1):
120 soup2 = self.index_to_soup(baseurl.format(number))
121 pagetext = soup2.find(attrs={'class': 'tresc'})
122 pos = len(appendtag.contents)
123 appendtag.insert(pos, pagetext)
124 for r in appendtag.findAll(attrs={'class': ['wiecej_xxx', 'tekst_koniec']}):
125 r.extract()
126 for r in appendtag.findAll('script'):
127 r.extract()
128
129 comments = appendtag.findAll(text=lambda text: isinstance(text, Comment))
130 for comment in comments:
131 comment.extract()
132
133 def preprocess_html(self, soup):
134 self.append_page(soup, soup.body)
135 for tag in soup.findAll(attrs={'class': 'img_box_right'}):
136 temp = tag.find('img')
137 src = ''
138 if temp:
139 src = temp.get('src', '')
140 for r in tag.findAll('a', recursive=False):
141 r.extract()
142 info = tag.find(attrs={'class': 'img_info'})
143 text = str(tag)
144 if not src:
145 src = re.search(r'src="[^"]*?"', text)
146 if src:
147 src = src.group(0)
148 src = src[5:].replace('//', '/')
149 if src:
150 tag.contents = []
151 tag.insert(0, BeautifulSoup('<img src="{0}{1}" />'.format(self.URL, src)))
152 if info:
153 tag.insert(len(tag.contents), info)
154 return soup