Debian Codegraph @

recipes/satmagazine.recipe

raw
1 #!/usr/bin/env python
2 ##
3 # Title: SatMagazine
4 ##
5 # License: GNU General Public License v3 - http://www.gnu.org/copyleft/gpl.html
6 ##
7 # Written: Feb 2012
8 # Last Edited: Mar 2012
9 ##
10
11 # Feb 2012: Initial release
12
13 __license__ = 'GNU General Public License v3 - http://www.gnu.org/copyleft/gpl.html'
14
15 '''
16 satmagazine.com
17 '''
18
19 import re
20
21 from calibre.web.feeds.news import BasicNewsRecipe
22
23
24 class SatMagazine(BasicNewsRecipe):
25 title = 'SatMagazine'
26 description = 'North American Satellite Markets...'
27 publisher = 'Satnews Publishers'
28 publication_type = 'magazine'
29 INDEX = 'http://www.satmagazine.com/cgi-bin/display_edition.cgi'
30 __author__ = 'kiavash'
31
32 language = 'en'
33 asciiize = True
34 timeout = 120
35 simultaneous_downloads = 2
36
37 # Flattens all the tables to make it compatible with Nook
38 conversion_options = {'linearize_tables': True}
39
40 keep_only_tags = [dict(name='span', attrs={'class': 'story'})]
41
42 no_stylesheets = True
43 remove_javascript = True
44
45 remove_attributes = ['border', 'cellspacing', 'align', 'cellpadding', 'colspan', 'valign', 'vspace', 'hspace', 'alt', 'width', 'height']
46
47 # Specify extra CSS - overrides ALL other CSS (IE. Added last).
48 extra_css = '''body { font-family: verdana, helvetica, sans-serif; }
49 .introduction, .first { font-weight: bold; }
50 .cross-head { font-weight: bold; font-size: 125%; }
51 .cap, .caption { display: block; font-size: 80%; font-style: italic; }
52 .cap, .caption, .caption img, .caption span { display: block; margin: 5px auto; }
53 .byl, .byd, .byline img, .byline-name, .byline-title, .author-name, .author-position,
54 .correspondent-portrait img, .byline-lead-in, .name, .bbc-role { display: block;
55 font-size: 80%; font-style: italic; margin: 1px auto; }
56 .story-date, .published { font-size: 80%; }
57 table { width: 100%; }
58 td img { display: block; margin: 5px auto; }
59 ul { padding-top: 10px; }
60 ol { padding-top: 10px; }
61 li { padding-top: 5px; padding-bottom: 5px; }
62 h1 { font-size: 175%; font-weight: bold; }
63 h2 { font-size: 150%; font-weight: bold; }
64 h3 { font-size: 125%; font-weight: bold; }
65 h4, h5, h6 { font-size: 100%; font-weight: bold; }'''
66
67 # Remove the line breaks, href links and float left/right and picture
68 # width/height.
69 preprocess_regexps = [
70 (re.compile(r'<br[ ]*/>', re.IGNORECASE), lambda m: ''),
71 (re.compile(r'<br[ ]*clear.*/>', re.IGNORECASE), lambda m: ''),
72 (re.compile(r'<a.*?>'), lambda h1: ''),
73 (re.compile(r'</a>'), lambda h2: ''),
74 (re.compile(r'float:.*?'), lambda h3: ''),
75 (re.compile(r'width:.*?px'), lambda h4: ''),
76 (re.compile(r'height:.*?px'), lambda h5: ''),
77 ]
78
79 def parse_index(self):
80
81 article_info = []
82 feeds = []
83
84 soup = self.index_to_soup(self.INDEX)
85
86 # Find Cover image
87 cover = soup.find('img', src=True, alt='Cover Image')
88 if cover is not None:
89 self.cover_url = cover['src']
90 self.log('Found Cover image:', self.cover_url)
91
92 # main part of the site that has the articles
93 soup = soup.find('div', attrs={'id': 'middlecontent'})
94
95 # Find the Magazine date
96 # contains the string with the date
97 ts = soup.find('span', attrs={'class': 'master_heading'})
98 ds = ' '.join(self.tag_to_string(ts).strip().split()[:2])
99 self.log('Found Current Issue:', ds)
100 self.timefmt = ' [%s]' % ds
101
102 articles = soup.findAll('span', attrs={'class': 'heading'})
103
104 descriptions = soup.findAll('span', attrs={'class': 'story'})
105
106 title_number = 0
107
108 # Goes through all the articles one by one and sort them out
109 for article in articles:
110 title = self.tag_to_string(article)
111 url = article.find('a').get('href')
112
113 self.log('\tFound article:', title, 'at', url)
114 desc = self.tag_to_string(descriptions[title_number])
115
116 article_info.append({'title': title, 'url': url, 'description': desc, 'date': self.timefmt})
117
118 title_number = title_number + 1
119
120 if article_info:
121 feeds.append((self.title, article_info))
122
123 return feeds
124
125 def preprocess_html(self, soup):
126
127 # Finds all the images
128 for figure in soup.findAll('img', attrs={'src': True}):
129 # if the image is an ad then remove it.
130 if (figure['alt'].find('_ad_') >= 0) or (figure['alt'].find('_snipe_') >= 0):
131 del figure['src']
132 del figure['alt']
133 del figure['border']
134 del figure['hspace']
135 del figure['vspace']
136 del figure['align']
137 del figure['size']
138 figure.name = 'font'
139 continue
140
141 # adds /n before and after the image
142 figure['style'] = 'display:block'
143
144 # Makes the title standing out
145 for title in soup.findAll('b'):
146 title.name = 'h3'
147
148 # Removes all unrelated links
149 for link in soup.findAll('a', attrs={'href': True}):
150 link.name = 'font'
151 del link['href']
152 del link['target']
153
154 return soup