Debian Codegraph @

recipes/mwjournal.recipe

raw
1 #!/usr/bin/env python
2 ##
3 # Title: Microwave Journal
4 # Contact: Kiavash (use Mobile Read)
5 ##
6 # License: GNU General Public License v3 - http://www.gnu.org/copyleft/gpl.html
7 # Copyright: Kiavash
8 ##
9 # Written: Jan 2012
10 # Last Edited: Feb 2012
11 ##
12
13 # Feb 2012: New Recipe compatible with the MWJournal 2.0 website
14
15 __license__ = 'GNU General Public License v3 - http://www.gnu.org/copyleft/gpl.html'
16 __copyright__ = 'Kiavash'
17 __author__ = 'Kaivash'
18
19 '''
20 microwavejournal.com
21 '''
22
23 import re
24
25 from calibre.web.feeds.news import BasicNewsRecipe
26
27
28 class MWJournal(BasicNewsRecipe):
29 title = 'Microwave Journal'
30 description = 'Microwave Journal Monthly Magazine'
31 publisher = 'Horizon House'
32 publication_type = 'magazine'
33 INDEX = 'http://www.microwavejournal.com/publications/'
34
35 language = 'en'
36 timeout = 30
37
38 Convert_Grayscale = False # Convert images to gray scale or not
39
40 keep_only_tags = [dict(name='div', attrs={'class': 'record'})]
41 no_stylesheets = True
42 remove_javascript = True
43 remove_tags = [
44 dict(name='font', attrs={'class': 'footer'}), # remove fonts
45 ]
46
47 remove_attributes = ['border', 'cellspacing', 'align', 'cellpadding', 'colspan', 'valign', 'vspace', 'hspace', 'alt', 'width', 'height']
48
49 # Specify extra CSS - overrides ALL other CSS (IE. Added last).
50 extra_css = '''body { font-family: verdana, helvetica, sans-serif; }
51 .introduction, .first { font-weight: bold; }
52 .cross-head { font-weight: bold; font-size: 125%; }
53 .cap, .caption { display: block; font-size: 80%; font-style: italic; }
54 .cap, .caption, .caption img, .caption span { display: block; margin: 5px auto; }
55 .byl, .byd, .byline img, .byline-name, .byline-title, .author-name, .author-position,
56 .correspondent-portrait img, .byline-lead-in, .name, .bbc-role { display: block;
57 font-size: 80%; font-style: italic; margin: 1px auto; }
58 .story-date, .published { font-size: 80%; }
59 table { width: 100%; }
60 td img { display: block; margin: 5px auto; }
61 ul { padding-top: 10px; }
62 ol { padding-top: 10px; }
63 li { padding-top: 5px; padding-bottom: 5px; }
64 h1 { font-size: 175%; font-weight: bold; }
65 h2 { font-size: 150%; font-weight: bold; }
66 h3 { font-size: 125%; font-weight: bold; }
67 h4, h5, h6 { font-size: 100%; font-weight: bold; }'''
68
69 # Remove the line breaks, href links and float left/right and picture
70 # width/height.
71 preprocess_regexps = [
72 (re.compile(r'<br[ ]*/>', re.IGNORECASE), lambda m: ''),
73 (re.compile(r'<br[ ]*clear.*/>', re.IGNORECASE), lambda m: ''),
74 (re.compile(r'<a.*?>'), lambda h1: ''),
75 (re.compile(r'</a>'), lambda h2: ''),
76 (re.compile(r'float:.*?'), lambda h3: ''),
77 (re.compile(r'width:.*?px'), lambda h4: ''),
78 (re.compile(r'height:.*?px'), lambda h5: ''),
79 ]
80
81 def print_version(self, url):
82 return url.replace('/articles/', '/articles/print/')
83
84 def parse_index(self):
85 articles = []
86
87 soup = self.index_to_soup(self.INDEX)
88 ts = soup.find('div', attrs={'class': 'box1 article publications-show'})
89 ds = self.tag_to_string(ts.find('h2'))
90 self.log('Found Current Issue:', ds)
91 self.timefmt = ' [%s]' % ds
92
93 cover = ts.find('img', src=True)
94 if cover is not None:
95 self.cover_url = 'http://www.microwavejournal.com' + cover['src']
96 self.log('Found Cover image:', self.cover_url)
97
98 feeds = []
99 seen_titles = set() # This is used to remove duplicant articles
100 sections = soup.find('div', attrs={'class': 'box2 publication'})
101 for section in sections.findAll('div', attrs={'class': 'records'}):
102 section_title = self.tag_to_string(section.find('h3'))
103 self.log('Found section:', section_title)
104 articles = []
105 for post in section.findAll('div', attrs={'class': 'record'}):
106 h = post.find('h2')
107 title = self.tag_to_string(h)
108 # Let's get rid of the useless Puzzler!
109 if title.find('The MWJ Puzzler') >= 0:
110 continue
111 if title in seen_titles:
112 continue
113 seen_titles.add(title)
114 a = post.find('a', href=True)
115 url = a['href']
116 if url.startswith('/'):
117 url = 'http://www.microwavejournal.com' + url
118 abstract = post.find('div', attrs={'class': 'abstract'})
119 p = abstract.find('p')
120 desc = None
121 self.log('\tFound article:', title, 'at', url)
122 if p is not None:
123 desc = self.tag_to_string(p)
124 self.log('\t\t', desc)
125 articles.append({'title': title, 'url': url, 'description': desc, 'date': self.timefmt})
126 if articles:
127 feeds.append((section_title, articles))
128 return feeds