Debian Codegraph @

recipes/daum_net.recipe

raw
1 import re
2 from datetime import date, timedelta
3
4 from calibre.web.feeds.recipes import BasicNewsRecipe
5
6
7 class MediaDaumRecipe(BasicNewsRecipe):
8 title = '\ubbf8\ub514\uc5b4 \ub2e4\uc74c \uc624\ub298\uc758 \uc8fc\uc694 \ub274\uc2a4'
9 description = 'Articles from media.daum.net'
10 __author__ = 'trustin'
11 language = 'ko'
12 max_articles = 100
13
14 timefmt = ''
15 masthead_url = 'http://img-media.daum-img.net/2010ci/service_news.gif'
16 cover_margins = (18, 18, 'grey99')
17 no_stylesheets = True
18 remove_tags_before = dict(id='GS_con')
19 remove_tags_after = dict(id='GS_con')
20 remove_tags = [
21 dict(
22 attrs={
23 'class': [
24 'bline',
25 'GS_vod',
26 ]
27 }
28 ),
29 dict(
30 id=[
31 'GS_swf_poll',
32 'ad250',
33 ]
34 ),
35 dict(name=['script', 'noscript', 'style', 'object']),
36 ]
37 preprocess_regexps = [
38 (re.compile(r'<\s+', re.DOTALL | re.IGNORECASE), lambda match: '&lt; '),
39 (re.compile(r'(<br[^>]*>[ \t\r\n]*){3,}', re.DOTALL | re.IGNORECASE), lambda match: ''),
40 (re.compile(r'(<br[^>]*>[ \t\r\n]*)*</div>', re.DOTALL | re.IGNORECASE), lambda match: '</div>'),
41 (re.compile(r'(<br[^>]*>[ \t\r\n]*)*</p>', re.DOTALL | re.IGNORECASE), lambda match: '</p>'),
42 (re.compile(r'(<br[^>]*>[ \t\r\n]*)*</td>', re.DOTALL | re.IGNORECASE), lambda match: '</td>'),
43 (re.compile(r'(<br[^>]*>[ \t\r\n]*)*</strong>', re.DOTALL | re.IGNORECASE), lambda match: '</strong>'),
44 (re.compile(r'(<br[^>]*>[ \t\r\n]*)*</b>', re.DOTALL | re.IGNORECASE), lambda match: '</b>'),
45 (re.compile(r'(<br[^>]*>[ \t\r\n]*)*</em>', re.DOTALL | re.IGNORECASE), lambda match: '</em>'),
46 (re.compile(r'(<br[^>]*>[ \t\r\n]*)*</i>', re.DOTALL | re.IGNORECASE), lambda match: '</i>'),
47 (re.compile(r'\(\uB05D\)[ \t\r\n]*<br[^>]*>.*</div>', re.DOTALL | re.IGNORECASE), lambda match: '</div>'),
48 (re.compile(r'(<br[^>]*>[ \t\r\n]*)*<div', re.DOTALL | re.IGNORECASE), lambda match: '<div'),
49 (re.compile(r'(<br[^>]*>[ \t\r\n]*)*<p', re.DOTALL | re.IGNORECASE), lambda match: '<p'),
50 (re.compile(r'(<br[^>]*>[ \t\r\n]*)*<table', re.DOTALL | re.IGNORECASE), lambda match: '<table'),
51 (re.compile(r'<strong>(<br[^>]*>[ \t\r\n]*)*', re.DOTALL | re.IGNORECASE), lambda match: '<strong>'),
52 (re.compile(r'<b>(<br[^>]*>[ \t\r\n]*)*', re.DOTALL | re.IGNORECASE), lambda match: '<b>'),
53 (re.compile(r'<em>(<br[^>]*>[ \t\r\n]*)*', re.DOTALL | re.IGNORECASE), lambda match: '<em>'),
54 (re.compile(r'<i>(<br[^>]*>[ \t\r\n]*)*', re.DOTALL | re.IGNORECASE), lambda match: '<i>'),
55 (
56 re.compile(
57 '(<br[^>]*>[ \t\r\n]*)*(\u25b6|\u25cf|\u261e|\u24d2|\\(c\\))*\\[[^\\]]*' # noqa: RUF039
58 '(\u24d2|\\(c\\)|\uae30\uc0ac|\uc778\uae30[^\\]]*\ub274\uc2a4)[^\\]]*\\].*</div>', # noqa: RUF039
59 re.DOTALL | re.IGNORECASE,
60 ),
61 lambda match: '</div>',
62 ),
63 ]
64
65 def parse_index(self):
66 today = date.today()
67 articles = []
68 articles = self.parse_list_page(articles, today)
69 articles = self.parse_list_page(articles, today - timedelta(1))
70 return [('\ubbf8\ub514\uc5b4 \ub2e4\uc74c \uc624\ub298\uc758 \uc8fc\uc694 \ub274\uc2a4', articles)]
71
72 def parse_list_page(self, articles, date):
73 if len(articles) >= self.max_articles:
74 return articles
75
76 for page in range(1, 10):
77 soup = self.index_to_soup(
78 'http://media.daum.net/primary/total/list.html?cateid=100044&date=%(date)s&page=%(page)d' % {'date': date.strftime('%Y%m%d'), 'page': page}
79 )
80 done = True
81 for item in soup.findAll('dl'):
82 dt = item.find('dt', {'class': 'tit'})
83 dd = item.find('dd', {'class': 'txt'})
84 if dt is None:
85 break
86 a = dt.find('a', href=True)
87 url = 'http://media.daum.net/primary/total/' + a['href']
88 title = self.tag_to_string(dt)
89 if dd is None:
90 description = ''
91 else:
92 description = self.tag_to_string(dd)
93 articles.append(dict(title=title, description=description, url=url, content=''))
94 done = len(articles) >= self.max_articles
95 if done:
96 break
97 if done:
98 break
99 return articles
100
101 def preprocess_html(self, soup):
102 return self.strip_anchors(soup)
103
104 def strip_anchors(self, soup):
105 for para in soup.findAll(True):
106 aTags = para.findAll('a')
107 for a in aTags:
108 if a.img is None:
109 a.replaceWith(a.renderContents().decode('utf-8', 'replace'))
110 return soup