| 1 |
import re |
| 2 |
from datetime import date, timedelta |
| 3 |
|
| 4 |
from calibre.web.feeds.recipes import BasicNewsRecipe |
| 5 |
|
| 6 |
|
| 7 |
class MediaDaumRecipe(BasicNewsRecipe): |
| 8 |
title = '\ubbf8\ub514\uc5b4 \ub2e4\uc74c \uc624\ub298\uc758 \uc8fc\uc694 \ub274\uc2a4' |
| 9 |
description = 'Articles from media.daum.net' |
| 10 |
__author__ = 'trustin' |
| 11 |
language = 'ko' |
| 12 |
max_articles = 100 |
| 13 |
|
| 14 |
timefmt = '' |
| 15 |
masthead_url = 'http://img-media.daum-img.net/2010ci/service_news.gif' |
| 16 |
cover_margins = (18, 18, 'grey99') |
| 17 |
no_stylesheets = True |
| 18 |
remove_tags_before = dict(id='GS_con') |
| 19 |
remove_tags_after = dict(id='GS_con') |
| 20 |
remove_tags = [ |
| 21 |
dict( |
| 22 |
attrs={ |
| 23 |
'class': [ |
| 24 |
'bline', |
| 25 |
'GS_vod', |
| 26 |
] |
| 27 |
} |
| 28 |
), |
| 29 |
dict( |
| 30 |
id=[ |
| 31 |
'GS_swf_poll', |
| 32 |
'ad250', |
| 33 |
] |
| 34 |
), |
| 35 |
dict(name=['script', 'noscript', 'style', 'object']), |
| 36 |
] |
| 37 |
preprocess_regexps = [ |
| 38 |
(re.compile(r'<\s+', re.DOTALL | re.IGNORECASE), lambda match: '< '), |
| 39 |
(re.compile(r'(<br[^>]*>[ \t\r\n]*){3,}', re.DOTALL | re.IGNORECASE), lambda match: ''), |
| 40 |
(re.compile(r'(<br[^>]*>[ \t\r\n]*)*</div>', re.DOTALL | re.IGNORECASE), lambda match: '</div>'), |
| 41 |
(re.compile(r'(<br[^>]*>[ \t\r\n]*)*</p>', re.DOTALL | re.IGNORECASE), lambda match: '</p>'), |
| 42 |
(re.compile(r'(<br[^>]*>[ \t\r\n]*)*</td>', re.DOTALL | re.IGNORECASE), lambda match: '</td>'), |
| 43 |
(re.compile(r'(<br[^>]*>[ \t\r\n]*)*</strong>', re.DOTALL | re.IGNORECASE), lambda match: '</strong>'), |
| 44 |
(re.compile(r'(<br[^>]*>[ \t\r\n]*)*</b>', re.DOTALL | re.IGNORECASE), lambda match: '</b>'), |
| 45 |
(re.compile(r'(<br[^>]*>[ \t\r\n]*)*</em>', re.DOTALL | re.IGNORECASE), lambda match: '</em>'), |
| 46 |
(re.compile(r'(<br[^>]*>[ \t\r\n]*)*</i>', re.DOTALL | re.IGNORECASE), lambda match: '</i>'), |
| 47 |
(re.compile(r'\(\uB05D\)[ \t\r\n]*<br[^>]*>.*</div>', re.DOTALL | re.IGNORECASE), lambda match: '</div>'), |
| 48 |
(re.compile(r'(<br[^>]*>[ \t\r\n]*)*<div', re.DOTALL | re.IGNORECASE), lambda match: '<div'), |
| 49 |
(re.compile(r'(<br[^>]*>[ \t\r\n]*)*<p', re.DOTALL | re.IGNORECASE), lambda match: '<p'), |
| 50 |
(re.compile(r'(<br[^>]*>[ \t\r\n]*)*<table', re.DOTALL | re.IGNORECASE), lambda match: '<table'), |
| 51 |
(re.compile(r'<strong>(<br[^>]*>[ \t\r\n]*)*', re.DOTALL | re.IGNORECASE), lambda match: '<strong>'), |
| 52 |
(re.compile(r'<b>(<br[^>]*>[ \t\r\n]*)*', re.DOTALL | re.IGNORECASE), lambda match: '<b>'), |
| 53 |
(re.compile(r'<em>(<br[^>]*>[ \t\r\n]*)*', re.DOTALL | re.IGNORECASE), lambda match: '<em>'), |
| 54 |
(re.compile(r'<i>(<br[^>]*>[ \t\r\n]*)*', re.DOTALL | re.IGNORECASE), lambda match: '<i>'), |
| 55 |
( |
| 56 |
re.compile( |
| 57 |
'(<br[^>]*>[ \t\r\n]*)*(\u25b6|\u25cf|\u261e|\u24d2|\\(c\\))*\\[[^\\]]*' # noqa: RUF039 |
| 58 |
'(\u24d2|\\(c\\)|\uae30\uc0ac|\uc778\uae30[^\\]]*\ub274\uc2a4)[^\\]]*\\].*</div>', # noqa: RUF039 |
| 59 |
re.DOTALL | re.IGNORECASE, |
| 60 |
), |
| 61 |
lambda match: '</div>', |
| 62 |
), |
| 63 |
] |
| 64 |
|
| 65 |
def parse_index(self): |
| 66 |
today = date.today() |
| 67 |
articles = [] |
| 68 |
articles = self.parse_list_page(articles, today) |
| 69 |
articles = self.parse_list_page(articles, today - timedelta(1)) |
| 70 |
return [('\ubbf8\ub514\uc5b4 \ub2e4\uc74c \uc624\ub298\uc758 \uc8fc\uc694 \ub274\uc2a4', articles)] |
| 71 |
|
| 72 |
def parse_list_page(self, articles, date): |
| 73 |
if len(articles) >= self.max_articles: |
| 74 |
return articles |
| 75 |
|
| 76 |
for page in range(1, 10): |
| 77 |
soup = self.index_to_soup( |
| 78 |
'http://media.daum.net/primary/total/list.html?cateid=100044&date=%(date)s&page=%(page)d' % {'date': date.strftime('%Y%m%d'), 'page': page} |
| 79 |
) |
| 80 |
done = True |
| 81 |
for item in soup.findAll('dl'): |
| 82 |
dt = item.find('dt', {'class': 'tit'}) |
| 83 |
dd = item.find('dd', {'class': 'txt'}) |
| 84 |
if dt is None: |
| 85 |
break |
| 86 |
a = dt.find('a', href=True) |
| 87 |
url = 'http://media.daum.net/primary/total/' + a['href'] |
| 88 |
title = self.tag_to_string(dt) |
| 89 |
if dd is None: |
| 90 |
description = '' |
| 91 |
else: |
| 92 |
description = self.tag_to_string(dd) |
| 93 |
articles.append(dict(title=title, description=description, url=url, content='')) |
| 94 |
done = len(articles) >= self.max_articles |
| 95 |
if done: |
| 96 |
break |
| 97 |
if done: |
| 98 |
break |
| 99 |
return articles |
| 100 |
|
| 101 |
def preprocess_html(self, soup): |
| 102 |
return self.strip_anchors(soup) |
| 103 |
|
| 104 |
def strip_anchors(self, soup): |
| 105 |
for para in soup.findAll(True): |
| 106 |
aTags = para.findAll('a') |
| 107 |
for a in aTags: |
| 108 |
if a.img is None: |
| 109 |
a.replaceWith(a.renderContents().decode('utf-8', 'replace')) |
| 110 |
return soup |