| 1 |
#!/usr/bin/env python |
| 2 |
# vim:fileencoding=utf-8 |
| 3 |
from datetime import datetime, timedelta, timezone |
| 4 |
from zoneinfo import ZoneInfo |
| 5 |
|
| 6 |
from calibre.utils.date import parse_date |
| 7 |
from calibre.web.feeds.news import BasicNewsRecipe, classes |
| 8 |
|
| 9 |
index = 'https://www.globaltimes.cn/' |
| 10 |
|
| 11 |
# the site publishes timestamps as bare local wall clock times, with no |
| 12 |
# timezone marker anywhere in the markup. Global Times is based in Beijing and |
| 13 |
# files on Beijing time, so parse them as such rather than as the timezone of |
| 14 |
# whoever happens to be running the recipe. |
| 15 |
CHINA_TZ = ZoneInfo('Asia/Shanghai') |
| 16 |
|
| 17 |
|
| 18 |
class GlobalTimes(BasicNewsRecipe): |
| 19 |
title = 'Global Times' |
| 20 |
__author__ = 'unkn0wn' |
| 21 |
description = 'DISCOVER CHINA, DISCOVER THE WORLD' |
| 22 |
language = 'en_CN' |
| 23 |
no_stylesheets = True |
| 24 |
remove_attributes = ['height', 'width', 'style'] |
| 25 |
ignore_duplicate_articles = {'url'} |
| 26 |
masthead_url = 'https://www.globaltimes.cn/img/[email protected]' |
| 27 |
encoding = 'utf-8' |
| 28 |
remove_empty_feeds = True |
| 29 |
resolve_internal_links = True |
| 30 |
oldest_article = 1 # days |
| 31 |
|
| 32 |
def get_cover_url(self): |
| 33 |
soup = self.index_to_soup('https://en.kiosko.net/cn/np/cn_global_times.html') |
| 34 |
return 'https:' + soup.find('img', attrs={'id': 'portada'})['src'] |
| 35 |
|
| 36 |
extra_css = ''' |
| 37 |
.article_column {font-size:small; color:#404040;} |
| 38 |
.author_share_left, .picture, .with_name_card, .pub_time {font-size:small; color:#202020;} |
| 39 |
blockquote, em {color:#202020;} |
| 40 |
''' |
| 41 |
|
| 42 |
recipe_specific_options = { |
| 43 |
'days': { |
| 44 |
'short': 'Oldest article to download from this news source. In days ', |
| 45 |
'long': 'For example, 0.5, gives you articles from the past 12 hours', |
| 46 |
'default': str(oldest_article), |
| 47 |
} |
| 48 |
} |
| 49 |
|
| 50 |
def __init__(self, *args, **kwargs): |
| 51 |
BasicNewsRecipe.__init__(self, *args, **kwargs) |
| 52 |
d = self.recipe_specific_options.get('days') |
| 53 |
if d and isinstance(d, str): |
| 54 |
self.oldest_article = float(d) |
| 55 |
|
| 56 |
keep_only_tags = [classes('article_column article_title author_share_left article_content')] |
| 57 |
remove_tags = [classes('author_card')] |
| 58 |
|
| 59 |
def preprocess_raw_html(self, raw, *a): |
| 60 |
return raw.replace('<br /><br />', '</p><p>').replace('<br><br>', '</p><p>') |
| 61 |
|
| 62 |
def preprocess_html(self, soup): |
| 63 |
h1 = soup.find(attrs={'class': 'article_title'}) |
| 64 |
if h1: |
| 65 |
h1.name = 'h1' |
| 66 |
for div in soup.findAll(attrs={'class': 'picture'}): |
| 67 |
div.name = 'div' |
| 68 |
p = soup.find(attrs={'class': 'author_share_left'}) |
| 69 |
if p: |
| 70 |
p.name = 'p' |
| 71 |
return soup |
| 72 |
|
| 73 |
def parse_index(self): |
| 74 |
self.log('\n***\nif this recipe fails, report it on: https://www.mobileread.com/forums/forumdisplay.php?f=228\n***\n') |
| 75 |
sec_url = index + '{}/index.html' |
| 76 |
|
| 77 |
section_list = ['china', 'source', 'opinion', 'In-depth', 'world', 'life', 'sport', 'cartoon'] |
| 78 |
|
| 79 |
feeds = [] |
| 80 |
|
| 81 |
for section in section_list: |
| 82 |
section_title = section.capitalize() |
| 83 |
section_url = sec_url.format(section) |
| 84 |
self.log(section_title, section_url) |
| 85 |
soup = self.index_to_soup(section_url) |
| 86 |
articles = self.articles_from_soup(soup) |
| 87 |
if articles: |
| 88 |
feeds.append((section_title, articles)) |
| 89 |
return feeds |
| 90 |
|
| 91 |
def articles_from_soup(self, soup): |
| 92 |
ans = [] |
| 93 |
dt = datetime.today().strftime('%Y%m') |
| 94 |
# parse_date() fills in missing fields, and crucially the timezone, |
| 95 |
# from this default |
| 96 |
cn_default = datetime.now(CHINA_TZ).replace(day=15, hour=0, minute=0, second=0, microsecond=0) |
| 97 |
for a in soup.findAll('a', attrs={'href': lambda x: x and x.startswith(index + 'page/' + dt + '/')}): |
| 98 |
if a.find('img'): |
| 99 |
continue |
| 100 |
url = a['href'] |
| 101 |
title = self.tag_to_string(a).strip() |
| 102 |
desc = '' |
| 103 |
p = a.find_next_sibling('p') |
| 104 |
if p: |
| 105 |
desc = self.tag_to_string(p).strip() |
| 106 |
src_time = a.find_next_sibling(attrs={'class': 'source_time'}) |
| 107 |
if src_time: |
| 108 |
# the text is of the form "By Author | Source | 2026/9/12 10:14:20" |
| 109 |
# with the author and source fields both optional, so the |
| 110 |
# timestamp is always the last pipe separated field |
| 111 |
time = self.tag_to_string(src_time).split('|')[-1].strip() |
| 112 |
try: |
| 113 |
date = parse_date(time, default=cn_default) |
| 114 |
except Exception: |
| 115 |
self.log.warn('Failed to parse the date:', time, 'for article:', url) |
| 116 |
else: |
| 117 |
today = (datetime.now(timezone.utc)).replace(microsecond=0) |
| 118 |
if (today - date) > timedelta(self.oldest_article): |
| 119 |
continue |
| 120 |
self.log('\t', title, '\n\t', desc, '\n\t\t', url) |
| 121 |
ans.append({'title': title, 'url': url, 'description': desc}) |
| 122 |
return ans |