Debian Codegraph @

recipes/globaltimes.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 from datetime import datetime, timedelta, timezone
4 from zoneinfo import ZoneInfo
5
6 from calibre.utils.date import parse_date
7 from calibre.web.feeds.news import BasicNewsRecipe, classes
8
9 index = 'https://www.globaltimes.cn/'
10
11 # the site publishes timestamps as bare local wall clock times, with no
12 # timezone marker anywhere in the markup. Global Times is based in Beijing and
13 # files on Beijing time, so parse them as such rather than as the timezone of
14 # whoever happens to be running the recipe.
15 CHINA_TZ = ZoneInfo('Asia/Shanghai')
16
17
18 class GlobalTimes(BasicNewsRecipe):
19 title = 'Global Times'
20 __author__ = 'unkn0wn'
21 description = 'DISCOVER CHINA, DISCOVER THE WORLD'
22 language = 'en_CN'
23 no_stylesheets = True
24 remove_attributes = ['height', 'width', 'style']
25 ignore_duplicate_articles = {'url'}
26 masthead_url = 'https://www.globaltimes.cn/img/[email protected]'
27 encoding = 'utf-8'
28 remove_empty_feeds = True
29 resolve_internal_links = True
30 oldest_article = 1 # days
31
32 def get_cover_url(self):
33 soup = self.index_to_soup('https://en.kiosko.net/cn/np/cn_global_times.html')
34 return 'https:' + soup.find('img', attrs={'id': 'portada'})['src']
35
36 extra_css = '''
37 .article_column {font-size:small; color:#404040;}
38 .author_share_left, .picture, .with_name_card, .pub_time {font-size:small; color:#202020;}
39 blockquote, em {color:#202020;}
40 '''
41
42 recipe_specific_options = {
43 'days': {
44 'short': 'Oldest article to download from this news source. In days ',
45 'long': 'For example, 0.5, gives you articles from the past 12 hours',
46 'default': str(oldest_article),
47 }
48 }
49
50 def __init__(self, *args, **kwargs):
51 BasicNewsRecipe.__init__(self, *args, **kwargs)
52 d = self.recipe_specific_options.get('days')
53 if d and isinstance(d, str):
54 self.oldest_article = float(d)
55
56 keep_only_tags = [classes('article_column article_title author_share_left article_content')]
57 remove_tags = [classes('author_card')]
58
59 def preprocess_raw_html(self, raw, *a):
60 return raw.replace('<br /><br />', '</p><p>').replace('<br><br>', '</p><p>')
61
62 def preprocess_html(self, soup):
63 h1 = soup.find(attrs={'class': 'article_title'})
64 if h1:
65 h1.name = 'h1'
66 for div in soup.findAll(attrs={'class': 'picture'}):
67 div.name = 'div'
68 p = soup.find(attrs={'class': 'author_share_left'})
69 if p:
70 p.name = 'p'
71 return soup
72
73 def parse_index(self):
74 self.log('\n***\nif this recipe fails, report it on: https://www.mobileread.com/forums/forumdisplay.php?f=228\n***\n')
75 sec_url = index + '{}/index.html'
76
77 section_list = ['china', 'source', 'opinion', 'In-depth', 'world', 'life', 'sport', 'cartoon']
78
79 feeds = []
80
81 for section in section_list:
82 section_title = section.capitalize()
83 section_url = sec_url.format(section)
84 self.log(section_title, section_url)
85 soup = self.index_to_soup(section_url)
86 articles = self.articles_from_soup(soup)
87 if articles:
88 feeds.append((section_title, articles))
89 return feeds
90
91 def articles_from_soup(self, soup):
92 ans = []
93 dt = datetime.today().strftime('%Y%m')
94 # parse_date() fills in missing fields, and crucially the timezone,
95 # from this default
96 cn_default = datetime.now(CHINA_TZ).replace(day=15, hour=0, minute=0, second=0, microsecond=0)
97 for a in soup.findAll('a', attrs={'href': lambda x: x and x.startswith(index + 'page/' + dt + '/')}):
98 if a.find('img'):
99 continue
100 url = a['href']
101 title = self.tag_to_string(a).strip()
102 desc = ''
103 p = a.find_next_sibling('p')
104 if p:
105 desc = self.tag_to_string(p).strip()
106 src_time = a.find_next_sibling(attrs={'class': 'source_time'})
107 if src_time:
108 # the text is of the form "By Author | Source | 2026/9/12 10:14:20"
109 # with the author and source fields both optional, so the
110 # timestamp is always the last pipe separated field
111 time = self.tag_to_string(src_time).split('|')[-1].strip()
112 try:
113 date = parse_date(time, default=cn_default)
114 except Exception:
115 self.log.warn('Failed to parse the date:', time, 'for article:', url)
116 else:
117 today = (datetime.now(timezone.utc)).replace(microsecond=0)
118 if (today - date) > timedelta(self.oldest_article):
119 continue
120 self.log('\t', title, '\n\t', desc, '\n\t\t', url)
121 ans.append({'title': title, 'url': url, 'description': desc})
122 return ans