Debian Codegraph @

fd_tool/fd_tool/api/metadata.py

raw
1 """This file provides classes to parse all dictionaries to extract the required
2 meta data for the freedict-database.xml. The meta data comes from the header of
3 each dictionary and gives details on the quality or size of a dictionary."""
4 import datetime
5 import html.parser
6 import io
7 import os
8 import re
9 from xml.etree import ElementTree as ET
10
11 from . import dictionary
12 from . import xmlhandlers
13 from .xmlhandlers import istag
14
15 class MetaDataParser(xmlhandlers.TeiHeadParser):
16 """Parse a TEI XML dictionary for required and optional meta data using
17 Python's iterparse facility. It guesses the path on it's own, unless a path
18 is given as optional parameter. For each element where data should be
19 extracted, there is a handle_<tagname> which is called if it exists.
20 This is a common class from which concrete parserfs can be derived. They all
21 share the facility to parse TEI XML, but they can differ in they way they
22 retrieve the TEI XML. While the local file system is a natural choice,
23 one could choose to retrieve the information over HTTP(s).
24 After the header has been parsed, the .dictionary attribute holds a reference
25 to a populated Dictionary object."""
26 # pattern to match mtaintainer in <resp/> tag
27 MAINTAINER_PATTERN = re.compile(r'^([^<]+)\s*<?([^<]+)?>$')
28 # match number of headwords in <extend/> pattern
29 HEADWORD_PATTERN = re.compile(r'(\d+(?:\.|,|\s*)\d*).*')
30 # ISO date pattern
31 DATE_PATTERN = re.compile(r'\d{4}-\d{2}-\d{2}')
32
33 def __init__(self, name, xml):
34 # can be a file object or a string with XML data
35 xml = self.get_file_object(xml)
36 super().__init__(xml)
37 self.dictionary = dictionary.Dictionary(name)
38 self.dictionary['date'] = None
39
40 def get_file_object(self, src):
41 """Transparently create a file object. If this method gets a file
42 object, it will return is unchanged, otherwise the given string is
43 wrapped in a StringIO object."""
44 if hasattr(src, 'read') and hasattr(src, 'close'):
45 return src # is already file object
46 elif isinstance(src, bytes):
47 return io.BytesIO(src)
48 elif isinstance(src, str):
49 return io.StringIO(src)
50 else:
51 raise ValueError("Either file object, str or byte array " + \
52 "expected, got %s." % type(src))
53
54 def parse(self):
55 try:
56 super().parse()
57 except ET.ParseError as e:
58 print(("Warning: while parsing {} an error occurred. Might be still "
59 "ok though.{}").format(self.dictionary.get_name(),
60 '; '.join(e.args)))
61
62 # check whether parsing was successful
63 if not self.dictionary.is_complete():
64 missing = [k for k in self.dictionary.get_mandatory_keys()
65 if not self.dictionary[k]]
66 raise ValueError("%s: the following information couldn't be read: "\
67 % self.dictionary.get_name() + ', '.join(missing))
68
69 def handle_tag(self, elem):
70 """Delegate parsing of XML tags to specialised functions."""
71 if istag(elem, 'date') or istag(elem, 'change'):
72 self.__extract_date(elem)
73 tag = elem.tag.split(self._namespace)[-1] # strip etree namespace
74 funcname = 'handle_%s' % tag
75 if not hasattr(self, funcname):
76 return
77 result = getattr(self, funcname)(elem)
78 if result:
79 self.dictionary.update(result)
80
81
82 def handle_sourceDesc(self, node):
83 """Extract a source url, if any."""
84 ptr = node.findall('.//%sptr' % self._namespace)
85 if ptr:
86 return {'sourceURL': ptr[0].get('target')}
87 # search for source URL within '<ref target=""')
88 ref = node.findall('.//%sref' % self._namespace)
89 if ref:
90 return {'sourceURL': ref[0].get('target')}
91 return None
92
93
94 def handle_edition(self, elem):
95 return {'edition': elem.text[:]}
96
97
98 def handle_notesStmt(self, node):
99 """Try to extract status."""
100 for note in node.findall(self._namespace + 'note'):
101 if note.get('type') == 'status':
102 return {'status': note.text}
103 raise ValueError('A notesStmt without a <note type="status"/> encountered.')
104
105
106
107 def handle_extent(self, elem):
108 """Extract extent (number of headwords)."""
109 match = MetaDataParser.HEADWORD_PATTERN.search(elem.text)
110 if not match:
111 raise ValueError("Could not extract number of headwords from " +
112 repr(elem.text))
113 headwords = ''.join(char for char in match.groups()[0] if char.isdigit())
114 return {'headwords': headwords}
115
116
117 def handle_respStmt(self, respStmt):
118 """Maintainer is in <respStmt/>, this one can contain nesting for author
119 and maintainer, hide complexity and return either maintainer, then
120 author or None."""
121 # find name attribute
122 name = respStmt.find(self._namespace + 'name')
123 if name is None:
124 return
125 resp = respStmt.findall(self._namespace + 'resp')
126 if resp is None or not any('maintainer' in t.text.lower() for t in resp):
127 return
128 maintainer = name.text
129 if not maintainer:
130 return
131 if 'up for grab' in maintainer.lower(): # not a real maintainer
132 return
133 # try to extract email address:
134 if '@' in maintainer:
135 maintainer = html.parser.unescape(maintainer)
136 match = self.MAINTAINER_PATTERN.search(maintainer)
137 if match:
138 return {'maintainerName': match.groups()[0].rstrip().lstrip(),
139 'maintainerEmail': match.groups()[1].rstrip().lstrip()
140 }
141 else:
142 return {'maintainerName': maintainer.rstrip().lstrip()}
143
144 def __extract_date(self, elem):
145 """If date has not been set with the <date/> attrbiute in the header,
146 guess it from change log."""
147 parse_date = lambda x: datetime.datetime.strptime(x, '%Y-%m-%d')
148 def set_date(isodate):
149 try:
150 if self.dictionary['date'] is None:
151 self.dictionary['date'] = isodate
152 elif parse_date(isodate) > parse_date(self.dictionary['date']):
153 self.dictionary['date'] = isodate
154 except ValueError:
155 pass # we cannot parse all date formats
156 if istag(elem, 'date'):
157 if elem.get('when'): # use when, is usually in desired format
158 set_date(elem.get('when').strip())
159 else: # check for correct format and if present, take it
160 if self.DATE_PATTERN.search(elem.text):
161 set_date(elem.text.strip())
162 elif istag(elem, 'change') and elem.get('when'):
163 set_date(elem.get('when').strip())
164
165 def __format_date(self, date):
166 """Bring date into the following format: YYYY-MM-dd."""
167 if re.search(r"\d+-\d+-\d+", date):
168 return date
169 try:
170 dateobj = datetime.datetime.strptime(date, "%d %B %Y")
171 date = '%d-%d-%d' % (dateobj.year, dateobj.month, dateobj.day)
172 except ValueError:
173 pass
174
175 def parse_dicts(self):
176 raise TypeError("This class is not meant to be used directly.""")
177
178
179
180 class LocalMetaDataParser(MetaDataParser):
181 """Parse meta data from XML dictionaries from a local fs path."""
182 def __init__(self, name, xml):
183 super().__init__(name, xml)
184
185 def get_meta_from_xml(path):
186 """Parse meta data for all dictionaries in the FreeDict Root. Returns a list
187 of Dictionary() objects."""
188 dictionaries = []
189 dict_pattern = re.compile(r'^[a-z]{3}-[a-z]{3}$')
190 for item in os.listdir(path):
191 full_path = os.path.join(path, item)
192 matched = dict_pattern.search(item)
193 if not matched: # eng-hun and hun-eng end on .header; auto-generated
194 continue # is not a dictionary
195 # append dictname.tei:
196 full_path = os.path.join(full_path, item) + '.tei'
197 if not os.path.exists(full_path):
198 full_path += '.header'
199 if not os.path.exists(full_path):
200 raise FileNotFoundError("For dictionary %s no dictionary file was found, assumed path: %s" \
201 % (item, os.path.abspath(full_path)))
202 with open(full_path, 'rb') as f:
203 dparser = LocalMetaDataParser(item, f)
204 dparser.parse()
205 dictionaries.append(dparser.dictionary)
206 return dictionaries
207
208