| 1 |
#!/usr/bin/env python3 |
| 2 |
# (c) 2020 Sebastian Humenda; Licence: GPL-3+ |
| 3 |
# pylint: disable=global-statement |
| 4 |
"""Restrictions: |
| 5 |
|
| 6 |
- Escaping within elements could be "lost" since the parser considers ' |
| 7 |
and other characters not to be worth escaping. |
| 8 |
- Name spacing is ignored. |
| 9 |
This script depends on eSpeakNG and /usr/share/isocodes. On Debian-based |
| 10 |
systems, install the packages isocodes and espeak-ng.""" |
| 11 |
import argparse |
| 12 |
import json |
| 13 |
import os |
| 14 |
import re |
| 15 |
import shutil |
| 16 |
import subprocess |
| 17 |
import sys |
| 18 |
import xml.sax as sax |
| 19 |
|
| 20 |
ISO_639_TABLE = "/usr/share/iso-codes/json/iso_639-3.json" |
| 21 |
# We rely on espeak-ng to output a single line of output for a given input line. |
| 22 |
# However, there seem to be characters that trigger additional newlines. So a |
| 23 |
# long word list, with a phrase per line, must have a one-to-one mapping on its |
| 24 |
# output. Therefore we replace those characters causing unexpected newlines. |
| 25 |
CHARACTER_REPLACEMENTS = { |
| 26 |
'"', |
| 27 |
"(", |
| 28 |
")", |
| 29 |
"{", |
| 30 |
"}", |
| 31 |
",", |
| 32 |
"...", |
| 33 |
".", |
| 34 |
":", |
| 35 |
";", |
| 36 |
"!", |
| 37 |
"?", |
| 38 |
"-", |
| 39 |
"…", |
| 40 |
"'", |
| 41 |
"[", |
| 42 |
"´", |
| 43 |
"&", |
| 44 |
"°", |
| 45 |
"$", |
| 46 |
"]", |
| 47 |
"®", |
| 48 |
"-", |
| 49 |
"̩", |
| 50 |
"£", |
| 51 |
"%", |
| 52 |
"\\", |
| 53 |
"§", |
| 54 |
"", |
| 55 |
"/", |
| 56 |
"+", |
| 57 |
"‘", |
| 58 |
"̃", |
| 59 |
"²", |
| 60 |
"³", |
| 61 |
"\n", |
| 62 |
"\r", |
| 63 |
chr(0x60C), # from ara-eng |
| 64 |
"−", |
| 65 |
"–", |
| 66 |
"—", |
| 67 |
"↑", |
| 68 |
"→", |
| 69 |
"←", |
| 70 |
"@", |
| 71 |
".", |
| 72 |
"〜", |
| 73 |
"×", |
| 74 |
"+", |
| 75 |
"&", |
| 76 |
"~", |
| 77 |
"↓", |
| 78 |
"、", |
| 79 |
} |
| 80 |
ESPEAKNG_LANG_CORRECTIONS = { |
| 81 |
"swh": "sw", # eSpeak 1.49 uses "sw" that stands for "swa", while FD uses "swh" |
| 82 |
} |
| 83 |
DEBUG = None |
| 84 |
|
| 85 |
# pylint: disable=too-few-public-methods |
| 86 |
class Orth: |
| 87 |
"""A representation of a <orth> element.""" |
| 88 |
|
| 89 |
def __init__(self, attrs): |
| 90 |
self.attrs = attrs |
| 91 |
self.text = [] |
| 92 |
|
| 93 |
|
| 94 |
class Entry: |
| 95 |
"""A (mostly) serialised entry. |
| 96 |
An entry may contain strings or Orth nodes. It also mimics an output stream |
| 97 |
using the write method.""" |
| 98 |
|
| 99 |
def __init__(self, attrs): |
| 100 |
attrs = "".join(f' {k}="{v}"' for k, v in attrs.items()) |
| 101 |
self.xml = [f"<entry{attrs}>"] |
| 102 |
|
| 103 |
def write(self, item): |
| 104 |
"""ToDo: text or orth.""" |
| 105 |
self.xml.append(item) |
| 106 |
|
| 107 |
def contains_pronunciation(self): |
| 108 |
return any( |
| 109 |
token for token in self.xml if isinstance(token, str) and "<pron" in token |
| 110 |
) |
| 111 |
|
| 112 |
|
| 113 |
class EspeakNg: |
| 114 |
RESTART_AFTER = 10000 |
| 115 |
|
| 116 |
def __init__(self, language): |
| 117 |
if not shutil.which("espeak-ng"): |
| 118 |
print("Espeak-ng not found, please install it and rerun the operation.") |
| 119 |
sys.exit(5) |
| 120 |
self.cmd = ("stdbuf", "-o0", "espeak-ng", "--ipa", "-v", language, "-q") |
| 121 |
self.proc = None |
| 122 |
self.headwords_transcribed = 0 |
| 123 |
self.non_alpha = set() |
| 124 |
self._start() |
| 125 |
|
| 126 |
def _start(self): |
| 127 |
self.proc = subprocess.Popen( |
| 128 |
self.cmd, |
| 129 |
stdin=subprocess.PIPE, |
| 130 |
stdout=subprocess.PIPE, |
| 131 |
stderr=subprocess.PIPE, |
| 132 |
bufsize=0, |
| 133 |
) |
| 134 |
self.headwords_transcribed = 0 |
| 135 |
|
| 136 |
def ipa(self, orth): |
| 137 |
"""Transcribe the given headword and return the IPA string. It attempts to |
| 138 |
remove any punctuation because this can trigger line breaks in the |
| 139 |
eSpeak-NG output. This scripts relies on a single line of output per |
| 140 |
headword.""" |
| 141 |
if self.headwords_transcribed >= EspeakNg.RESTART_AFTER: |
| 142 |
self.reset() |
| 143 |
headword = "".join(orth.text) |
| 144 |
headword = self.escape(headword) |
| 145 |
self.non_alpha.update( |
| 146 |
c for c in headword if not (c.isalpha() or c.isdigit() or c.isspace()) |
| 147 |
) |
| 148 |
self.proc.stdin.write(headword.encode(sys.getdefaultencoding()) + b"\n") |
| 149 |
self.proc.stdin.flush() |
| 150 |
buf = self.proc.stdout.readline() |
| 151 |
word = buf.decode(sys.getdefaultencoding()).strip() |
| 152 |
if DEBUG: |
| 153 |
open(DEBUG, "a").write(f"{headword} :: {word}\n") |
| 154 |
self.headwords_transcribed += 1 |
| 155 |
return word |
| 156 |
|
| 157 |
def escape(self, word): |
| 158 |
"""Escape any punctuation, see ipa().""" |
| 159 |
for ch in CHARACTER_REPLACEMENTS: |
| 160 |
word = word.replace(ch, " ") |
| 161 |
return word.replace(" ", " ") |
| 162 |
|
| 163 |
def close(self): |
| 164 |
remainder = self.proc.communicate()[0] |
| 165 |
if remainder: |
| 166 |
import textwrap |
| 167 |
|
| 168 |
non_alpha = "".join(self.non_alpha) |
| 169 |
|
| 170 |
print( |
| 171 |
"\n".join( |
| 172 |
textwrap.wrap( |
| 173 |
"Error: {} lines found that were not " |
| 174 |
"recognised by this script during IPA transcription of " |
| 175 |
"eSpeak-NG. This is a bug due to special characters that lead " |
| 176 |
" to unexpected newlines in the eSpeak-NG output. Expected is " |
| 177 |
"a line per headword. Use --debug to save the transcriptions to a " |
| 178 |
"separate file." |
| 179 |
) |
| 180 |
).format(len(remainder.split(b"\n"))) |
| 181 |
+ f"\nPotentially problematic characters: {non_alpha}" |
| 182 |
) |
| 183 |
sys.exit(9) |
| 184 |
ret = self.proc.wait() |
| 185 |
if ret != 0: |
| 186 |
raise OSError(f"Espeak-ng gave non-zero exit code {ret}.") |
| 187 |
|
| 188 |
def reset(self): |
| 189 |
"""Reset eSpeak-NG to avoid filling the OS stdin buffer.""" |
| 190 |
self.close() |
| 191 |
self._start() |
| 192 |
|
| 193 |
|
| 194 |
class TeiConsumer(sax.handler.ContentHandler): |
| 195 |
"""A streaming SAX parser that calls a registered handler function to |
| 196 |
process <orth/> elements.""" |
| 197 |
|
| 198 |
def __init__(self, file_handle, ipa_generator): |
| 199 |
# This is either None or a list. If it is a list, all text data will be |
| 200 |
# appended. |
| 201 |
self.current_orth = None |
| 202 |
self.outstream = file_handle |
| 203 |
self.outstream_backup = None |
| 204 |
super().__init__() |
| 205 |
self.ipa_generator = ipa_generator |
| 206 |
|
| 207 |
def endDocument(self): |
| 208 |
self.outstream.write("\n") |
| 209 |
self.outstream.close() |
| 210 |
self.ipa_generator.close() |
| 211 |
|
| 212 |
def startElement(self, name, attrs): |
| 213 |
"""Signals the start of an element in non-namespace mode.""" |
| 214 |
if not attrs: |
| 215 |
attrs = {} |
| 216 |
if name == "entry": |
| 217 |
self.outstream_backup = self.outstream |
| 218 |
self.outstream = Entry(attrs) |
| 219 |
elif name == "orth": |
| 220 |
self.current_orth = Orth(attrs) |
| 221 |
else: |
| 222 |
attrs = attrs.items() |
| 223 |
self.outstream.write(f"<{name}") |
| 224 |
if attrs: |
| 225 |
for k, v in attrs: |
| 226 |
self.outstream.write(f' {k}="{v}"') |
| 227 |
self.outstream.write(">") |
| 228 |
|
| 229 |
def endElement(self, name): |
| 230 |
"""Signals the end of an element in non-namespace mode.""" |
| 231 |
if name == "orth": |
| 232 |
# add the Orth to the Entry |
| 233 |
self.outstream.write(self.current_orth) |
| 234 |
self.current_orth = None |
| 235 |
elif name == "entry": |
| 236 |
entry = self.outstream |
| 237 |
self.outstream = self.outstream_backup |
| 238 |
for item in entry.xml: |
| 239 |
if isinstance(item, Orth): |
| 240 |
attrs = "".join(f' {k}="{v}"' for k, v in item.attrs.items()) |
| 241 |
text = sax.saxutils.escape("".join(item.text)) |
| 242 |
self.outstream.write(f"<orth{attrs}>{text}</orth>") |
| 243 |
# It could be that one orth has transcriptions, another |
| 244 |
# doesn't, these cases are ignored in this version |
| 245 |
if not entry.contains_pronunciation(): |
| 246 |
# transcriptions are rarely empty |
| 247 |
transcribed = sax.saxutils.escape(self.ipa_generator.ipa(item)) |
| 248 |
if transcribed: |
| 249 |
self.outstream.write(f"\n<pron>{transcribed}</pron>") |
| 250 |
else: |
| 251 |
self.outstream.write(item) |
| 252 |
self.outstream.write(f"</{name}>") |
| 253 |
else: |
| 254 |
self.outstream.write(f"</{name}>") |
| 255 |
|
| 256 |
def characters(self, content): |
| 257 |
"""Receive notification of character data.""" |
| 258 |
if self.current_orth: |
| 259 |
self.current_orth.text.append(content) |
| 260 |
else: |
| 261 |
self.outstream.write(sax.saxutils.escape(content)) |
| 262 |
|
| 263 |
def ignorableWhitespace(self, whitespace): |
| 264 |
"""Receive notification of ignorable whitespace in element content.""" |
| 265 |
self.outstream.write(whitespace) |
| 266 |
|
| 267 |
|
| 268 |
def espeakng_voice(iso_639_3_code): |
| 269 |
"""Return the Espeak-NG voice from the given ISO 639-3 code. |
| 270 |
StopIteration is raised if language is unknown.""" |
| 271 |
# Read all available languages first |
| 272 |
if not os.path.exists(ISO_639_TABLE): |
| 273 |
print("Error: failed to load {ISO_639_TABLE}.") |
| 274 |
print("This file can be found in the isocodes package on Debian-based systems.") |
| 275 |
sys.exit(3) |
| 276 |
proc = subprocess.run(["espeak-ng", "--voices"], capture_output=True) |
| 277 |
espeakng_voices = set() |
| 278 |
for voice in proc.stdout.decode(sys.getdefaultencoding()).split("\n"): |
| 279 |
voice = re.search(r"^\s*[0-9]*\s*([a-z]+)(-[0-9]*[a-z]*)?\s.*$", voice) |
| 280 |
if voice: |
| 281 |
espeakng_voices.add(voice.group(1)) |
| 282 |
all_iso_639 = json.load(open(ISO_639_TABLE))["639-3"] |
| 283 |
# Let StopIteration propagate |
| 284 |
lang = next(l for l in all_iso_639 if l["alpha_3"] == iso_639_3_code) |
| 285 |
# get the ISO-639-1 code (alpha_2) and afterwards, try to detect whether |
| 286 |
# there's a correction that maps from a code used by FD to one used by |
| 287 |
# eSpeakNG |
| 288 |
lang = lang.get("alpha_2", iso_639_3_code) |
| 289 |
lang = ESPEAKNG_LANG_CORRECTIONS.get(lang, lang) |
| 290 |
if lang not in espeakng_voices: |
| 291 |
print(f'Error, espeak did not recognise language "{lang}".') |
| 292 |
sys.exit(6) |
| 293 |
return lang |
| 294 |
|
| 295 |
|
| 296 |
def xml_preamble(tei_file): |
| 297 |
"""Extract everything until <TEI since the SAX parser ignores it and it's |
| 298 |
part of the document. This function seeks back to the beginning afterwards.""" |
| 299 |
lines = [] |
| 300 |
found = False |
| 301 |
for _i in range(100): # upper limit |
| 302 |
line = tei_file.readline() |
| 303 |
idx = line.find("<TEI") |
| 304 |
if idx >= 0: |
| 305 |
lines.append(line[:idx]) |
| 306 |
break |
| 307 |
else: |
| 308 |
lines.append(line) |
| 309 |
tei_file.seek(0) |
| 310 |
return "".join(lines) if found else "" |
| 311 |
|
| 312 |
|
| 313 |
def handle_cmdline(): |
| 314 |
global DEBUG |
| 315 |
parser = argparse.ArgumentParser( |
| 316 |
description="Enrich a TEI file with pronunciation info." |
| 317 |
) |
| 318 |
parser.add_argument( |
| 319 |
"tei_dict", metavar="TEI_DICT", nargs="?", help="input file name" |
| 320 |
) |
| 321 |
parser.add_argument( |
| 322 |
"-o", |
| 323 |
dest="outpath", |
| 324 |
metavar="OUTPUT_PATH", |
| 325 |
nargs=1, |
| 326 |
help="write processed TEI to given output file. default is \ |
| 327 |
xxx-yyy-phoneme.tei", |
| 328 |
) |
| 329 |
parser.add_argument( |
| 330 |
"--supports-lang", |
| 331 |
dest="supports_lang", |
| 332 |
metavar="LANG", |
| 333 |
nargs=1, |
| 334 |
help="test wheter given language (ISO 639-3) is supported by eSpeak", |
| 335 |
) |
| 336 |
parser.add_argument( |
| 337 |
"--debug", |
| 338 |
dest="debug", |
| 339 |
metavar="DEBUGFILE", |
| 340 |
nargs=1, |
| 341 |
help="write phoneme transcription to specified file. The\ |
| 342 |
file will have a headword per line, separated by colons from \ |
| 343 |
the transcription. This helps to find the problematic \ |
| 344 |
headword if transcriptions start to mismatch.", |
| 345 |
) |
| 346 |
|
| 347 |
args = parser.parse_args() |
| 348 |
teipath = None |
| 349 |
# Handle --supports-lang that aborts after the check, return results |
| 350 |
# otherwise |
| 351 |
if args.supports_lang: |
| 352 |
try: |
| 353 |
espeakng_voice(args.supports_lang[0]) |
| 354 |
except StopIteration: # raised to propagate missing language |
| 355 |
sys.exit(4) |
| 356 |
else: |
| 357 |
sys.exit(0) |
| 358 |
elif not args.tei_dict: |
| 359 |
print("Error, need to provide input TEI file") |
| 360 |
sys.exit(9) |
| 361 |
teipath = args.tei_dict |
| 362 |
if args.outpath: |
| 363 |
outpath = args.outpath[0] |
| 364 |
else: |
| 365 |
outpath = f"{os.path.splitext(teipath)[0]}-phonetics.tei" |
| 366 |
|
| 367 |
if args.debug: |
| 368 |
DEBUG = args.debug[0] |
| 369 |
return (teipath, outpath) |
| 370 |
|
| 371 |
|
| 372 |
def main(): |
| 373 |
teipath, outpath = handle_cmdline() |
| 374 |
|
| 375 |
# Guess eSpeak-NG voice to use |
| 376 |
lang = os.path.splitext(os.path.basename(teipath))[0].split("-")[0] |
| 377 |
ipa_gen = EspeakNg(espeakng_voice(lang)) |
| 378 |
|
| 379 |
# Extract preamble (not parsed by SAX parser), write and call SAX parser |
| 380 |
# afterwards |
| 381 |
with open(teipath) as teifile: |
| 382 |
outfile = open(outpath, "w") # closed by TeiConsumer |
| 383 |
outfile.write(xml_preamble(teifile)) |
| 384 |
parser = sax.make_parser() |
| 385 |
teihandler = TeiConsumer(outfile, ipa_gen) |
| 386 |
parser.setContentHandler(teihandler) |
| 387 |
parser.parse(teifile) |
| 388 |
|
| 389 |
|
| 390 |
if __name__ == "__main__": |
| 391 |
main() |