| 1 |
#! /usr/bin/env python3 |
| 2 |
# Written by Martin v. Löwis <[email protected]> |
| 3 |
|
| 4 |
"""Generate binary message catalog from textual translation description. |
| 5 |
|
| 6 |
This program converts a textual Uniforum-style message catalog (.po file) into |
| 7 |
a binary GNU catalog (.mo file). This is essentially the same function as the |
| 8 |
GNU msgfmt program, however, it is a simpler implementation. Currently it |
| 9 |
does not handle plural forms but it does handle message contexts. |
| 10 |
|
| 11 |
Usage: msgfmt.py [OPTIONS] filename.po |
| 12 |
|
| 13 |
Options: |
| 14 |
-o file |
| 15 |
--output-file=file |
| 16 |
Specify the output file to write to. If omitted, output will go to a |
| 17 |
file named filename.mo (based off the input file name). |
| 18 |
|
| 19 |
-h |
| 20 |
--help |
| 21 |
Print this message and exit. |
| 22 |
|
| 23 |
-V |
| 24 |
--version |
| 25 |
Display version information and exit. |
| 26 |
""" |
| 27 |
|
| 28 |
import array |
| 29 |
import ast |
| 30 |
import codecs |
| 31 |
import getopt |
| 32 |
import os |
| 33 |
import struct |
| 34 |
import sys |
| 35 |
from email.parser import HeaderParser |
| 36 |
|
| 37 |
__version__ = "1.2" |
| 38 |
|
| 39 |
|
| 40 |
MESSAGES = {} |
| 41 |
|
| 42 |
|
| 43 |
def usage(code, msg=''): |
| 44 |
print(__doc__, file=sys.stderr) |
| 45 |
if msg: |
| 46 |
print(msg, file=sys.stderr) |
| 47 |
sys.exit(code) |
| 48 |
|
| 49 |
|
| 50 |
def add(ctxt, id, str, fuzzy): |
| 51 |
"Add a non-fuzzy translation to the dictionary." |
| 52 |
global MESSAGES |
| 53 |
if not fuzzy and str: |
| 54 |
if ctxt is None: |
| 55 |
MESSAGES[id] = str |
| 56 |
else: |
| 57 |
MESSAGES[b"%b\x04%b" % (ctxt, id)] = str |
| 58 |
|
| 59 |
|
| 60 |
def generate(): |
| 61 |
"Return the generated output." |
| 62 |
global MESSAGES |
| 63 |
# the keys are sorted in the .mo file |
| 64 |
keys = sorted(MESSAGES.keys()) |
| 65 |
offsets = [] |
| 66 |
ids = strs = b'' |
| 67 |
for id in keys: |
| 68 |
# For each string, we need size and file offset. Each string is NUL |
| 69 |
# terminated; the NUL does not count into the size. |
| 70 |
offsets.append((len(ids), len(id), len(strs), len(MESSAGES[id]))) |
| 71 |
ids += id + b'\0' |
| 72 |
strs += MESSAGES[id] + b'\0' |
| 73 |
output = '' |
| 74 |
# The header is 7 32-bit unsigned integers. We don't use hash tables, so |
| 75 |
# the keys start right after the index tables. |
| 76 |
# translated string. |
| 77 |
keystart = 7*4+16*len(keys) |
| 78 |
# and the values start after the keys |
| 79 |
valuestart = keystart + len(ids) |
| 80 |
koffsets = [] |
| 81 |
voffsets = [] |
| 82 |
# The string table first has the list of keys, then the list of values. |
| 83 |
# Each entry has first the size of the string, then the file offset. |
| 84 |
for o1, l1, o2, l2 in offsets: |
| 85 |
koffsets += [l1, o1+keystart] |
| 86 |
voffsets += [l2, o2+valuestart] |
| 87 |
offsets = koffsets + voffsets |
| 88 |
output = struct.pack("Iiiiiii", |
| 89 |
0x950412de, # Magic |
| 90 |
0, # Version |
| 91 |
len(keys), # # of entries |
| 92 |
7*4, # start of key index |
| 93 |
7*4+len(keys)*8, # start of value index |
| 94 |
0, 0) # size and offset of hash table |
| 95 |
output += array.array("i", offsets).tobytes() |
| 96 |
output += ids |
| 97 |
output += strs |
| 98 |
return output |
| 99 |
|
| 100 |
|
| 101 |
def make(filename, outfile): |
| 102 |
ID = 1 |
| 103 |
STR = 2 |
| 104 |
CTXT = 3 |
| 105 |
|
| 106 |
# Compute .mo name from .po name and arguments |
| 107 |
if filename.endswith('.po'): |
| 108 |
infile = filename |
| 109 |
else: |
| 110 |
infile = filename + '.po' |
| 111 |
if outfile is None: |
| 112 |
outfile = os.path.splitext(infile)[0] + '.mo' |
| 113 |
|
| 114 |
try: |
| 115 |
with open(infile, 'rb') as f: |
| 116 |
lines = f.readlines() |
| 117 |
except OSError as msg: |
| 118 |
print(msg, file=sys.stderr) |
| 119 |
sys.exit(1) |
| 120 |
|
| 121 |
if lines[0].startswith(codecs.BOM_UTF8): |
| 122 |
print( |
| 123 |
f"The file {infile} starts with a UTF-8 BOM which is not allowed in .po files.\n" |
| 124 |
"Please save the file without a BOM and try again.", |
| 125 |
file=sys.stderr |
| 126 |
) |
| 127 |
sys.exit(1) |
| 128 |
|
| 129 |
section = msgctxt = None |
| 130 |
msgid = msgstr = b'' |
| 131 |
fuzzy = 0 |
| 132 |
|
| 133 |
# Start off assuming Latin-1, so everything decodes without failure, |
| 134 |
# until we know the exact encoding |
| 135 |
encoding = 'latin-1' |
| 136 |
|
| 137 |
# Parse the catalog |
| 138 |
lno = 0 |
| 139 |
for l in lines: |
| 140 |
l = l.decode(encoding) |
| 141 |
lno += 1 |
| 142 |
# If we get a comment line after a msgstr, this is a new entry |
| 143 |
if l[0] == '#' and section == STR: |
| 144 |
add(msgctxt, msgid, msgstr, fuzzy) |
| 145 |
section = msgctxt = None |
| 146 |
fuzzy = 0 |
| 147 |
# Record a fuzzy mark |
| 148 |
if l[:2] == '#,' and 'fuzzy' in l: |
| 149 |
fuzzy = 1 |
| 150 |
# Skip comments |
| 151 |
if l[0] == '#': |
| 152 |
continue |
| 153 |
# Now we are in a msgid or msgctxt section, output previous section |
| 154 |
if l.startswith('msgctxt'): |
| 155 |
if section == STR: |
| 156 |
add(msgctxt, msgid, msgstr, fuzzy) |
| 157 |
section = CTXT |
| 158 |
l = l[7:] |
| 159 |
msgctxt = b'' |
| 160 |
elif l.startswith('msgid') and not l.startswith('msgid_plural'): |
| 161 |
if section == STR: |
| 162 |
if not msgid: |
| 163 |
# Filter out POT-Creation-Date |
| 164 |
# See issue #131852 |
| 165 |
msgstr = b''.join(line for line in msgstr.splitlines(True) |
| 166 |
if not line.startswith(b'POT-Creation-Date:')) |
| 167 |
|
| 168 |
# See whether there is an encoding declaration |
| 169 |
p = HeaderParser() |
| 170 |
charset = p.parsestr(msgstr.decode(encoding)).get_content_charset() |
| 171 |
if charset: |
| 172 |
encoding = charset |
| 173 |
add(msgctxt, msgid, msgstr, fuzzy) |
| 174 |
msgctxt = None |
| 175 |
section = ID |
| 176 |
l = l[5:] |
| 177 |
msgid = msgstr = b'' |
| 178 |
is_plural = False |
| 179 |
# This is a message with plural forms |
| 180 |
elif l.startswith('msgid_plural'): |
| 181 |
if section != ID: |
| 182 |
print(f'msgid_plural not preceded by msgid on {infile}:{lno}', |
| 183 |
file=sys.stderr) |
| 184 |
sys.exit(1) |
| 185 |
l = l[12:] |
| 186 |
msgid += b'\0' # separator of singular and plural |
| 187 |
is_plural = True |
| 188 |
# Now we are in a msgstr section |
| 189 |
elif l.startswith('msgstr'): |
| 190 |
section = STR |
| 191 |
if l.startswith('msgstr['): |
| 192 |
if not is_plural: |
| 193 |
print(f'plural without msgid_plural on {infile}:{lno}', |
| 194 |
file=sys.stderr) |
| 195 |
sys.exit(1) |
| 196 |
l = l.split(']', 1)[1] |
| 197 |
if msgstr: |
| 198 |
msgstr += b'\0' # Separator of the various plural forms |
| 199 |
else: |
| 200 |
if is_plural: |
| 201 |
print(f'indexed msgstr required for plural on {infile}:{lno}', |
| 202 |
file=sys.stderr) |
| 203 |
sys.exit(1) |
| 204 |
l = l[6:] |
| 205 |
# Skip empty lines |
| 206 |
l = l.strip() |
| 207 |
if not l: |
| 208 |
continue |
| 209 |
l = ast.literal_eval(l) |
| 210 |
if section == CTXT: |
| 211 |
msgctxt += l.encode(encoding) |
| 212 |
elif section == ID: |
| 213 |
msgid += l.encode(encoding) |
| 214 |
elif section == STR: |
| 215 |
msgstr += l.encode(encoding) |
| 216 |
else: |
| 217 |
print(f'Syntax error on {infile}:{lno} before:', file=sys.stderr) |
| 218 |
print(l, file=sys.stderr) |
| 219 |
sys.exit(1) |
| 220 |
# Add last entry |
| 221 |
if section == STR: |
| 222 |
add(msgctxt, msgid, msgstr, fuzzy) |
| 223 |
|
| 224 |
# Compute output |
| 225 |
output = generate() |
| 226 |
|
| 227 |
try: |
| 228 |
with open(outfile,"wb") as f: |
| 229 |
f.write(output) |
| 230 |
except OSError as msg: |
| 231 |
print(msg, file=sys.stderr) |
| 232 |
|
| 233 |
|
| 234 |
def main(): |
| 235 |
try: |
| 236 |
opts, args = getopt.getopt(sys.argv[1:], 'hVo:', |
| 237 |
['help', 'version', 'output-file=']) |
| 238 |
except getopt.error as msg: |
| 239 |
usage(1, msg) |
| 240 |
|
| 241 |
outfile = None |
| 242 |
# parse options |
| 243 |
for opt, arg in opts: |
| 244 |
if opt in ('-h', '--help'): |
| 245 |
usage(0) |
| 246 |
elif opt in ('-V', '--version'): |
| 247 |
print("msgfmt.py", __version__) |
| 248 |
sys.exit(0) |
| 249 |
elif opt in ('-o', '--output-file'): |
| 250 |
outfile = arg |
| 251 |
# do it |
| 252 |
if not args: |
| 253 |
print('No input file given', file=sys.stderr) |
| 254 |
print("Try `msgfmt --help' for more information.", file=sys.stderr) |
| 255 |
return |
| 256 |
|
| 257 |
for filename in args: |
| 258 |
make(filename, outfile) |
| 259 |
|
| 260 |
|
| 261 |
if __name__ == '__main__': |
| 262 |
main() |