Debian Codegraph @

Tools/i18n/msgfmt.py

raw
1 #! /usr/bin/env python3
2 # Written by Martin v. Löwis <[email protected]>
3
4 """Generate binary message catalog from textual translation description.
5
6 This program converts a textual Uniforum-style message catalog (.po file) into
7 a binary GNU catalog (.mo file). This is essentially the same function as the
8 GNU msgfmt program, however, it is a simpler implementation. Currently it
9 does not handle plural forms but it does handle message contexts.
10
11 Usage: msgfmt.py [OPTIONS] filename.po
12
13 Options:
14 -o file
15 --output-file=file
16 Specify the output file to write to. If omitted, output will go to a
17 file named filename.mo (based off the input file name).
18
19 -h
20 --help
21 Print this message and exit.
22
23 -V
24 --version
25 Display version information and exit.
26 """
27
28 import array
29 import ast
30 import codecs
31 import getopt
32 import os
33 import struct
34 import sys
35 from email.parser import HeaderParser
36
37 __version__ = "1.2"
38
39
40 MESSAGES = {}
41
42
43 def usage(code, msg=''):
44 print(__doc__, file=sys.stderr)
45 if msg:
46 print(msg, file=sys.stderr)
47 sys.exit(code)
48
49
50 def add(ctxt, id, str, fuzzy):
51 "Add a non-fuzzy translation to the dictionary."
52 global MESSAGES
53 if not fuzzy and str:
54 if ctxt is None:
55 MESSAGES[id] = str
56 else:
57 MESSAGES[b"%b\x04%b" % (ctxt, id)] = str
58
59
60 def generate():
61 "Return the generated output."
62 global MESSAGES
63 # the keys are sorted in the .mo file
64 keys = sorted(MESSAGES.keys())
65 offsets = []
66 ids = strs = b''
67 for id in keys:
68 # For each string, we need size and file offset. Each string is NUL
69 # terminated; the NUL does not count into the size.
70 offsets.append((len(ids), len(id), len(strs), len(MESSAGES[id])))
71 ids += id + b'\0'
72 strs += MESSAGES[id] + b'\0'
73 output = ''
74 # The header is 7 32-bit unsigned integers. We don't use hash tables, so
75 # the keys start right after the index tables.
76 # translated string.
77 keystart = 7*4+16*len(keys)
78 # and the values start after the keys
79 valuestart = keystart + len(ids)
80 koffsets = []
81 voffsets = []
82 # The string table first has the list of keys, then the list of values.
83 # Each entry has first the size of the string, then the file offset.
84 for o1, l1, o2, l2 in offsets:
85 koffsets += [l1, o1+keystart]
86 voffsets += [l2, o2+valuestart]
87 offsets = koffsets + voffsets
88 output = struct.pack("Iiiiiii",
89 0x950412de, # Magic
90 0, # Version
91 len(keys), # # of entries
92 7*4, # start of key index
93 7*4+len(keys)*8, # start of value index
94 0, 0) # size and offset of hash table
95 output += array.array("i", offsets).tobytes()
96 output += ids
97 output += strs
98 return output
99
100
101 def make(filename, outfile):
102 ID = 1
103 STR = 2
104 CTXT = 3
105
106 # Compute .mo name from .po name and arguments
107 if filename.endswith('.po'):
108 infile = filename
109 else:
110 infile = filename + '.po'
111 if outfile is None:
112 outfile = os.path.splitext(infile)[0] + '.mo'
113
114 try:
115 with open(infile, 'rb') as f:
116 lines = f.readlines()
117 except OSError as msg:
118 print(msg, file=sys.stderr)
119 sys.exit(1)
120
121 if lines[0].startswith(codecs.BOM_UTF8):
122 print(
123 f"The file {infile} starts with a UTF-8 BOM which is not allowed in .po files.\n"
124 "Please save the file without a BOM and try again.",
125 file=sys.stderr
126 )
127 sys.exit(1)
128
129 section = msgctxt = None
130 msgid = msgstr = b''
131 fuzzy = 0
132
133 # Start off assuming Latin-1, so everything decodes without failure,
134 # until we know the exact encoding
135 encoding = 'latin-1'
136
137 # Parse the catalog
138 lno = 0
139 for l in lines:
140 l = l.decode(encoding)
141 lno += 1
142 # If we get a comment line after a msgstr, this is a new entry
143 if l[0] == '#' and section == STR:
144 add(msgctxt, msgid, msgstr, fuzzy)
145 section = msgctxt = None
146 fuzzy = 0
147 # Record a fuzzy mark
148 if l[:2] == '#,' and 'fuzzy' in l:
149 fuzzy = 1
150 # Skip comments
151 if l[0] == '#':
152 continue
153 # Now we are in a msgid or msgctxt section, output previous section
154 if l.startswith('msgctxt'):
155 if section == STR:
156 add(msgctxt, msgid, msgstr, fuzzy)
157 section = CTXT
158 l = l[7:]
159 msgctxt = b''
160 elif l.startswith('msgid') and not l.startswith('msgid_plural'):
161 if section == STR:
162 if not msgid:
163 # Filter out POT-Creation-Date
164 # See issue #131852
165 msgstr = b''.join(line for line in msgstr.splitlines(True)
166 if not line.startswith(b'POT-Creation-Date:'))
167
168 # See whether there is an encoding declaration
169 p = HeaderParser()
170 charset = p.parsestr(msgstr.decode(encoding)).get_content_charset()
171 if charset:
172 encoding = charset
173 add(msgctxt, msgid, msgstr, fuzzy)
174 msgctxt = None
175 section = ID
176 l = l[5:]
177 msgid = msgstr = b''
178 is_plural = False
179 # This is a message with plural forms
180 elif l.startswith('msgid_plural'):
181 if section != ID:
182 print(f'msgid_plural not preceded by msgid on {infile}:{lno}',
183 file=sys.stderr)
184 sys.exit(1)
185 l = l[12:]
186 msgid += b'\0' # separator of singular and plural
187 is_plural = True
188 # Now we are in a msgstr section
189 elif l.startswith('msgstr'):
190 section = STR
191 if l.startswith('msgstr['):
192 if not is_plural:
193 print(f'plural without msgid_plural on {infile}:{lno}',
194 file=sys.stderr)
195 sys.exit(1)
196 l = l.split(']', 1)[1]
197 if msgstr:
198 msgstr += b'\0' # Separator of the various plural forms
199 else:
200 if is_plural:
201 print(f'indexed msgstr required for plural on {infile}:{lno}',
202 file=sys.stderr)
203 sys.exit(1)
204 l = l[6:]
205 # Skip empty lines
206 l = l.strip()
207 if not l:
208 continue
209 l = ast.literal_eval(l)
210 if section == CTXT:
211 msgctxt += l.encode(encoding)
212 elif section == ID:
213 msgid += l.encode(encoding)
214 elif section == STR:
215 msgstr += l.encode(encoding)
216 else:
217 print(f'Syntax error on {infile}:{lno} before:', file=sys.stderr)
218 print(l, file=sys.stderr)
219 sys.exit(1)
220 # Add last entry
221 if section == STR:
222 add(msgctxt, msgid, msgstr, fuzzy)
223
224 # Compute output
225 output = generate()
226
227 try:
228 with open(outfile,"wb") as f:
229 f.write(output)
230 except OSError as msg:
231 print(msg, file=sys.stderr)
232
233
234 def main():
235 try:
236 opts, args = getopt.getopt(sys.argv[1:], 'hVo:',
237 ['help', 'version', 'output-file='])
238 except getopt.error as msg:
239 usage(1, msg)
240
241 outfile = None
242 # parse options
243 for opt, arg in opts:
244 if opt in ('-h', '--help'):
245 usage(0)
246 elif opt in ('-V', '--version'):
247 print("msgfmt.py", __version__)
248 sys.exit(0)
249 elif opt in ('-o', '--output-file'):
250 outfile = arg
251 # do it
252 if not args:
253 print('No input file given', file=sys.stderr)
254 print("Try `msgfmt --help' for more information.", file=sys.stderr)
255 return
256
257 for filename in args:
258 make(filename, outfile)
259
260
261 if __name__ == '__main__':
262 main()