| 1 |
#!/usr/bin/python |
| 2 |
################################################################### |
| 3 |
# unimap.py |
| 4 |
# Generates utf8raw.tex file containing math character definitions |
| 5 |
# from modified Unicode character database unimap.txt. |
| 6 |
# |
| 7 |
# Copyright (C) 2003 David Necas (Yeti) |
| 8 |
# |
| 9 |
# This program is free software; you can redistribute it and/or modify |
| 10 |
# it under the terms of the GNU General Public License as published by |
| 11 |
# the Free Software Foundation; either version 2 of the License, or |
| 12 |
# (at your option) any later version. |
| 13 |
# |
| 14 |
# This program is distributed in the hope that it will be useful, |
| 15 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 16 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 17 |
# GNU General Public License for more details. |
| 18 |
# |
| 19 |
# You should have received a copy of the GNU General Public License |
| 20 |
# along with this program; if not, write to the Free Software |
| 21 |
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA |
| 22 |
# |
| 23 |
################################################################### |
| 24 |
# Usage: ./unimap.py |
| 25 |
# It takes no arguments, input and output file names are `unimap.txt' and |
| 26 |
# `utf8raw.tex' and are hardcoded below (should be fixed once someone finds |
| 27 |
# a reason for running it on different files). |
| 28 |
# |
| 29 |
# The source file unimap.txt is basically Unicode character name list |
| 30 |
# http://www.unicode.org/Public/UNIDATA/NamesList.txt |
| 31 |
# with additional lines defining TeX expansions of particular characters. |
| 32 |
# These lines have format similar to other character info lines, with |
| 33 |
# backslash (\) as the line type mark: |
| 34 |
# <Tab>\<space>\TeXcontrolsequence |
| 35 |
# \TeXcontrolsequence is the control sequence the character should be |
| 36 |
# mapped to. |
| 37 |
# |
| 38 |
# The NamesList.txt file is huge and the number of supported characters is |
| 39 |
# still relatively small. Thus only diff NamesList.txt -> unimap.txt is |
| 40 |
# normally distributed (unimap.diff). Once you have NamesList.txt, you can |
| 41 |
# create unimap.txt with following command: |
| 42 |
# patch -o unimap.txt NamesList.txt unimap.diff |
| 43 |
|
| 44 |
import re |
| 45 |
from time import asctime, gmtime |
| 46 |
|
| 47 |
database = 'unimap.txt' # Input file |
| 48 |
output = 'utf8raw.tex' # Output file |
| 49 |
|
| 50 |
# Compatibility with Pyhton-2.1 |
| 51 |
if not __builtins__.__dict__.has_key('True'): |
| 52 |
True = 1; False = 0 |
| 53 |
if not __builtins__.__dict__.has_key('file'): |
| 54 |
file = open |
| 55 |
if not __builtins__.__dict__.has_key('dict'): |
| 56 |
def dict(l): |
| 57 |
d = {} |
| 58 |
for x in l: d[x[0]] = x[1] |
| 59 |
return d |
| 60 |
|
| 61 |
charline_re = re.compile(r'^[0-9A-F]{4,}\t') |
| 62 |
comsect_re = re.compile(r'^@+\t') |
| 63 |
line_template = '\\mubyte %s %s\\endmubyte %% U+%04X %s\n' |
| 64 |
|
| 65 |
class LineType: |
| 66 |
"""NamesList.txt line types. Something between an enum and a hash.""" |
| 67 |
Empty = 0 |
| 68 |
Comment = '++' |
| 69 |
Section = '@@' |
| 70 |
Character = 'AA' |
| 71 |
IsNot = 'x' |
| 72 |
Alias = '=' |
| 73 |
Note = '*' |
| 74 |
Combining = ':' |
| 75 |
Render = '#' |
| 76 |
TeX = '\\' |
| 77 |
|
| 78 |
LineType.map = dict([(val, name) for name, val in LineType.__dict__.items() |
| 79 |
if name[0].isupper()]) |
| 80 |
|
| 81 |
def linetype(line): |
| 82 |
"""Determine line type of a NamesList.txt file and extract the text.""" |
| 83 |
if not line: |
| 84 |
return LineType.Empty, None |
| 85 |
if line.startswith('@'): |
| 86 |
if line[1:].startswith('@') or line[1:].startswith('+'): |
| 87 |
return LineType.Comment, comsect_re.sub('', line).strip() |
| 88 |
return LineType.Section, comsect_re.sub('', line).strip() |
| 89 |
m = charline_re.match(line) |
| 90 |
if m: |
| 91 |
return LineType.Character, (int(line[:m.end()], 16), |
| 92 |
line[m.end():].strip().lower()) |
| 93 |
if not line.startswith('\t'): |
| 94 |
raise ValueError, 'Queer line doesn\'t start with @ or Tab' |
| 95 |
line = line.strip() |
| 96 |
if not line: |
| 97 |
return LineType.Empty, None |
| 98 |
if not LineType.map.has_key(line[0]): |
| 99 |
raise ValueError, 'Queer character info line (marker %s)' % line[0] |
| 100 |
return line[0], line[1:].strip() |
| 101 |
|
| 102 |
def utf8chars(u): |
| 103 |
"""Format an Unicode character in a \\mubyte-friendly style. |
| 104 |
|
| 105 |
character ordinal value should be < 0x10000.""" |
| 106 |
if u < 0x80: |
| 107 |
return '^^%02x' % u |
| 108 |
if u < 0x800: |
| 109 |
return '^^%02x^^%02x' % (0xc0 | (u >> 6), |
| 110 |
0x80 | (0x3f & u)) |
| 111 |
return '^^%02x^^%02x^^%02x' % (0xe0 | (u >> 12), |
| 112 |
0x80 | (0x3f & (u >> 6)), |
| 113 |
0x80 | (0x3f & u)) |
| 114 |
|
| 115 |
fh = file(database, 'r') |
| 116 |
# skip some initial noise |
| 117 |
while True: |
| 118 |
line = fh.readline() |
| 119 |
try: |
| 120 |
typ, val = linetype(line) |
| 121 |
except ValueError: |
| 122 |
continue |
| 123 |
if typ == LineType.Section: |
| 124 |
break |
| 125 |
|
| 126 |
fw = file(output, 'w') |
| 127 |
fw.write('%% Generated from %s %s\n' % (database, asctime(gmtime()))) |
| 128 |
while typ: |
| 129 |
if typ == LineType.Section: |
| 130 |
sect = val |
| 131 |
elif typ == LineType.Character: |
| 132 |
char = val |
| 133 |
elif typ == LineType.TeX: |
| 134 |
if not val.startswith('\\'): |
| 135 |
raise ValueError, '%s is not a control seq (U%X)' % (val, char[0]) |
| 136 |
if sect: |
| 137 |
fw.write('\n%% %s\n' % sect) |
| 138 |
sect = None |
| 139 |
fw.write(line_template % (val, utf8chars(char[0]), char[0], char[1])) |
| 140 |
typ, val = linetype(fh.readline()) |
| 141 |
fh.close() |
| 142 |
fw.write('\n\\endinput\n') |
| 143 |
fw.close() |
| 144 |
|