Debian Codegraph @

ext/charconv/gen-eucjtab.scm

raw
1 ;;;
2 ;;; Auxiliary script to generate EUC_JISX0213 <-> Unicode 3.2 table
3 ;;;
4 ;;; Copyright (c) 2000-2024 Shiro Kawai <[email protected]>
5 ;;;
6 ;;; Redistribution and use in source and binary forms, with or without
7 ;;; modification, are permitted provided that the following conditions
8 ;;; are met:
9 ;;;
10 ;;; 1. Redistributions of source code must retain the above copyright
11 ;;; notice, this list of conditions and the following disclaimer.
12 ;;;
13 ;;; 2. Redistributions in binary form must reproduce the above copyright
14 ;;; notice, this list of conditions and the following disclaimer in the
15 ;;; documentation and/or other materials provided with the distribution.
16 ;;;
17 ;;; 3. Neither the name of the authors nor the names of its contributors
18 ;;; may be used to endorse or promote products derived from this
19 ;;; software without specific prior written permission.
20 ;;;
21 ;;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 ;;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 ;;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 ;;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 ;;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 ;;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
27 ;;; TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
28 ;;; PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
29 ;;; LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
30 ;;; NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
31 ;;; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 ;;;
33
34 (use scheme.list)
35 (use gauche.collection)
36
37 (define (output-file-header)
38 (print "/* This file is automatically generated from")
39 (print " EUC-JISX0213 (JIS X 0213:2000 Appendix 3) vs Unicode mapping table")
40 (print " Date: 16 Apr 2002 13:10:00 GMT")
41 (print " License:")
42 (print " Copyright (C) 2001 [email protected], All Rights Reserved.")
43 (print " Copyright (C) 2001 I'O, All Rights Reserved.")
44 (print " You can use, modify, distribute this table freely.")
45 (print "*/"))
46
47 ;; Parse the table "euc-jp-2000-std.txt", and returns a list of all characters.
48 ;; Each character is represented as
49 ;; (EUC-JP-codepoint . UCS4-codepoint)
50 ;; Or
51 ;; (EUC-JP-codepoint . (UCS4-codepoint . UCS4-combining-character))
52 (define (parse)
53 (generator-fold (^[line knil]
54 (rxmatch-case line
55 (#/^0x([0-9A-F]+)\s+U\+([0-9A-F]+)(\+[0-9A-F]+)?/
56 (#f code uni uni2)
57 (let ((n-code (string->number code 16))
58 (n-uni (string->number uni 16))
59 (n-uni2 (and uni2 (string->number uni2 16))))
60 (if n-uni2
61 (acons n-code (cons n-uni n-uni2) knil)
62 (acons n-code n-uni knil))))
63 (else knil)))
64 '()
65 read-line))
66
67 (define (ucs4->utf8 code)
68 (cond ((< code #x80)
69 (list code))
70 ((< code #x800)
71 (list (logior (logand (ash code -6) #x1f) #xc0)
72 (logior (logand code #x3f) #x80)))
73 ((< code #x10000)
74 (list (logior (logand (ash code -12) #x0f) #xe0)
75 (logior (logand (ash code -6) #x3f) #x80)
76 (logior (logand code #x3f) #x80)))
77 ((< code #x200000)
78 (list (logior (logand (ash code -18) #x07) #xf0)
79 (logior (logand (ash code -12) #x3f) #x80)
80 (logior (logand (ash code -6) #x3f) #x80)
81 (logior (logand code #x3f) #x80)))
82 ((< code #x4000000)
83 (list (logior (logand (ash code -24) #x03) #xf8)
84 (logior (logand (ash code -18) #x3f) #x80)
85 (logior (logand (ash code -12) #x3f) #x80)
86 (logior (logand (ash code -6) #x3f) #x80)
87 (logior (logand code #x3f) #x80)))
88 (else
89 (list (logior (logand (ash code -30) #x01) #xfc)
90 (logior (logand (ash code -24) #x3f) #x80)
91 (logior (logand (ash code -18) #x3f) #x80)
92 (logior (logand (ash code -12) #x3f) #x80)
93 (logior (logand (ash code -6) #x3f) #x80)
94 (logior (logand code #x3f) #x80)))
95 ))
96
97 ;; Generates EUC_JP to UCS map table.
98 ;; Map is separated in three parts, JISX0201 KANA, JISX0213 plane 1
99 ;; and JISX0213 plane 2.
100 ;; For JISX0201 KANA, the value of the table is UCS2 value.
101 ;; For JISX0213, the value of the table may be either one of the followings:
102 ;; (1) UCS4, if the value < 0x30000
103 ;; (2) Two UCS2 values, otherwise.
104 ;;
105 (define (generate-eucj->ucs data)
106
107 (define (ucsvalue-of cell)
108 (let1 ucs (cdr cell)
109 (if (pair? ucs)
110 (+ (ash (car ucs) 16) (cdr ucs))
111 ucs)))
112
113 ;; JISX 0201 kana region
114 (define (jisx0201 data)
115 (print "/****** EUC_JP -> UCS2 JISX0201-KANA (0x8e??) ******/")
116 (print "/* index = e2 - 0xa1 */")
117 (print "static const unsigned short euc_jisx0201_to_ucs2[] = {")
118 (begin0 (write-data data car cdr #x8ea1 #x8ee0 8)
119 (print "};")
120 (newline)))
121 ;; JISX 0213 plane 1 region
122 (define (jisx0213-1 data)
123 (print "/****** EUC_JP -> UCS4 JISX0213 plane 1 *******/")
124 (print "/* index = (e1 - 0xa1, e2 - 0xa1) */")
125 (print "static const unsigned int euc_jisx0213_1_to_ucs2[][94] = {")
126 (begin0
127 (let loop ((e1 #xa1) (data data))
128 (if (= e1 #xff)
129 data
130 (loop (+ e1 1)
131 (begin (print " {")
132 (begin0 (write-data data car ucsvalue-of
133 (+ (* e1 256) #xa1)
134 (+ (* e1 256) #xff)
135 8)
136 (print " },"))))))
137 (print "};")
138 (newline)))
139 ;; JISX 0213 plane 2 region
140 ;; tricky: the second byte is one of 0xa1, 0xa3, 0xa4, 0xa5, 0xa8,
141 ;; 0xac, 0xad, 0xae, 0xaf, and 0xee - 0xfe.
142 (define (jisx0213-2 data)
143 (define e1list '(#xa1 #xa3 #xa4 #xa5 #xa8 #xac #xad #xae #xaf
144 #xee #xef #xf0 #xf1 #xf2 #xf3 #xf4 #xf5 #xf6
145 #xf7 #xf8 #xf9 #xfa #xfb #xfc #xfd #xfe))
146 (print "/****** EUC_JP -> UCS4 JISX0213 plane 2 (0x8f????) *****/")
147 (print "/* table to traslate second byte into the first index */")
148 (print "static const short euc_jisx0213_2_index[] = {")
149 (let loop ((e1 #xa1) (count 0) (e1list e1list))
150 (cond ((= e1 #xff))
151 ((= e1 (car e1list))
152 (format #t " ~a," count)
153 (loop (+ e1 1) (+ count 1) (cdr e1list)))
154 (else
155 (format #t " -1,")
156 (loop (+ e1 1) count e1list))))
157 (print "\n};\n")
158 (print "/* index = (e1table, e2 - 0xa1) */")
159 (print "static const unsigned int euc_jisx0213_2_to_ucs2[][94] = {")
160 (let loop ((e1list e1list) (data data))
161 (if (null? e1list)
162 data
163 (loop (cdr e1list)
164 (let1 e1v (* (car e1list) 256)
165 (print " {")
166 (begin0 (write-data data car ucsvalue-of
167 (+ e1v #x8f00a1)
168 (+ e1v #x8f00ff)
169 8)
170 (print " },"))))))
171 (print "};")
172 (newline))
173
174 ;; Body of generate-eucj->ucs
175 (output-file-header)
176 (let* ((sorted (sort data (lambda (a b) (< (car a) (car b)))))
177 (data (jisx0201 sorted))
178 (data1 (jisx0213-1 data)))
179 (jisx0213-2 data1)
180 #f))
181
182 (define (write-data data key-of value-of start end unit)
183 (let loop ((column -1)
184 (data data)
185 (next start))
186 (cond ((>= next end) data)
187 ((>= column unit) (newline) (loop -1 data next))
188 ((= column -1)
189 (format #t " /* 0x~4,'0x -- 0x~4,'0x */\n"
190 next (min (+ next unit -1) (- end 1)))
191 (loop 0 data next))
192 ((pair? data)
193 (let1 key (key-of (car data))
194 (cond ((< key next) (loop column (cdr data) next))
195 ((= key next)
196 (format #t " 0x~4,'0x," (value-of (car data)))
197 (loop (+ column 1) (cdr data) (+ next 1)))
198 (else
199 (format #t " 0x0000,")
200 (loop (+ column 1) data (+ next 1))))))
201 (else
202 (format #t " 0x0000,")
203 (loop (+ column 1) data (+ next 1))))))
204
205 ;; Generates UCS to EUC_JP table
206 ;; The table is constructed hierarchically.
207 ;; This procedure first creates a tree of tables using vectors,
208 ;; then writes out them.
209
210 (define (generate-utf8->eucj data)
211 (define root (make-hash-table 'eqv?))
212
213 (define (ensure-node container ref set key)
214 (or (ref container key #f)
215 (rlet1 v (make-vector 64 #f)
216 (set container key v))))
217
218 (define (intern utf8 euc container)
219 (cond ((null? (cdr utf8))
220 (vector-set! container (- (car utf8) #x80) euc))
221 (else
222 (intern (cdr utf8) euc
223 (ensure-node container vector-ref vector-set!
224 (- (car utf8) #x80))))
225 ))
226
227 ;; Look up EUC
228 ;; NB: the original table has entries for 0x80-0x9f as <control>, but
229 ;; they are not really assigned, so we exclude them.
230 (define (euc-entry data)
231 (cond ((not data) 0)
232 ((<= data #x9f) 0)
233 ((> data #xffff) (- (logand data #xffff) #x8000))
234 (else data)))
235
236 ;; emit the table of 2-byte utf8 range
237 (define (emit-utf2b)
238 (dolist (u0 '(#xc2 #xc3 #xc4 #xc5 #xc7 #xc9 #xca #xcb #xcc #xce #xcf
239 #xd0 #xd1))
240 (format #t "\n/* 2-byte UTF8: [~X XX] */\n" u0)
241 (format #t "static const unsigned short utf2euc_~x[64] = {\n" u0)
242 (let1 v (hash-table-get root u0)
243 (dotimes (i 64)
244 (format #t " 0x~4,'0x," (euc-entry (vector-ref v i)))
245 (when (= (modulo i 8) 7) (newline))))
246 (print "};")))
247
248 ;; emit the table of 3-byte utf8 range
249 (define (emit-utf3b)
250 (dolist (u0 '(#xe2 #xe3 #xe4 #xe5 #xe6 #xe7 #xe8 #xe9 #xef))
251 (format #t "\n/* 3-byte UTF8: [~X XX XX] */\n" u0)
252 (format #t "static const unsigned char utf2euc_~x[64] = {\n" u0)
253 (let* ((v1 (hash-table-get root u0))
254 (s64 (iota 64)))
255 (fold (lambda (u1 count)
256 (begin0
257 (if (vector-ref v1 u1)
258 (begin (format #t " ~2d," count) (+ count 1))
259 (begin (format #t " 0,") count))
260 (when (= (modulo u1 8) 7) (newline))))
261 1
262 s64)
263 (print "};\n")
264 (format #t "static const unsigned short utf2euc_~x_xx[][64] = {\n" u0)
265 (for-each (lambda (u1)
266 (and-let* ((v2 (vector-ref v1 u1)))
267 (format #t " {/* [~X ~X XX] */\n" u0 (+ u1 #x80))
268 (dotimes (i 64)
269 (format #t " 0x~4,'0x," (euc-entry (vector-ref v2 i)))
270 (when (= (modulo i 8) 7) (newline)))
271 (print " },\n")))
272 s64)
273 (print "};\n"))))
274
275 ;; emit the table of 4-byte utf8 range. u0 is always #xf0.
276 ;;
277 (define (emit-utf4b)
278 (let1 v0 (hash-table-get root #xf0)
279 (dolist (u1 '(#xa0 #xa1 #xa2 #xa3 #xa4 #xa5 #xa6 #xa7 #xa8 #xa9 #xaa))
280 (format #t "\n/* 4-byte UTF8: [F0 ~X XX XX] */\n" u1)
281 (format #t "static const unsigned short utf2euc_f0_~x[] = {\n" u1)
282 (let1 v1 (vector-ref v0 (- u1 #x80))
283 (dotimes (u2 64)
284 (and-let* ((v2 (vector-ref v1 u2)))
285 (dotimes (u3 64)
286 (and-let* ((euc (vector-ref v2 u3)))
287 (format #t " 0x~2,'0x~2,'0x, 0x~4,'0x, /* [F0 ~X ~2,'0X ~2,'0X] */\n"
288 (+ u2 #x80) (+ u3 #x80) (euc-entry euc) u1 u2 u3))))))
289 (print " 0, 0\n};\n"))))
290
291 ;; build the table tree
292 (dolist (entry data)
293 (unless (pair? (cdr entry))
294 (let1 utf8 (ucs4->utf8 (cdr entry))
295 (unless (null? (cdr utf8))
296 (intern (cdr utf8) (car entry)
297 (ensure-node root hash-table-get hash-table-put! (car utf8))
298 )))))
299
300 ;; emit the tables
301 (output-file-header)
302 (emit-utf2b)
303 (emit-utf3b)
304 (emit-utf4b)
305 )
306
307 ;;
308 ;; Bind things together
309 ;;
310
311 (define (main args)
312 (let1 data (with-input-from-file (cadr args) parse)
313 (with-output-to-file "eucj2ucs.c"
314 (lambda () (generate-eucj->ucs data)))
315 (with-output-to-file "ucs2eucj.c"
316 (lambda () (generate-utf8->eucj data)))
317 )
318 0)