| 1 |
;;; |
| 2 |
;;; Auxiliary script to generate EUC_JISX0213 <-> Unicode 3.2 table |
| 3 |
;;; |
| 4 |
;;; Copyright (c) 2000-2024 Shiro Kawai <[email protected]> |
| 5 |
;;; |
| 6 |
;;; Redistribution and use in source and binary forms, with or without |
| 7 |
;;; modification, are permitted provided that the following conditions |
| 8 |
;;; are met: |
| 9 |
;;; |
| 10 |
;;; 1. Redistributions of source code must retain the above copyright |
| 11 |
;;; notice, this list of conditions and the following disclaimer. |
| 12 |
;;; |
| 13 |
;;; 2. Redistributions in binary form must reproduce the above copyright |
| 14 |
;;; notice, this list of conditions and the following disclaimer in the |
| 15 |
;;; documentation and/or other materials provided with the distribution. |
| 16 |
;;; |
| 17 |
;;; 3. Neither the name of the authors nor the names of its contributors |
| 18 |
;;; may be used to endorse or promote products derived from this |
| 19 |
;;; software without specific prior written permission. |
| 20 |
;;; |
| 21 |
;;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 22 |
;;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 23 |
;;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 24 |
;;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 25 |
;;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 26 |
;;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED |
| 27 |
;;; TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR |
| 28 |
;;; PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF |
| 29 |
;;; LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING |
| 30 |
;;; NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| 31 |
;;; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 32 |
;;; |
| 33 |
|
| 34 |
(use scheme.list) |
| 35 |
(use gauche.collection) |
| 36 |
|
| 37 |
(define (output-file-header) |
| 38 |
(print "/* This file is automatically generated from") |
| 39 |
(print " EUC-JISX0213 (JIS X 0213:2000 Appendix 3) vs Unicode mapping table") |
| 40 |
(print " Date: 16 Apr 2002 13:10:00 GMT") |
| 41 |
(print " License:") |
| 42 |
(print " Copyright (C) 2001 [email protected], All Rights Reserved.") |
| 43 |
(print " Copyright (C) 2001 I'O, All Rights Reserved.") |
| 44 |
(print " You can use, modify, distribute this table freely.") |
| 45 |
(print "*/")) |
| 46 |
|
| 47 |
;; Parse the table "euc-jp-2000-std.txt", and returns a list of all characters. |
| 48 |
;; Each character is represented as |
| 49 |
;; (EUC-JP-codepoint . UCS4-codepoint) |
| 50 |
;; Or |
| 51 |
;; (EUC-JP-codepoint . (UCS4-codepoint . UCS4-combining-character)) |
| 52 |
(define (parse) |
| 53 |
(generator-fold (^[line knil] |
| 54 |
(rxmatch-case line |
| 55 |
(#/^0x([0-9A-F]+)\s+U\+([0-9A-F]+)(\+[0-9A-F]+)?/ |
| 56 |
(#f code uni uni2) |
| 57 |
(let ((n-code (string->number code 16)) |
| 58 |
(n-uni (string->number uni 16)) |
| 59 |
(n-uni2 (and uni2 (string->number uni2 16)))) |
| 60 |
(if n-uni2 |
| 61 |
(acons n-code (cons n-uni n-uni2) knil) |
| 62 |
(acons n-code n-uni knil)))) |
| 63 |
(else knil))) |
| 64 |
'() |
| 65 |
read-line)) |
| 66 |
|
| 67 |
(define (ucs4->utf8 code) |
| 68 |
(cond ((< code #x80) |
| 69 |
(list code)) |
| 70 |
((< code #x800) |
| 71 |
(list (logior (logand (ash code -6) #x1f) #xc0) |
| 72 |
(logior (logand code #x3f) #x80))) |
| 73 |
((< code #x10000) |
| 74 |
(list (logior (logand (ash code -12) #x0f) #xe0) |
| 75 |
(logior (logand (ash code -6) #x3f) #x80) |
| 76 |
(logior (logand code #x3f) #x80))) |
| 77 |
((< code #x200000) |
| 78 |
(list (logior (logand (ash code -18) #x07) #xf0) |
| 79 |
(logior (logand (ash code -12) #x3f) #x80) |
| 80 |
(logior (logand (ash code -6) #x3f) #x80) |
| 81 |
(logior (logand code #x3f) #x80))) |
| 82 |
((< code #x4000000) |
| 83 |
(list (logior (logand (ash code -24) #x03) #xf8) |
| 84 |
(logior (logand (ash code -18) #x3f) #x80) |
| 85 |
(logior (logand (ash code -12) #x3f) #x80) |
| 86 |
(logior (logand (ash code -6) #x3f) #x80) |
| 87 |
(logior (logand code #x3f) #x80))) |
| 88 |
(else |
| 89 |
(list (logior (logand (ash code -30) #x01) #xfc) |
| 90 |
(logior (logand (ash code -24) #x3f) #x80) |
| 91 |
(logior (logand (ash code -18) #x3f) #x80) |
| 92 |
(logior (logand (ash code -12) #x3f) #x80) |
| 93 |
(logior (logand (ash code -6) #x3f) #x80) |
| 94 |
(logior (logand code #x3f) #x80))) |
| 95 |
)) |
| 96 |
|
| 97 |
;; Generates EUC_JP to UCS map table. |
| 98 |
;; Map is separated in three parts, JISX0201 KANA, JISX0213 plane 1 |
| 99 |
;; and JISX0213 plane 2. |
| 100 |
;; For JISX0201 KANA, the value of the table is UCS2 value. |
| 101 |
;; For JISX0213, the value of the table may be either one of the followings: |
| 102 |
;; (1) UCS4, if the value < 0x30000 |
| 103 |
;; (2) Two UCS2 values, otherwise. |
| 104 |
;; |
| 105 |
(define (generate-eucj->ucs data) |
| 106 |
|
| 107 |
(define (ucsvalue-of cell) |
| 108 |
(let1 ucs (cdr cell) |
| 109 |
(if (pair? ucs) |
| 110 |
(+ (ash (car ucs) 16) (cdr ucs)) |
| 111 |
ucs))) |
| 112 |
|
| 113 |
;; JISX 0201 kana region |
| 114 |
(define (jisx0201 data) |
| 115 |
(print "/****** EUC_JP -> UCS2 JISX0201-KANA (0x8e??) ******/") |
| 116 |
(print "/* index = e2 - 0xa1 */") |
| 117 |
(print "static const unsigned short euc_jisx0201_to_ucs2[] = {") |
| 118 |
(begin0 (write-data data car cdr #x8ea1 #x8ee0 8) |
| 119 |
(print "};") |
| 120 |
(newline))) |
| 121 |
;; JISX 0213 plane 1 region |
| 122 |
(define (jisx0213-1 data) |
| 123 |
(print "/****** EUC_JP -> UCS4 JISX0213 plane 1 *******/") |
| 124 |
(print "/* index = (e1 - 0xa1, e2 - 0xa1) */") |
| 125 |
(print "static const unsigned int euc_jisx0213_1_to_ucs2[][94] = {") |
| 126 |
(begin0 |
| 127 |
(let loop ((e1 #xa1) (data data)) |
| 128 |
(if (= e1 #xff) |
| 129 |
data |
| 130 |
(loop (+ e1 1) |
| 131 |
(begin (print " {") |
| 132 |
(begin0 (write-data data car ucsvalue-of |
| 133 |
(+ (* e1 256) #xa1) |
| 134 |
(+ (* e1 256) #xff) |
| 135 |
8) |
| 136 |
(print " },")))))) |
| 137 |
(print "};") |
| 138 |
(newline))) |
| 139 |
;; JISX 0213 plane 2 region |
| 140 |
;; tricky: the second byte is one of 0xa1, 0xa3, 0xa4, 0xa5, 0xa8, |
| 141 |
;; 0xac, 0xad, 0xae, 0xaf, and 0xee - 0xfe. |
| 142 |
(define (jisx0213-2 data) |
| 143 |
(define e1list '(#xa1 #xa3 #xa4 #xa5 #xa8 #xac #xad #xae #xaf |
| 144 |
#xee #xef #xf0 #xf1 #xf2 #xf3 #xf4 #xf5 #xf6 |
| 145 |
#xf7 #xf8 #xf9 #xfa #xfb #xfc #xfd #xfe)) |
| 146 |
(print "/****** EUC_JP -> UCS4 JISX0213 plane 2 (0x8f????) *****/") |
| 147 |
(print "/* table to traslate second byte into the first index */") |
| 148 |
(print "static const short euc_jisx0213_2_index[] = {") |
| 149 |
(let loop ((e1 #xa1) (count 0) (e1list e1list)) |
| 150 |
(cond ((= e1 #xff)) |
| 151 |
((= e1 (car e1list)) |
| 152 |
(format #t " ~a," count) |
| 153 |
(loop (+ e1 1) (+ count 1) (cdr e1list))) |
| 154 |
(else |
| 155 |
(format #t " -1,") |
| 156 |
(loop (+ e1 1) count e1list)))) |
| 157 |
(print "\n};\n") |
| 158 |
(print "/* index = (e1table, e2 - 0xa1) */") |
| 159 |
(print "static const unsigned int euc_jisx0213_2_to_ucs2[][94] = {") |
| 160 |
(let loop ((e1list e1list) (data data)) |
| 161 |
(if (null? e1list) |
| 162 |
data |
| 163 |
(loop (cdr e1list) |
| 164 |
(let1 e1v (* (car e1list) 256) |
| 165 |
(print " {") |
| 166 |
(begin0 (write-data data car ucsvalue-of |
| 167 |
(+ e1v #x8f00a1) |
| 168 |
(+ e1v #x8f00ff) |
| 169 |
8) |
| 170 |
(print " },")))))) |
| 171 |
(print "};") |
| 172 |
(newline)) |
| 173 |
|
| 174 |
;; Body of generate-eucj->ucs |
| 175 |
(output-file-header) |
| 176 |
(let* ((sorted (sort data (lambda (a b) (< (car a) (car b))))) |
| 177 |
(data (jisx0201 sorted)) |
| 178 |
(data1 (jisx0213-1 data))) |
| 179 |
(jisx0213-2 data1) |
| 180 |
#f)) |
| 181 |
|
| 182 |
(define (write-data data key-of value-of start end unit) |
| 183 |
(let loop ((column -1) |
| 184 |
(data data) |
| 185 |
(next start)) |
| 186 |
(cond ((>= next end) data) |
| 187 |
((>= column unit) (newline) (loop -1 data next)) |
| 188 |
((= column -1) |
| 189 |
(format #t " /* 0x~4,'0x -- 0x~4,'0x */\n" |
| 190 |
next (min (+ next unit -1) (- end 1))) |
| 191 |
(loop 0 data next)) |
| 192 |
((pair? data) |
| 193 |
(let1 key (key-of (car data)) |
| 194 |
(cond ((< key next) (loop column (cdr data) next)) |
| 195 |
((= key next) |
| 196 |
(format #t " 0x~4,'0x," (value-of (car data))) |
| 197 |
(loop (+ column 1) (cdr data) (+ next 1))) |
| 198 |
(else |
| 199 |
(format #t " 0x0000,") |
| 200 |
(loop (+ column 1) data (+ next 1)))))) |
| 201 |
(else |
| 202 |
(format #t " 0x0000,") |
| 203 |
(loop (+ column 1) data (+ next 1)))))) |
| 204 |
|
| 205 |
;; Generates UCS to EUC_JP table |
| 206 |
;; The table is constructed hierarchically. |
| 207 |
;; This procedure first creates a tree of tables using vectors, |
| 208 |
;; then writes out them. |
| 209 |
|
| 210 |
(define (generate-utf8->eucj data) |
| 211 |
(define root (make-hash-table 'eqv?)) |
| 212 |
|
| 213 |
(define (ensure-node container ref set key) |
| 214 |
(or (ref container key #f) |
| 215 |
(rlet1 v (make-vector 64 #f) |
| 216 |
(set container key v)))) |
| 217 |
|
| 218 |
(define (intern utf8 euc container) |
| 219 |
(cond ((null? (cdr utf8)) |
| 220 |
(vector-set! container (- (car utf8) #x80) euc)) |
| 221 |
(else |
| 222 |
(intern (cdr utf8) euc |
| 223 |
(ensure-node container vector-ref vector-set! |
| 224 |
(- (car utf8) #x80)))) |
| 225 |
)) |
| 226 |
|
| 227 |
;; Look up EUC |
| 228 |
;; NB: the original table has entries for 0x80-0x9f as <control>, but |
| 229 |
;; they are not really assigned, so we exclude them. |
| 230 |
(define (euc-entry data) |
| 231 |
(cond ((not data) 0) |
| 232 |
((<= data #x9f) 0) |
| 233 |
((> data #xffff) (- (logand data #xffff) #x8000)) |
| 234 |
(else data))) |
| 235 |
|
| 236 |
;; emit the table of 2-byte utf8 range |
| 237 |
(define (emit-utf2b) |
| 238 |
(dolist (u0 '(#xc2 #xc3 #xc4 #xc5 #xc7 #xc9 #xca #xcb #xcc #xce #xcf |
| 239 |
#xd0 #xd1)) |
| 240 |
(format #t "\n/* 2-byte UTF8: [~X XX] */\n" u0) |
| 241 |
(format #t "static const unsigned short utf2euc_~x[64] = {\n" u0) |
| 242 |
(let1 v (hash-table-get root u0) |
| 243 |
(dotimes (i 64) |
| 244 |
(format #t " 0x~4,'0x," (euc-entry (vector-ref v i))) |
| 245 |
(when (= (modulo i 8) 7) (newline)))) |
| 246 |
(print "};"))) |
| 247 |
|
| 248 |
;; emit the table of 3-byte utf8 range |
| 249 |
(define (emit-utf3b) |
| 250 |
(dolist (u0 '(#xe2 #xe3 #xe4 #xe5 #xe6 #xe7 #xe8 #xe9 #xef)) |
| 251 |
(format #t "\n/* 3-byte UTF8: [~X XX XX] */\n" u0) |
| 252 |
(format #t "static const unsigned char utf2euc_~x[64] = {\n" u0) |
| 253 |
(let* ((v1 (hash-table-get root u0)) |
| 254 |
(s64 (iota 64))) |
| 255 |
(fold (lambda (u1 count) |
| 256 |
(begin0 |
| 257 |
(if (vector-ref v1 u1) |
| 258 |
(begin (format #t " ~2d," count) (+ count 1)) |
| 259 |
(begin (format #t " 0,") count)) |
| 260 |
(when (= (modulo u1 8) 7) (newline)))) |
| 261 |
1 |
| 262 |
s64) |
| 263 |
(print "};\n") |
| 264 |
(format #t "static const unsigned short utf2euc_~x_xx[][64] = {\n" u0) |
| 265 |
(for-each (lambda (u1) |
| 266 |
(and-let* ((v2 (vector-ref v1 u1))) |
| 267 |
(format #t " {/* [~X ~X XX] */\n" u0 (+ u1 #x80)) |
| 268 |
(dotimes (i 64) |
| 269 |
(format #t " 0x~4,'0x," (euc-entry (vector-ref v2 i))) |
| 270 |
(when (= (modulo i 8) 7) (newline))) |
| 271 |
(print " },\n"))) |
| 272 |
s64) |
| 273 |
(print "};\n")))) |
| 274 |
|
| 275 |
;; emit the table of 4-byte utf8 range. u0 is always #xf0. |
| 276 |
;; |
| 277 |
(define (emit-utf4b) |
| 278 |
(let1 v0 (hash-table-get root #xf0) |
| 279 |
(dolist (u1 '(#xa0 #xa1 #xa2 #xa3 #xa4 #xa5 #xa6 #xa7 #xa8 #xa9 #xaa)) |
| 280 |
(format #t "\n/* 4-byte UTF8: [F0 ~X XX XX] */\n" u1) |
| 281 |
(format #t "static const unsigned short utf2euc_f0_~x[] = {\n" u1) |
| 282 |
(let1 v1 (vector-ref v0 (- u1 #x80)) |
| 283 |
(dotimes (u2 64) |
| 284 |
(and-let* ((v2 (vector-ref v1 u2))) |
| 285 |
(dotimes (u3 64) |
| 286 |
(and-let* ((euc (vector-ref v2 u3))) |
| 287 |
(format #t " 0x~2,'0x~2,'0x, 0x~4,'0x, /* [F0 ~X ~2,'0X ~2,'0X] */\n" |
| 288 |
(+ u2 #x80) (+ u3 #x80) (euc-entry euc) u1 u2 u3)))))) |
| 289 |
(print " 0, 0\n};\n")))) |
| 290 |
|
| 291 |
;; build the table tree |
| 292 |
(dolist (entry data) |
| 293 |
(unless (pair? (cdr entry)) |
| 294 |
(let1 utf8 (ucs4->utf8 (cdr entry)) |
| 295 |
(unless (null? (cdr utf8)) |
| 296 |
(intern (cdr utf8) (car entry) |
| 297 |
(ensure-node root hash-table-get hash-table-put! (car utf8)) |
| 298 |
))))) |
| 299 |
|
| 300 |
;; emit the tables |
| 301 |
(output-file-header) |
| 302 |
(emit-utf2b) |
| 303 |
(emit-utf3b) |
| 304 |
(emit-utf4b) |
| 305 |
) |
| 306 |
|
| 307 |
;; |
| 308 |
;; Bind things together |
| 309 |
;; |
| 310 |
|
| 311 |
(define (main args) |
| 312 |
(let1 data (with-input-from-file (cadr args) parse) |
| 313 |
(with-output-to-file "eucj2ucs.c" |
| 314 |
(lambda () (generate-eucj->ucs data))) |
| 315 |
(with-output-to-file "ucs2eucj.c" |
| 316 |
(lambda () (generate-utf8->eucj data))) |
| 317 |
) |
| 318 |
0) |