| 1 |
;;; |
| 2 |
;;; Generating conversion state machine |
| 3 |
;;; |
| 4 |
|
| 5 |
;; For now, we just generate conversion matrix and some other tables. |
| 6 |
;; Eventually, we'll define DSL from which convertion C routines are generated. |
| 7 |
|
| 8 |
(use gauche.cgen) |
| 9 |
(use gauche.mop.instance-pool) |
| 10 |
(use srfi.42) |
| 11 |
(use util.match) |
| 12 |
|
| 13 |
;; |
| 14 |
;; Encoding Scheme |
| 15 |
;; |
| 16 |
|
| 17 |
(define-class <encoding-scheme> (<instance-pool-mixin>) |
| 18 |
((name :init-keyword :name) ;symbol, short Scheme name |
| 19 |
(c-enum :init-keyword :c-enum) ;symbol, C-enum name |
| 20 |
(aliases :init-keyword :aliases) ;list of strings, alias names. |
| 21 |
; we ignore '_' and '-' for match. |
| 22 |
(reset :init-keyword :reset ;string name of c reset procedure |
| 23 |
:init-value #f) ; reset proedure is to reset the |
| 24 |
; output state. |
| 25 |
(states :init-keyword :states ;If stateful encoding, list of states |
| 26 |
:init-value '()) ; (suitable for C enums) |
| 27 |
(initial-state :init-keyword :initial-state |
| 28 |
:init-value #f) |
| 29 |
)) |
| 30 |
(define-method write-object ((obj <encoding-scheme>) port) |
| 31 |
(format port "#<encoding ~a>" (~ obj'name))) |
| 32 |
|
| 33 |
(define-syntax define-encoding-scheme |
| 34 |
(syntax-rules () |
| 35 |
[(_ name c-enum aliases . opts) |
| 36 |
(define name (make <encoding-scheme> |
| 37 |
:name 'name |
| 38 |
:c-enum 'c-enum |
| 39 |
:aliases 'aliases |
| 40 |
. opts))])) |
| 41 |
|
| 42 |
(define (encoding-schemes) (instance-pool->list <encoding-scheme>)) |
| 43 |
|
| 44 |
(define (find-encoding-scheme name) |
| 45 |
(instance-pool-find <encoding-scheme> (^e (eq? name (~ e'name))))) |
| 46 |
|
| 47 |
(define (emit-encoding-tables) |
| 48 |
(cgen-extern "/* Encodings */" |
| 49 |
"enum {") |
| 50 |
(do-ec (: e (encoding-schemes)) |
| 51 |
(cgen-extern #" ~(~ e'c-enum),")) |
| 52 |
(cgen-extern " NUM_JCODES" |
| 53 |
"};" |
| 54 |
"") |
| 55 |
|
| 56 |
(do-ec (: states (delete-duplicates (map (cut ~ <>'states) |
| 57 |
(encoding-schemes)))) |
| 58 |
(if (not (null? states))) |
| 59 |
(begin |
| 60 |
(cgen-extern "enum {") |
| 61 |
(do-ec (: s states) (cgen-extern #" ~|s|,")) |
| 62 |
(cgen-extern "};" ""))) |
| 63 |
|
| 64 |
(cgen-body "static struct conv_support_rec conv_supports[] = {") |
| 65 |
(do-ec (: e (encoding-schemes)) |
| 66 |
(: a (~ e'aliases)) |
| 67 |
(cgen-body (format " { ~15a, ~15a }," |
| 68 |
(cgen-safe-string a) |
| 69 |
(~ e 'c-enum)))) |
| 70 |
(cgen-body " { NULL, 0 }" |
| 71 |
"};")) |
| 72 |
|
| 73 |
;; |
| 74 |
;; Conversion |
| 75 |
;; |
| 76 |
|
| 77 |
;; Conversion is defined with one or two procedure name(s). |
| 78 |
|
| 79 |
(define-class <conversion> (<instance-pool-mixin>) |
| 80 |
((procs :init-keyword :procs) ;name(s) of conversion routines |
| 81 |
(from :init-keyword :from) ;list of encoding names |
| 82 |
(to :init-keyword :to) ;list of encoding names |
| 83 |
(proc-name :init-value #f) ;final proc name (automatically set) |
| 84 |
)) |
| 85 |
|
| 86 |
(define-method initialize ((c <conversion>) initargs) |
| 87 |
(define (ssplit sym) (string-split (x->string sym) #\_)) |
| 88 |
(next-method) |
| 89 |
(match (~ c 'procs) |
| 90 |
[(f0) (set! (~ c'proc-name) f0)] |
| 91 |
;; autogenerate fused function name |
| 92 |
[(f0 f1) |
| 93 |
(set! (~ c'proc-name) #"~(car (ssplit f0))_~(cadr (ssplit f1))")] |
| 94 |
[(f0 f1 f2) |
| 95 |
(set! (~ c'proc-name) #"~(car (ssplit f0))_~(cadr (ssplit f2))")])) |
| 96 |
|
| 97 |
(define-syntax define-conversion |
| 98 |
(syntax-rules () |
| 99 |
[(_ (procs ...) from to) |
| 100 |
(make <conversion> :procs '(procs ...) :from 'from :to 'to)] |
| 101 |
[(_ proc from to) |
| 102 |
(make <conversion> :procs '(proc) :from 'from :to 'to)])) |
| 103 |
|
| 104 |
;; Find a conversion from FROM encoding to TO encoding (both <encoding-scheme>). |
| 105 |
;; If it's not defined, we search for a combination and fuse them. |
| 106 |
;; - If there's a direct path from->utf8 and utf8->to, use them. |
| 107 |
;; - If there's a direct path from->eucj and eucj->to, use them. |
| 108 |
;; - Otherwise, we search direct path hopping eucj and utf8. |
| 109 |
;; Here, 'direct path' means the converion is supported natively and not |
| 110 |
;; fused. |
| 111 |
(define (find-conversion from to) |
| 112 |
(define (find-1 f t) |
| 113 |
($ instance-pool-find <conversion> |
| 114 |
(^c (and (memq (~ f'name) (~ c'from)) |
| 115 |
(memq (~ t'name) (~ c'to)))))) |
| 116 |
(define (direct-path? c) |
| 117 |
(= (length (~ c'procs)) 1)) |
| 118 |
(define (find-direct f t) |
| 119 |
(and-let1 c (find-1 f t) |
| 120 |
(and (direct-path? c) c))) |
| 121 |
(define (find-1hop f t pivot) |
| 122 |
(and-let* ([->pivot (find-direct f pivot)] |
| 123 |
[pivot-> (find-direct pivot t)]) |
| 124 |
(make <conversion> :from from :to to |
| 125 |
:procs (list (~ ->pivot'proc-name) (~ pivot->'proc-name))))) |
| 126 |
(define (find-2hop f t) |
| 127 |
(or (and-let1 ->eucj (find-direct f eucj) |
| 128 |
(if-let1 c (find-1 eucj t) |
| 129 |
(make <conversion> :from f :to t |
| 130 |
:procs (list (~ ->eucj'proc-name) (~ c'proc-name))) |
| 131 |
(and-let1 c2 (find-direct utf8 t) |
| 132 |
(let1 c1 (make <conversion> :from eucj :to t |
| 133 |
:procs (list 'eucj_utf8 (~ c2'proc-name))) |
| 134 |
(make <conversion> :from f :to t |
| 135 |
:procs (list (~ ->eucj'proc-name) (~ c1'proc-name))))))) |
| 136 |
(and-let1 eucj-> (find-direct eucj t) |
| 137 |
(if-let1 c (find-1 f eucj) |
| 138 |
(make <conversion> :from f :to t |
| 139 |
:procs (list (~ c'proc-name) (~ eucj->'proc-name))) |
| 140 |
(and-let1 c1 (find-direct f utf8) |
| 141 |
(let1 c2 (make <conversion> :from f :to eucj |
| 142 |
:procs (list (~ c1'proc-name) 'utf8_eucj)) |
| 143 |
(make <conversion> :from f :to t |
| 144 |
:procs (list (~ c2'proc-name) (~ eucj->'proc-name))))))))) |
| 145 |
(or (find-1 from to) |
| 146 |
(and (not (eq? from to)) |
| 147 |
(or (find-1hop from to utf8) |
| 148 |
(find-1hop from to eucj) |
| 149 |
(find-2hop from to))))) |
| 150 |
|
| 151 |
(define (emit-conversion-matrix) |
| 152 |
(define (fmt-entry oname conv reset istate ostate) |
| 153 |
($ format |
| 154 |
"~60a /* ~a */" |
| 155 |
(format " { ~a, ~a, ~a, ~a }," conv (or reset 'NULL) |
| 156 |
(or istate 0) (or ostate 0) ) |
| 157 |
oname)) |
| 158 |
(define (emit-fused-proc proc-name procs) |
| 159 |
(cgen-body #"/* ~proc-name : ~(car procs) + ~(cadr procs) */" |
| 160 |
#"static ScmSize ~proc-name(ScmConvInfo *cinfo," |
| 161 |
#" const char *inptr, ScmSize inroom," |
| 162 |
#" char *outptr, ScmSize outroom, ScmSize *outchars)" |
| 163 |
#"{") |
| 164 |
(match-let1 (f0 f1) procs |
| 165 |
(cgen-body #" char buf[INTERMEDIATE_BUF_SIZE];" |
| 166 |
#" ScmSize bufcount;" |
| 167 |
#" ScmSize r0 = ~f0(cinfo, inptr, inroom," |
| 168 |
#" buf, INTERMEDIATE_BUF_SIZE, &bufcount);" |
| 169 |
#" if (r0 < 0) return r0;" |
| 170 |
#" if (bufcount == 0) { *outchars = 0; return r0; }" |
| 171 |
#" ScmSize r1 = ~f1(cinfo, buf, bufcount," |
| 172 |
#" outptr, outroom, outchars);" |
| 173 |
#" if (r1 < 0) return r1;" |
| 174 |
#" return r0;")) |
| 175 |
(cgen-body "}" "")) |
| 176 |
|
| 177 |
;; Find available conversion routines and initialize coversion matrix. |
| 178 |
;; Some conversions are automatically generated in this process. |
| 179 |
(cgen-body "static struct conv_converter_rec" |
| 180 |
" conv_converter[NUM_JCODES][NUM_JCODES] = {") |
| 181 |
|
| 182 |
(dolist [f (encoding-schemes)] |
| 183 |
(cgen-body #" /* ~(~ f'name) => */" |
| 184 |
#" {") |
| 185 |
(dolist [t (encoding-schemes)] |
| 186 |
(if-let1 c (find-conversion f t) |
| 187 |
(cgen-body (fmt-entry (~ t'name) (~ c'proc-name) (~ t'reset) |
| 188 |
(~ f'initial-state) (~ t'initial-state))) |
| 189 |
(if (or (eq? f t) |
| 190 |
(eq? (~ f'name) 'none) |
| 191 |
(eq? (~ t'name) 'none)) |
| 192 |
(cgen-body (fmt-entry (~ t'name) "ident" #f 0 0)) |
| 193 |
(cgen-body (fmt-entry (~ t'name) 'NULL #f 0 0))))) |
| 194 |
(cgen-body #" },")) |
| 195 |
(cgen-body "};" "") |
| 196 |
|
| 197 |
;; Generate declartions in the header file, and fused conversion definitions |
| 198 |
(let1 protos '() |
| 199 |
(do-ec (: f (instance-pool->list <conversion>)) |
| 200 |
(if (not (member (~ f'proc-name) protos))) |
| 201 |
(begin |
| 202 |
(cgen-extern #"static ScmConvProc ~(~ f'proc-name);") |
| 203 |
(when (= (length (~ f'procs)) 2) |
| 204 |
(emit-fused-proc (~ f'proc-name) (~ f'procs))) |
| 205 |
(push! protos (~ f'proc-name))))) |
| 206 |
) |
| 207 |
|
| 208 |
(define (main args) |
| 209 |
(match (cdr args) |
| 210 |
[(outfile) |
| 211 |
(parameterize ([cgen-current-unit (make <cgen-unit> |
| 212 |
:name outfile |
| 213 |
:preamble "/* Generated by gen-tabs.scm */" |
| 214 |
:init-prologue "" |
| 215 |
:init-epilogue "")]) |
| 216 |
(emit-encoding-tables) |
| 217 |
(emit-conversion-matrix) |
| 218 |
(cgen-emit-c (cgen-current-unit)) |
| 219 |
(cgen-emit-h (cgen-current-unit)))]) |
| 220 |
0) |
| 221 |
|
| 222 |
;;; |
| 223 |
;;; Definitions |
| 224 |
;;; |
| 225 |
|
| 226 |
(define-encoding-scheme ascii JCODE_ASCII |
| 227 |
("ascii" "usascii" |
| 228 |
"isoir6" "iso646us" "us" "ibm367" "cp367" "csascii")) |
| 229 |
|
| 230 |
(define-encoding-scheme eucj JCODE_EUCJ |
| 231 |
("eucjp" "eucj" "eucjisx0213")) |
| 232 |
|
| 233 |
(define-encoding-scheme sjis JCODE_SJIS |
| 234 |
("sjis" "shiftjis")) |
| 235 |
|
| 236 |
(define-encoding-scheme utf8 JCODE_UTF8 |
| 237 |
("utf8") |
| 238 |
:states '(UTF_DEFAULT UTF_BE UTF_LE UTF_OPTIONAL_BOM) |
| 239 |
:initial-state 'UTF_DEFAULT) |
| 240 |
|
| 241 |
(define-encoding-scheme utf8bom JCODE_UTF8BOM |
| 242 |
("utf8bom") |
| 243 |
:states '(UTF_DEFAULT UTF_BE UTF_LE UTF_OPTIONAL_BOM) |
| 244 |
:initial-state 'UTF_OPTIONAL_BOM) |
| 245 |
|
| 246 |
(define-encoding-scheme utf16 JCODE_UTF16 |
| 247 |
("utf16") |
| 248 |
:states '(UTF_DEFAULT UTF_BE UTF_LE UTF_OPTIONAL_BOM) |
| 249 |
:initial-state 'UTF_DEFAULT) |
| 250 |
|
| 251 |
(define-encoding-scheme utf16be JCODE_UTF16BE |
| 252 |
("utf16be") |
| 253 |
:states '(UTF_DEFAULT UTF_BE UTF_LE UTF_OPTIONAL_BOM) |
| 254 |
:initial-state 'UTF_BE) |
| 255 |
|
| 256 |
(define-encoding-scheme utf16le JCODE_UTF16LE |
| 257 |
("utf16le") |
| 258 |
:states '(UTF_DEFAULT UTF_BE UTF_LE UTF_OPTIONAL_BOM) |
| 259 |
:initial-state 'UTF_LE) |
| 260 |
|
| 261 |
(define-encoding-scheme utf32 JCODE_UTF32 |
| 262 |
("utf32") |
| 263 |
:states '(UTF_DEFAULT UTF_BE UTF_LE UTF_OPTIONAL_BOM) |
| 264 |
:initial-state 'UTF_DEFAULT) |
| 265 |
|
| 266 |
(define-encoding-scheme utf32be JCODE_UTF32BE |
| 267 |
("utf32be") |
| 268 |
:states '(UTF_DEFAULT UTF_BE UTF_LE UTF_OPTIONAL_BOM) |
| 269 |
:initial-state 'UTF_BE) |
| 270 |
|
| 271 |
(define-encoding-scheme utf32le JCODE_UTF32LE |
| 272 |
("utf32le") |
| 273 |
:states '(UTF_DEFAULT UTF_BE UTF_LE UTF_OPTIONAL_BOM) |
| 274 |
:initial-state 'UTF_LE) |
| 275 |
|
| 276 |
(define-encoding-scheme iso2022jp JCODE_ISO2022JP |
| 277 |
("iso2022jp" "csiso2022jp" "iso2022jp1" "iso2022jp2" "iso2022jp3") |
| 278 |
:states '(JIS_ASCII JIS_ROMAN JIS_KANA JIS_78 JIS_0212 JIS_0213_1 JIS_0213_2 |
| 279 |
JIS_UNKNOWN) |
| 280 |
:initial-state 'JIS_ASCII |
| 281 |
:reset "jis_reset") |
| 282 |
|
| 283 |
(define-encoding-scheme iso8859-1 JCODE_ISO8859_1 ("iso88591" "latin1")) |
| 284 |
(define-encoding-scheme iso8859-2 JCODE_ISO8859_2 ("iso88592" "latin2")) |
| 285 |
(define-encoding-scheme iso8859-3 JCODE_ISO8859_3 ("iso88593" "latin3")) |
| 286 |
(define-encoding-scheme iso8859-4 JCODE_ISO8859_4 ("iso88594" "latin4")) |
| 287 |
(define-encoding-scheme iso8859-5 JCODE_ISO8859_5 ("iso88595" "latin5")) |
| 288 |
(define-encoding-scheme iso8859-6 JCODE_ISO8859_6 ("iso88596" "latin6")) |
| 289 |
(define-encoding-scheme iso8859-7 JCODE_ISO8859_7 ("iso88597" "latin7")) |
| 290 |
(define-encoding-scheme iso8859-8 JCODE_ISO8859_8 ("iso88598" "latin8")) |
| 291 |
(define-encoding-scheme iso8859-9 JCODE_ISO8859_9 ("iso88599" "latin9")) |
| 292 |
(define-encoding-scheme iso8859-10 JCODE_ISO8859_10 ("iso885910" "latin10")) |
| 293 |
(define-encoding-scheme iso8859-11 JCODE_ISO8859_11("iso885911" "latin11")) |
| 294 |
(define-encoding-scheme iso8859-13 JCODE_ISO8859_13 ("iso885913" "latin13")) |
| 295 |
(define-encoding-scheme iso8859-14 JCODE_ISO8859_14 ("iso885914" "latin14")) |
| 296 |
(define-encoding-scheme iso8859-15 JCODE_ISO8859_15 ("iso885915" "latin15")) |
| 297 |
(define-encoding-scheme iso8859-16 JCODE_ISO8859_16 ("iso885916" "latin16")) |
| 298 |
|
| 299 |
(define-encoding-scheme none JCODE_NONE ("none")) |
| 300 |
|
| 301 |
(define-conversion ascii_x (ascii) (eucj sjis utf8 iso2022jp iso8859-1)) |
| 302 |
(define-conversion utf8_utf16 (ascii) (utf16 utf16be utf16le)) |
| 303 |
(define-conversion utf8_utf32 (ascii) (utf32 utf32be utf32le)) |
| 304 |
|
| 305 |
(define-conversion eucj_ascii (eucj) (ascii)) |
| 306 |
(define-conversion eucj_sjis (eucj) (sjis)) |
| 307 |
(define-conversion eucj_utf8 (eucj) (utf8)) |
| 308 |
(define-conversion eucj_jis (eucj) (iso2022jp)) |
| 309 |
(define-conversion eucj_lat1 (eucj) (iso8859-1)) |
| 310 |
|
| 311 |
(define-conversion sjis_ascii (sjis) (ascii)) |
| 312 |
(define-conversion sjis_eucj (sjis) (eucj)) |
| 313 |
|
| 314 |
(define-conversion utf8_ascii (utf8) (ascii)) |
| 315 |
(define-conversion utf8_eucj (utf8) (eucj)) |
| 316 |
(define-conversion utf8_utf16 (utf8) (utf16 utf16be utf16le)) |
| 317 |
(define-conversion utf8_utf32 (utf8) (utf32 utf32be utf32le)) |
| 318 |
(define-conversion utf8_lat1 (utf8) (iso8859-1)) |
| 319 |
(define-conversion utf8_lat2 (utf8) (iso8859-2)) |
| 320 |
(define-conversion utf8_lat3 (utf8) (iso8859-3)) |
| 321 |
(define-conversion utf8_lat4 (utf8) (iso8859-4)) |
| 322 |
(define-conversion utf8_lat5 (utf8) (iso8859-5)) |
| 323 |
(define-conversion utf8_lat6 (utf8) (iso8859-6)) |
| 324 |
(define-conversion utf8_lat7 (utf8) (iso8859-7)) |
| 325 |
(define-conversion utf8_lat8 (utf8) (iso8859-8)) |
| 326 |
(define-conversion utf8_lat9 (utf8) (iso8859-9)) |
| 327 |
(define-conversion utf8_lat10 (utf8) (iso8859-10)) |
| 328 |
(define-conversion utf8_lat11 (utf8) (iso8859-11)) |
| 329 |
(define-conversion utf8_lat13 (utf8) (iso8859-13)) |
| 330 |
(define-conversion utf8_lat14 (utf8) (iso8859-14)) |
| 331 |
(define-conversion utf8_lat15 (utf8) (iso8859-15)) |
| 332 |
(define-conversion utf8_lat16 (utf8) (iso8859-16)) |
| 333 |
|
| 334 |
(define-conversion utf8bom_utf8 (utf8bom) (utf8)) |
| 335 |
|
| 336 |
(define-conversion utf16_utf8 (utf16 utf16be utf16le) (utf8)) |
| 337 |
(define-conversion utf16_utf16 (utf16 utf16be utf16le) (utf16 utf16be utf16le)) |
| 338 |
|
| 339 |
(define-conversion utf32_utf8 (utf32 utf32be utf32le) (utf8)) |
| 340 |
(define-conversion utf32_utf32 (utf32 utf32be utf32le) (utf32 utf32be utf32le)) |
| 341 |
|
| 342 |
(define-conversion jis_eucj (iso2022jp) (eucj)) |
| 343 |
|
| 344 |
(define-conversion lat1_ascii (iso8859-1) (ascii)) |
| 345 |
(define-conversion lat1_utf8 (iso8859-1) (utf8)) |
| 346 |
|
| 347 |
(define-conversion lat2_utf8 (iso8859-2) (utf8)) |
| 348 |
(define-conversion lat3_utf8 (iso8859-3) (utf8)) |
| 349 |
(define-conversion lat4_utf8 (iso8859-4) (utf8)) |
| 350 |
(define-conversion lat5_utf8 (iso8859-5) (utf8)) |
| 351 |
(define-conversion lat6_utf8 (iso8859-6) (utf8)) |
| 352 |
(define-conversion lat7_utf8 (iso8859-7) (utf8)) |
| 353 |
(define-conversion lat8_utf8 (iso8859-8) (utf8)) |
| 354 |
(define-conversion lat9_utf8 (iso8859-9) (utf8)) |
| 355 |
(define-conversion lat10_utf8 (iso8859-10) (utf8)) |
| 356 |
(define-conversion lat11_utf8 (iso8859-11) (utf8)) |
| 357 |
(define-conversion lat13_utf8 (iso8859-13) (utf8)) |
| 358 |
(define-conversion lat14_utf8 (iso8859-14) (utf8)) |
| 359 |
(define-conversion lat15_utf8 (iso8859-15) (utf8)) |
| 360 |
(define-conversion lat15_utf8 (iso8859-15) (utf8)) |
| 361 |
(define-conversion lat16_utf8 (iso8859-16) (utf8)) |