Debian Codegraph @

ext/charconv/gen-tabs.scm

raw
1 ;;;
2 ;;; Generating conversion state machine
3 ;;;
4
5 ;; For now, we just generate conversion matrix and some other tables.
6 ;; Eventually, we'll define DSL from which convertion C routines are generated.
7
8 (use gauche.cgen)
9 (use gauche.mop.instance-pool)
10 (use srfi.42)
11 (use util.match)
12
13 ;;
14 ;; Encoding Scheme
15 ;;
16
17 (define-class <encoding-scheme> (<instance-pool-mixin>)
18 ((name :init-keyword :name) ;symbol, short Scheme name
19 (c-enum :init-keyword :c-enum) ;symbol, C-enum name
20 (aliases :init-keyword :aliases) ;list of strings, alias names.
21 ; we ignore '_' and '-' for match.
22 (reset :init-keyword :reset ;string name of c reset procedure
23 :init-value #f) ; reset proedure is to reset the
24 ; output state.
25 (states :init-keyword :states ;If stateful encoding, list of states
26 :init-value '()) ; (suitable for C enums)
27 (initial-state :init-keyword :initial-state
28 :init-value #f)
29 ))
30 (define-method write-object ((obj <encoding-scheme>) port)
31 (format port "#<encoding ~a>" (~ obj'name)))
32
33 (define-syntax define-encoding-scheme
34 (syntax-rules ()
35 [(_ name c-enum aliases . opts)
36 (define name (make <encoding-scheme>
37 :name 'name
38 :c-enum 'c-enum
39 :aliases 'aliases
40 . opts))]))
41
42 (define (encoding-schemes) (instance-pool->list <encoding-scheme>))
43
44 (define (find-encoding-scheme name)
45 (instance-pool-find <encoding-scheme> (^e (eq? name (~ e'name)))))
46
47 (define (emit-encoding-tables)
48 (cgen-extern "/* Encodings */"
49 "enum {")
50 (do-ec (: e (encoding-schemes))
51 (cgen-extern #" ~(~ e'c-enum),"))
52 (cgen-extern " NUM_JCODES"
53 "};"
54 "")
55
56 (do-ec (: states (delete-duplicates (map (cut ~ <>'states)
57 (encoding-schemes))))
58 (if (not (null? states)))
59 (begin
60 (cgen-extern "enum {")
61 (do-ec (: s states) (cgen-extern #" ~|s|,"))
62 (cgen-extern "};" "")))
63
64 (cgen-body "static struct conv_support_rec conv_supports[] = {")
65 (do-ec (: e (encoding-schemes))
66 (: a (~ e'aliases))
67 (cgen-body (format " { ~15a, ~15a },"
68 (cgen-safe-string a)
69 (~ e 'c-enum))))
70 (cgen-body " { NULL, 0 }"
71 "};"))
72
73 ;;
74 ;; Conversion
75 ;;
76
77 ;; Conversion is defined with one or two procedure name(s).
78
79 (define-class <conversion> (<instance-pool-mixin>)
80 ((procs :init-keyword :procs) ;name(s) of conversion routines
81 (from :init-keyword :from) ;list of encoding names
82 (to :init-keyword :to) ;list of encoding names
83 (proc-name :init-value #f) ;final proc name (automatically set)
84 ))
85
86 (define-method initialize ((c <conversion>) initargs)
87 (define (ssplit sym) (string-split (x->string sym) #\_))
88 (next-method)
89 (match (~ c 'procs)
90 [(f0) (set! (~ c'proc-name) f0)]
91 ;; autogenerate fused function name
92 [(f0 f1)
93 (set! (~ c'proc-name) #"~(car (ssplit f0))_~(cadr (ssplit f1))")]
94 [(f0 f1 f2)
95 (set! (~ c'proc-name) #"~(car (ssplit f0))_~(cadr (ssplit f2))")]))
96
97 (define-syntax define-conversion
98 (syntax-rules ()
99 [(_ (procs ...) from to)
100 (make <conversion> :procs '(procs ...) :from 'from :to 'to)]
101 [(_ proc from to)
102 (make <conversion> :procs '(proc) :from 'from :to 'to)]))
103
104 ;; Find a conversion from FROM encoding to TO encoding (both <encoding-scheme>).
105 ;; If it's not defined, we search for a combination and fuse them.
106 ;; - If there's a direct path from->utf8 and utf8->to, use them.
107 ;; - If there's a direct path from->eucj and eucj->to, use them.
108 ;; - Otherwise, we search direct path hopping eucj and utf8.
109 ;; Here, 'direct path' means the converion is supported natively and not
110 ;; fused.
111 (define (find-conversion from to)
112 (define (find-1 f t)
113 ($ instance-pool-find <conversion>
114 (^c (and (memq (~ f'name) (~ c'from))
115 (memq (~ t'name) (~ c'to))))))
116 (define (direct-path? c)
117 (= (length (~ c'procs)) 1))
118 (define (find-direct f t)
119 (and-let1 c (find-1 f t)
120 (and (direct-path? c) c)))
121 (define (find-1hop f t pivot)
122 (and-let* ([->pivot (find-direct f pivot)]
123 [pivot-> (find-direct pivot t)])
124 (make <conversion> :from from :to to
125 :procs (list (~ ->pivot'proc-name) (~ pivot->'proc-name)))))
126 (define (find-2hop f t)
127 (or (and-let1 ->eucj (find-direct f eucj)
128 (if-let1 c (find-1 eucj t)
129 (make <conversion> :from f :to t
130 :procs (list (~ ->eucj'proc-name) (~ c'proc-name)))
131 (and-let1 c2 (find-direct utf8 t)
132 (let1 c1 (make <conversion> :from eucj :to t
133 :procs (list 'eucj_utf8 (~ c2'proc-name)))
134 (make <conversion> :from f :to t
135 :procs (list (~ ->eucj'proc-name) (~ c1'proc-name)))))))
136 (and-let1 eucj-> (find-direct eucj t)
137 (if-let1 c (find-1 f eucj)
138 (make <conversion> :from f :to t
139 :procs (list (~ c'proc-name) (~ eucj->'proc-name)))
140 (and-let1 c1 (find-direct f utf8)
141 (let1 c2 (make <conversion> :from f :to eucj
142 :procs (list (~ c1'proc-name) 'utf8_eucj))
143 (make <conversion> :from f :to t
144 :procs (list (~ c2'proc-name) (~ eucj->'proc-name)))))))))
145 (or (find-1 from to)
146 (and (not (eq? from to))
147 (or (find-1hop from to utf8)
148 (find-1hop from to eucj)
149 (find-2hop from to)))))
150
151 (define (emit-conversion-matrix)
152 (define (fmt-entry oname conv reset istate ostate)
153 ($ format
154 "~60a /* ~a */"
155 (format " { ~a, ~a, ~a, ~a }," conv (or reset 'NULL)
156 (or istate 0) (or ostate 0) )
157 oname))
158 (define (emit-fused-proc proc-name procs)
159 (cgen-body #"/* ~proc-name : ~(car procs) + ~(cadr procs) */"
160 #"static ScmSize ~proc-name(ScmConvInfo *cinfo,"
161 #" const char *inptr, ScmSize inroom,"
162 #" char *outptr, ScmSize outroom, ScmSize *outchars)"
163 #"{")
164 (match-let1 (f0 f1) procs
165 (cgen-body #" char buf[INTERMEDIATE_BUF_SIZE];"
166 #" ScmSize bufcount;"
167 #" ScmSize r0 = ~f0(cinfo, inptr, inroom,"
168 #" buf, INTERMEDIATE_BUF_SIZE, &bufcount);"
169 #" if (r0 < 0) return r0;"
170 #" if (bufcount == 0) { *outchars = 0; return r0; }"
171 #" ScmSize r1 = ~f1(cinfo, buf, bufcount,"
172 #" outptr, outroom, outchars);"
173 #" if (r1 < 0) return r1;"
174 #" return r0;"))
175 (cgen-body "}" ""))
176
177 ;; Find available conversion routines and initialize coversion matrix.
178 ;; Some conversions are automatically generated in this process.
179 (cgen-body "static struct conv_converter_rec"
180 " conv_converter[NUM_JCODES][NUM_JCODES] = {")
181
182 (dolist [f (encoding-schemes)]
183 (cgen-body #" /* ~(~ f'name) => */"
184 #" {")
185 (dolist [t (encoding-schemes)]
186 (if-let1 c (find-conversion f t)
187 (cgen-body (fmt-entry (~ t'name) (~ c'proc-name) (~ t'reset)
188 (~ f'initial-state) (~ t'initial-state)))
189 (if (or (eq? f t)
190 (eq? (~ f'name) 'none)
191 (eq? (~ t'name) 'none))
192 (cgen-body (fmt-entry (~ t'name) "ident" #f 0 0))
193 (cgen-body (fmt-entry (~ t'name) 'NULL #f 0 0)))))
194 (cgen-body #" },"))
195 (cgen-body "};" "")
196
197 ;; Generate declartions in the header file, and fused conversion definitions
198 (let1 protos '()
199 (do-ec (: f (instance-pool->list <conversion>))
200 (if (not (member (~ f'proc-name) protos)))
201 (begin
202 (cgen-extern #"static ScmConvProc ~(~ f'proc-name);")
203 (when (= (length (~ f'procs)) 2)
204 (emit-fused-proc (~ f'proc-name) (~ f'procs)))
205 (push! protos (~ f'proc-name)))))
206 )
207
208 (define (main args)
209 (match (cdr args)
210 [(outfile)
211 (parameterize ([cgen-current-unit (make <cgen-unit>
212 :name outfile
213 :preamble "/* Generated by gen-tabs.scm */"
214 :init-prologue ""
215 :init-epilogue "")])
216 (emit-encoding-tables)
217 (emit-conversion-matrix)
218 (cgen-emit-c (cgen-current-unit))
219 (cgen-emit-h (cgen-current-unit)))])
220 0)
221
222 ;;;
223 ;;; Definitions
224 ;;;
225
226 (define-encoding-scheme ascii JCODE_ASCII
227 ("ascii" "usascii"
228 "isoir6" "iso646us" "us" "ibm367" "cp367" "csascii"))
229
230 (define-encoding-scheme eucj JCODE_EUCJ
231 ("eucjp" "eucj" "eucjisx0213"))
232
233 (define-encoding-scheme sjis JCODE_SJIS
234 ("sjis" "shiftjis"))
235
236 (define-encoding-scheme utf8 JCODE_UTF8
237 ("utf8")
238 :states '(UTF_DEFAULT UTF_BE UTF_LE UTF_OPTIONAL_BOM)
239 :initial-state 'UTF_DEFAULT)
240
241 (define-encoding-scheme utf8bom JCODE_UTF8BOM
242 ("utf8bom")
243 :states '(UTF_DEFAULT UTF_BE UTF_LE UTF_OPTIONAL_BOM)
244 :initial-state 'UTF_OPTIONAL_BOM)
245
246 (define-encoding-scheme utf16 JCODE_UTF16
247 ("utf16")
248 :states '(UTF_DEFAULT UTF_BE UTF_LE UTF_OPTIONAL_BOM)
249 :initial-state 'UTF_DEFAULT)
250
251 (define-encoding-scheme utf16be JCODE_UTF16BE
252 ("utf16be")
253 :states '(UTF_DEFAULT UTF_BE UTF_LE UTF_OPTIONAL_BOM)
254 :initial-state 'UTF_BE)
255
256 (define-encoding-scheme utf16le JCODE_UTF16LE
257 ("utf16le")
258 :states '(UTF_DEFAULT UTF_BE UTF_LE UTF_OPTIONAL_BOM)
259 :initial-state 'UTF_LE)
260
261 (define-encoding-scheme utf32 JCODE_UTF32
262 ("utf32")
263 :states '(UTF_DEFAULT UTF_BE UTF_LE UTF_OPTIONAL_BOM)
264 :initial-state 'UTF_DEFAULT)
265
266 (define-encoding-scheme utf32be JCODE_UTF32BE
267 ("utf32be")
268 :states '(UTF_DEFAULT UTF_BE UTF_LE UTF_OPTIONAL_BOM)
269 :initial-state 'UTF_BE)
270
271 (define-encoding-scheme utf32le JCODE_UTF32LE
272 ("utf32le")
273 :states '(UTF_DEFAULT UTF_BE UTF_LE UTF_OPTIONAL_BOM)
274 :initial-state 'UTF_LE)
275
276 (define-encoding-scheme iso2022jp JCODE_ISO2022JP
277 ("iso2022jp" "csiso2022jp" "iso2022jp1" "iso2022jp2" "iso2022jp3")
278 :states '(JIS_ASCII JIS_ROMAN JIS_KANA JIS_78 JIS_0212 JIS_0213_1 JIS_0213_2
279 JIS_UNKNOWN)
280 :initial-state 'JIS_ASCII
281 :reset "jis_reset")
282
283 (define-encoding-scheme iso8859-1 JCODE_ISO8859_1 ("iso88591" "latin1"))
284 (define-encoding-scheme iso8859-2 JCODE_ISO8859_2 ("iso88592" "latin2"))
285 (define-encoding-scheme iso8859-3 JCODE_ISO8859_3 ("iso88593" "latin3"))
286 (define-encoding-scheme iso8859-4 JCODE_ISO8859_4 ("iso88594" "latin4"))
287 (define-encoding-scheme iso8859-5 JCODE_ISO8859_5 ("iso88595" "latin5"))
288 (define-encoding-scheme iso8859-6 JCODE_ISO8859_6 ("iso88596" "latin6"))
289 (define-encoding-scheme iso8859-7 JCODE_ISO8859_7 ("iso88597" "latin7"))
290 (define-encoding-scheme iso8859-8 JCODE_ISO8859_8 ("iso88598" "latin8"))
291 (define-encoding-scheme iso8859-9 JCODE_ISO8859_9 ("iso88599" "latin9"))
292 (define-encoding-scheme iso8859-10 JCODE_ISO8859_10 ("iso885910" "latin10"))
293 (define-encoding-scheme iso8859-11 JCODE_ISO8859_11("iso885911" "latin11"))
294 (define-encoding-scheme iso8859-13 JCODE_ISO8859_13 ("iso885913" "latin13"))
295 (define-encoding-scheme iso8859-14 JCODE_ISO8859_14 ("iso885914" "latin14"))
296 (define-encoding-scheme iso8859-15 JCODE_ISO8859_15 ("iso885915" "latin15"))
297 (define-encoding-scheme iso8859-16 JCODE_ISO8859_16 ("iso885916" "latin16"))
298
299 (define-encoding-scheme none JCODE_NONE ("none"))
300
301 (define-conversion ascii_x (ascii) (eucj sjis utf8 iso2022jp iso8859-1))
302 (define-conversion utf8_utf16 (ascii) (utf16 utf16be utf16le))
303 (define-conversion utf8_utf32 (ascii) (utf32 utf32be utf32le))
304
305 (define-conversion eucj_ascii (eucj) (ascii))
306 (define-conversion eucj_sjis (eucj) (sjis))
307 (define-conversion eucj_utf8 (eucj) (utf8))
308 (define-conversion eucj_jis (eucj) (iso2022jp))
309 (define-conversion eucj_lat1 (eucj) (iso8859-1))
310
311 (define-conversion sjis_ascii (sjis) (ascii))
312 (define-conversion sjis_eucj (sjis) (eucj))
313
314 (define-conversion utf8_ascii (utf8) (ascii))
315 (define-conversion utf8_eucj (utf8) (eucj))
316 (define-conversion utf8_utf16 (utf8) (utf16 utf16be utf16le))
317 (define-conversion utf8_utf32 (utf8) (utf32 utf32be utf32le))
318 (define-conversion utf8_lat1 (utf8) (iso8859-1))
319 (define-conversion utf8_lat2 (utf8) (iso8859-2))
320 (define-conversion utf8_lat3 (utf8) (iso8859-3))
321 (define-conversion utf8_lat4 (utf8) (iso8859-4))
322 (define-conversion utf8_lat5 (utf8) (iso8859-5))
323 (define-conversion utf8_lat6 (utf8) (iso8859-6))
324 (define-conversion utf8_lat7 (utf8) (iso8859-7))
325 (define-conversion utf8_lat8 (utf8) (iso8859-8))
326 (define-conversion utf8_lat9 (utf8) (iso8859-9))
327 (define-conversion utf8_lat10 (utf8) (iso8859-10))
328 (define-conversion utf8_lat11 (utf8) (iso8859-11))
329 (define-conversion utf8_lat13 (utf8) (iso8859-13))
330 (define-conversion utf8_lat14 (utf8) (iso8859-14))
331 (define-conversion utf8_lat15 (utf8) (iso8859-15))
332 (define-conversion utf8_lat16 (utf8) (iso8859-16))
333
334 (define-conversion utf8bom_utf8 (utf8bom) (utf8))
335
336 (define-conversion utf16_utf8 (utf16 utf16be utf16le) (utf8))
337 (define-conversion utf16_utf16 (utf16 utf16be utf16le) (utf16 utf16be utf16le))
338
339 (define-conversion utf32_utf8 (utf32 utf32be utf32le) (utf8))
340 (define-conversion utf32_utf32 (utf32 utf32be utf32le) (utf32 utf32be utf32le))
341
342 (define-conversion jis_eucj (iso2022jp) (eucj))
343
344 (define-conversion lat1_ascii (iso8859-1) (ascii))
345 (define-conversion lat1_utf8 (iso8859-1) (utf8))
346
347 (define-conversion lat2_utf8 (iso8859-2) (utf8))
348 (define-conversion lat3_utf8 (iso8859-3) (utf8))
349 (define-conversion lat4_utf8 (iso8859-4) (utf8))
350 (define-conversion lat5_utf8 (iso8859-5) (utf8))
351 (define-conversion lat6_utf8 (iso8859-6) (utf8))
352 (define-conversion lat7_utf8 (iso8859-7) (utf8))
353 (define-conversion lat8_utf8 (iso8859-8) (utf8))
354 (define-conversion lat9_utf8 (iso8859-9) (utf8))
355 (define-conversion lat10_utf8 (iso8859-10) (utf8))
356 (define-conversion lat11_utf8 (iso8859-11) (utf8))
357 (define-conversion lat13_utf8 (iso8859-13) (utf8))
358 (define-conversion lat14_utf8 (iso8859-14) (utf8))
359 (define-conversion lat15_utf8 (iso8859-15) (utf8))
360 (define-conversion lat15_utf8 (iso8859-15) (utf8))
361 (define-conversion lat16_utf8 (iso8859-16) (utf8))