Debian Codegraph @

ext/charconv/gen-guess.scm

raw
1 ;;;
2 ;;; Auxiliary script to generate japanese code guessing table
3 ;;;
4 ;;; Copyright (c) 2000-2024 Shiro Kawai <[email protected]>
5 ;;;
6 ;;; Redistribution and use in source and binary forms, with or without
7 ;;; modification, are permitted provided that the following conditions
8 ;;; are met:
9 ;;;
10 ;;; 1. Redistributions of source code must retain the above copyright
11 ;;; notice, this list of conditions and the following disclaimer.
12 ;;;
13 ;;; 2. Redistributions in binary form must reproduce the above copyright
14 ;;; notice, this list of conditions and the following disclaimer in the
15 ;;; documentation and/or other materials provided with the distribution.
16 ;;;
17 ;;; 3. Neither the name of the authors nor the names of its contributors
18 ;;; may be used to endorse or promote products derived from this
19 ;;; software without specific prior written permission.
20 ;;;
21 ;;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 ;;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 ;;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 ;;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 ;;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 ;;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
27 ;;; TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
28 ;;; PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
29 ;;; LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
30 ;;; NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
31 ;;; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 ;;;
33
34 (use scheme.list)
35 (use gauche.sequence)
36
37 ;; This is a simple state machine compiler.
38 ;;
39 ;; <state-machine> : (define-dfa <name> <state> ...)
40 ;; <state> : (<name> (<input-set> <next-state> <score>) ...)
41 ;; <name> : symbol
42 ;; <next-state> : symbol
43 ;; <score> : real
44 ;; <input-set> : (<byte-or-range> ...)
45 ;; <byte-or-range> : <byte> | (<byte> <byte>)
46 ;; <byte> : integer between 0 and #xff | ASCII char
47 ;;
48 ;; When evaluated, the DFA generates a state transition table in
49 ;; C source format.
50
51 (define-class <dfa> ()
52 ((name :init-keyword :name :accessor name-of)
53 (states :init-keyword :states :accessor states-of)
54 (instances :allocation :class :init-value '())))
55
56 (define-class <state> ()
57 ((name :init-keyword :name :accessor name-of)
58 (index :init-keyword :index :accessor index-of)
59 (arcs :init-keyword :arcs :accessor arcs-of :init-value '())))
60
61 (define-class <arc> ()
62 ((from-state :init-keyword :from-state :accessor from-state-of)
63 (to-state :init-keyword :to-state :accessor to-state-of)
64 (ranges :init-keyword :ranges :accessor ranges-of)
65 (index :init-keyword :index :accessor index-of)
66 (score :init-keyword :score :accessor score-of)))
67
68 ;; Create DFA
69
70 (define-syntax define-dfa
71 (syntax-rules ()
72 ((_ name . states)
73 (define name (make <dfa>
74 :name 'name
75 :states (resolve-states 'states))))))
76
77 (define-method initialize ((self <dfa>) initargs)
78 (next-method)
79 (slot-push! self 'instances self))
80
81 (define (all-dfas) (reverse (class-slot-ref <dfa> 'instances)))
82
83 (define (resolve-states state-defs)
84 (let ((states (map (lambda (d i) (make <state> :name (car d) :index i))
85 state-defs
86 (iota (length state-defs)))))
87 (fold (lambda (s d i)
88 (let1 num-arcs (length (cdr d))
89 (set! (arcs-of s)
90 (map (lambda (arc aindex)
91 (make <arc>
92 :from-state s
93 :to-state (or (find (lambda (e)
94 (eq? (name-of e) (cadr arc)))
95 states)
96 (error "no such state" (cadr arc)))
97 :ranges (car arc)
98 :index aindex
99 :score (caddr arc)))
100 (cdr d)
101 (iota num-arcs i)))
102 (+ i num-arcs)))
103 0
104 states state-defs)
105 states))
106
107 ;; Emit state table
108 (define (emit-dfa-table dfa)
109 (format #t "static signed char guess_~a_st[][256] = {\n" (name-of dfa))
110 (for-each emit-state-table (states-of dfa))
111 (print "};\n")
112 (format #t "static guess_arc guess_~a_ar[] = {\n" (name-of dfa))
113 (for-each emit-arc-table
114 (append-map arcs-of (states-of dfa)))
115 (print "};\n")
116 )
117
118 (define (emit-state-table state)
119 (define (b2i byte) ;byte->integer
120 (if (char? byte) (char->integer byte) byte))
121 (let1 arc-vec (make-vector 256 -1)
122 (dolist (br (arcs-of state))
123 (dolist (range (ranges-of br))
124 (if (pair? range)
125 (vector-fill! arc-vec (index-of br)
126 (b2i (car range)) (+ (b2i (cadr range)) 1))
127 (set! (ref arc-vec (b2i range)) (index-of br)))))
128 (format #t " { /* state ~a */" (name-of state))
129 (dotimes (i 256)
130 (when (zero? (modulo i 16)) (newline))
131 (format #t " ~2d," (ref arc-vec i)))
132 (print "\n },")
133 ))
134
135 (define (emit-arc-table arc)
136 (format #t " { ~2d, ~5s }, /* ~a -> ~a */\n"
137 (index-of (to-state-of arc))
138 (score-of arc)
139 (name-of (from-state-of arc))
140 (name-of (to-state-of arc))))
141 ;;
142 ;; main
143 ;;
144
145 (define (main args)
146 (unless (= (length args) 2)
147 (error "usage: ~a <outout-file.c>" (car args)))
148 (with-output-to-file (cadr args)
149 (lambda ()
150 (print "/* State transition table for character code guessing */")
151 (print "/* This file is automatically generated by guess.scm */")
152 (newline)
153 (for-each emit-dfa-table (all-dfas))))
154 0)
155
156 ;;;============================================================
157 ;;; DFA definitions
158 ;;;
159
160 ;;;
161 ;;; EUC-JP
162 ;;;
163
164 (define-dfa eucj
165 ;; first byte
166 (init
167 (((#x00 #x7f)) init 1.0) ; ASCII range
168 ((#x8e) jis0201_kana 0.8) ; JISX 0201 kana
169 ((#x8f) jis0213_2 0.95) ; JISX 0213 plane 2
170 (((#xa1 #xfe)) jis0213_1 1.0) ; JISX 0213 plane 1
171 )
172 ;; jis x 0201 kana
173 (jis0201_kana
174 (((#xa1 #xdf)) init 1.0)
175 )
176 ;; jis x 0208 and jis x 0213 plane 1
177 (jis0213_1
178 (((#xa1 #xfe)) init 1.0))
179 ;; jis x 0213 plane 2
180 (jis0213_2
181 (((#xa1 #xfe)) init 1.0))
182 )
183
184 ;;;
185 ;;; Shift_JIS
186 ;;;
187
188 (define-dfa sjis
189 ;; first byte
190 (init
191 (((#x00 #x7f)) init 1.0) ;ascii
192 (((#x81 #x9f) (#xe1 #xef)) jis0213 1.0) ;jisx0213 plane 1
193 (((#xa1 #xdf)) init 0.8) ;jisx0201 kana
194 (((#xf0 #xfc)) jis0213 0.95) ;jisx0213 plane 2
195 (((#xfd #xff)) init 0.8)) ;vendor extension
196 (jis0213
197 (((#x40 #x7e) (#x80 #xfc)) init 1.0))
198 )
199
200 ;;;
201 ;;; UTF-8
202 ;;;
203
204 (define-dfa utf8
205 (init
206 (((#x00 #x7f)) init 1.0)
207 (((#xc2 #xdf)) 1byte_more 1.0)
208 (((#xe0 #xef)) 2byte_more 1.0)
209 (((#xf0 #xf7)) 3byte_more 1.0)
210 (((#xf8 #xfb)) 4byte_more 1.0)
211 (((#xfc #xfd)) 5byte_more 1.0))
212 (1byte_more
213 (((#x80 #xbf)) init 1.0))
214 (2byte_more
215 (((#x80 #xbf)) 1byte_more 1.0))
216 (3byte_more
217 (((#x80 #xbf)) 2byte_more 1.0))
218 (4byte_more
219 (((#x80 #xbf)) 3byte_more 1.0))
220 (5byte_more
221 (((#x80 #xbf)) 4byte_more 1.0))
222 )
223
224 ;;;
225 ;;; JIS (ISO2022JP)
226 ;;;
227
228 ;; NB: for now, we just check the sequence of <ESC> $ or <ESC> '('.
229 '(define-dfa jis
230 (init
231 ((#x1b) esc 1.0)
232 (((#x00 #x1a) (#x1c #x1f)) init 1.0) ;C0
233 (((#x20 #x7f)) init 1.0) ;ASCII
234 (((#xa1 #xdf)) init 0.7) ;JIS8bit kana
235 )
236 (esc
237 ((#x0d #x0a) init 0.9) ;cancel
238 ((#\( ) esc-paren 1.0)
239 ((#\$ ) esc-$ 1.0)
240 ((#\& ) esc-& 1.0)
241 )
242 (esc-paren
243 ((#\B #\J #\H) init 1.0)
244 ((#\I) jis0201kana 0.8)
245 )
246 (esc-$
247 ((#\@ #\B) kanji 1.0)
248 ((#\( ) esc-$-paren 1.0)
249 )
250 (esc-$-paren
251 ((#\D #\O #\P) kanji 1.0))
252 (esc-&
253 ((#\@ ) init 1.0))
254 (jis0201kana
255 ((#x1b) esc 1.0)
256 (((#x20 #x5f)) jis0201kana 1.0))
257 (kanji
258 ((#x1b) esc 1.0)
259 (((#x21 #x7e)) kanji-2 1.0))
260 (kanji-2
261 (((#x21 #x7e)) kanji 1.0))
262 )