| 1 |
;;; |
| 2 |
;;; Auxiliary script to generate japanese code guessing table |
| 3 |
;;; |
| 4 |
;;; Copyright (c) 2000-2024 Shiro Kawai <[email protected]> |
| 5 |
;;; |
| 6 |
;;; Redistribution and use in source and binary forms, with or without |
| 7 |
;;; modification, are permitted provided that the following conditions |
| 8 |
;;; are met: |
| 9 |
;;; |
| 10 |
;;; 1. Redistributions of source code must retain the above copyright |
| 11 |
;;; notice, this list of conditions and the following disclaimer. |
| 12 |
;;; |
| 13 |
;;; 2. Redistributions in binary form must reproduce the above copyright |
| 14 |
;;; notice, this list of conditions and the following disclaimer in the |
| 15 |
;;; documentation and/or other materials provided with the distribution. |
| 16 |
;;; |
| 17 |
;;; 3. Neither the name of the authors nor the names of its contributors |
| 18 |
;;; may be used to endorse or promote products derived from this |
| 19 |
;;; software without specific prior written permission. |
| 20 |
;;; |
| 21 |
;;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 22 |
;;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 23 |
;;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 24 |
;;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 25 |
;;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 26 |
;;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED |
| 27 |
;;; TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR |
| 28 |
;;; PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF |
| 29 |
;;; LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING |
| 30 |
;;; NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| 31 |
;;; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 32 |
;;; |
| 33 |
|
| 34 |
(use scheme.list) |
| 35 |
(use gauche.sequence) |
| 36 |
|
| 37 |
;; This is a simple state machine compiler. |
| 38 |
;; |
| 39 |
;; <state-machine> : (define-dfa <name> <state> ...) |
| 40 |
;; <state> : (<name> (<input-set> <next-state> <score>) ...) |
| 41 |
;; <name> : symbol |
| 42 |
;; <next-state> : symbol |
| 43 |
;; <score> : real |
| 44 |
;; <input-set> : (<byte-or-range> ...) |
| 45 |
;; <byte-or-range> : <byte> | (<byte> <byte>) |
| 46 |
;; <byte> : integer between 0 and #xff | ASCII char |
| 47 |
;; |
| 48 |
;; When evaluated, the DFA generates a state transition table in |
| 49 |
;; C source format. |
| 50 |
|
| 51 |
(define-class <dfa> () |
| 52 |
((name :init-keyword :name :accessor name-of) |
| 53 |
(states :init-keyword :states :accessor states-of) |
| 54 |
(instances :allocation :class :init-value '()))) |
| 55 |
|
| 56 |
(define-class <state> () |
| 57 |
((name :init-keyword :name :accessor name-of) |
| 58 |
(index :init-keyword :index :accessor index-of) |
| 59 |
(arcs :init-keyword :arcs :accessor arcs-of :init-value '()))) |
| 60 |
|
| 61 |
(define-class <arc> () |
| 62 |
((from-state :init-keyword :from-state :accessor from-state-of) |
| 63 |
(to-state :init-keyword :to-state :accessor to-state-of) |
| 64 |
(ranges :init-keyword :ranges :accessor ranges-of) |
| 65 |
(index :init-keyword :index :accessor index-of) |
| 66 |
(score :init-keyword :score :accessor score-of))) |
| 67 |
|
| 68 |
;; Create DFA |
| 69 |
|
| 70 |
(define-syntax define-dfa |
| 71 |
(syntax-rules () |
| 72 |
((_ name . states) |
| 73 |
(define name (make <dfa> |
| 74 |
:name 'name |
| 75 |
:states (resolve-states 'states)))))) |
| 76 |
|
| 77 |
(define-method initialize ((self <dfa>) initargs) |
| 78 |
(next-method) |
| 79 |
(slot-push! self 'instances self)) |
| 80 |
|
| 81 |
(define (all-dfas) (reverse (class-slot-ref <dfa> 'instances))) |
| 82 |
|
| 83 |
(define (resolve-states state-defs) |
| 84 |
(let ((states (map (lambda (d i) (make <state> :name (car d) :index i)) |
| 85 |
state-defs |
| 86 |
(iota (length state-defs))))) |
| 87 |
(fold (lambda (s d i) |
| 88 |
(let1 num-arcs (length (cdr d)) |
| 89 |
(set! (arcs-of s) |
| 90 |
(map (lambda (arc aindex) |
| 91 |
(make <arc> |
| 92 |
:from-state s |
| 93 |
:to-state (or (find (lambda (e) |
| 94 |
(eq? (name-of e) (cadr arc))) |
| 95 |
states) |
| 96 |
(error "no such state" (cadr arc))) |
| 97 |
:ranges (car arc) |
| 98 |
:index aindex |
| 99 |
:score (caddr arc))) |
| 100 |
(cdr d) |
| 101 |
(iota num-arcs i))) |
| 102 |
(+ i num-arcs))) |
| 103 |
0 |
| 104 |
states state-defs) |
| 105 |
states)) |
| 106 |
|
| 107 |
;; Emit state table |
| 108 |
(define (emit-dfa-table dfa) |
| 109 |
(format #t "static signed char guess_~a_st[][256] = {\n" (name-of dfa)) |
| 110 |
(for-each emit-state-table (states-of dfa)) |
| 111 |
(print "};\n") |
| 112 |
(format #t "static guess_arc guess_~a_ar[] = {\n" (name-of dfa)) |
| 113 |
(for-each emit-arc-table |
| 114 |
(append-map arcs-of (states-of dfa))) |
| 115 |
(print "};\n") |
| 116 |
) |
| 117 |
|
| 118 |
(define (emit-state-table state) |
| 119 |
(define (b2i byte) ;byte->integer |
| 120 |
(if (char? byte) (char->integer byte) byte)) |
| 121 |
(let1 arc-vec (make-vector 256 -1) |
| 122 |
(dolist (br (arcs-of state)) |
| 123 |
(dolist (range (ranges-of br)) |
| 124 |
(if (pair? range) |
| 125 |
(vector-fill! arc-vec (index-of br) |
| 126 |
(b2i (car range)) (+ (b2i (cadr range)) 1)) |
| 127 |
(set! (ref arc-vec (b2i range)) (index-of br))))) |
| 128 |
(format #t " { /* state ~a */" (name-of state)) |
| 129 |
(dotimes (i 256) |
| 130 |
(when (zero? (modulo i 16)) (newline)) |
| 131 |
(format #t " ~2d," (ref arc-vec i))) |
| 132 |
(print "\n },") |
| 133 |
)) |
| 134 |
|
| 135 |
(define (emit-arc-table arc) |
| 136 |
(format #t " { ~2d, ~5s }, /* ~a -> ~a */\n" |
| 137 |
(index-of (to-state-of arc)) |
| 138 |
(score-of arc) |
| 139 |
(name-of (from-state-of arc)) |
| 140 |
(name-of (to-state-of arc)))) |
| 141 |
;; |
| 142 |
;; main |
| 143 |
;; |
| 144 |
|
| 145 |
(define (main args) |
| 146 |
(unless (= (length args) 2) |
| 147 |
(error "usage: ~a <outout-file.c>" (car args))) |
| 148 |
(with-output-to-file (cadr args) |
| 149 |
(lambda () |
| 150 |
(print "/* State transition table for character code guessing */") |
| 151 |
(print "/* This file is automatically generated by guess.scm */") |
| 152 |
(newline) |
| 153 |
(for-each emit-dfa-table (all-dfas)))) |
| 154 |
0) |
| 155 |
|
| 156 |
;;;============================================================ |
| 157 |
;;; DFA definitions |
| 158 |
;;; |
| 159 |
|
| 160 |
;;; |
| 161 |
;;; EUC-JP |
| 162 |
;;; |
| 163 |
|
| 164 |
(define-dfa eucj |
| 165 |
;; first byte |
| 166 |
(init |
| 167 |
(((#x00 #x7f)) init 1.0) ; ASCII range |
| 168 |
((#x8e) jis0201_kana 0.8) ; JISX 0201 kana |
| 169 |
((#x8f) jis0213_2 0.95) ; JISX 0213 plane 2 |
| 170 |
(((#xa1 #xfe)) jis0213_1 1.0) ; JISX 0213 plane 1 |
| 171 |
) |
| 172 |
;; jis x 0201 kana |
| 173 |
(jis0201_kana |
| 174 |
(((#xa1 #xdf)) init 1.0) |
| 175 |
) |
| 176 |
;; jis x 0208 and jis x 0213 plane 1 |
| 177 |
(jis0213_1 |
| 178 |
(((#xa1 #xfe)) init 1.0)) |
| 179 |
;; jis x 0213 plane 2 |
| 180 |
(jis0213_2 |
| 181 |
(((#xa1 #xfe)) init 1.0)) |
| 182 |
) |
| 183 |
|
| 184 |
;;; |
| 185 |
;;; Shift_JIS |
| 186 |
;;; |
| 187 |
|
| 188 |
(define-dfa sjis |
| 189 |
;; first byte |
| 190 |
(init |
| 191 |
(((#x00 #x7f)) init 1.0) ;ascii |
| 192 |
(((#x81 #x9f) (#xe1 #xef)) jis0213 1.0) ;jisx0213 plane 1 |
| 193 |
(((#xa1 #xdf)) init 0.8) ;jisx0201 kana |
| 194 |
(((#xf0 #xfc)) jis0213 0.95) ;jisx0213 plane 2 |
| 195 |
(((#xfd #xff)) init 0.8)) ;vendor extension |
| 196 |
(jis0213 |
| 197 |
(((#x40 #x7e) (#x80 #xfc)) init 1.0)) |
| 198 |
) |
| 199 |
|
| 200 |
;;; |
| 201 |
;;; UTF-8 |
| 202 |
;;; |
| 203 |
|
| 204 |
(define-dfa utf8 |
| 205 |
(init |
| 206 |
(((#x00 #x7f)) init 1.0) |
| 207 |
(((#xc2 #xdf)) 1byte_more 1.0) |
| 208 |
(((#xe0 #xef)) 2byte_more 1.0) |
| 209 |
(((#xf0 #xf7)) 3byte_more 1.0) |
| 210 |
(((#xf8 #xfb)) 4byte_more 1.0) |
| 211 |
(((#xfc #xfd)) 5byte_more 1.0)) |
| 212 |
(1byte_more |
| 213 |
(((#x80 #xbf)) init 1.0)) |
| 214 |
(2byte_more |
| 215 |
(((#x80 #xbf)) 1byte_more 1.0)) |
| 216 |
(3byte_more |
| 217 |
(((#x80 #xbf)) 2byte_more 1.0)) |
| 218 |
(4byte_more |
| 219 |
(((#x80 #xbf)) 3byte_more 1.0)) |
| 220 |
(5byte_more |
| 221 |
(((#x80 #xbf)) 4byte_more 1.0)) |
| 222 |
) |
| 223 |
|
| 224 |
;;; |
| 225 |
;;; JIS (ISO2022JP) |
| 226 |
;;; |
| 227 |
|
| 228 |
;; NB: for now, we just check the sequence of <ESC> $ or <ESC> '('. |
| 229 |
'(define-dfa jis |
| 230 |
(init |
| 231 |
((#x1b) esc 1.0) |
| 232 |
(((#x00 #x1a) (#x1c #x1f)) init 1.0) ;C0 |
| 233 |
(((#x20 #x7f)) init 1.0) ;ASCII |
| 234 |
(((#xa1 #xdf)) init 0.7) ;JIS8bit kana |
| 235 |
) |
| 236 |
(esc |
| 237 |
((#x0d #x0a) init 0.9) ;cancel |
| 238 |
((#\( ) esc-paren 1.0) |
| 239 |
((#\$ ) esc-$ 1.0) |
| 240 |
((#\& ) esc-& 1.0) |
| 241 |
) |
| 242 |
(esc-paren |
| 243 |
((#\B #\J #\H) init 1.0) |
| 244 |
((#\I) jis0201kana 0.8) |
| 245 |
) |
| 246 |
(esc-$ |
| 247 |
((#\@ #\B) kanji 1.0) |
| 248 |
((#\( ) esc-$-paren 1.0) |
| 249 |
) |
| 250 |
(esc-$-paren |
| 251 |
((#\D #\O #\P) kanji 1.0)) |
| 252 |
(esc-& |
| 253 |
((#\@ ) init 1.0)) |
| 254 |
(jis0201kana |
| 255 |
((#x1b) esc 1.0) |
| 256 |
(((#x20 #x5f)) jis0201kana 1.0)) |
| 257 |
(kanji |
| 258 |
((#x1b) esc 1.0) |
| 259 |
(((#x21 #x7e)) kanji-2 1.0)) |
| 260 |
(kanji-2 |
| 261 |
(((#x21 #x7e)) kanji 1.0)) |
| 262 |
) |