Debian Codegraph @

ext/charconv/convaux.scm

raw
1 ;;;
2 ;;; convaux - auxiliary charconv routines
3 ;;;
4 ;;; Copyright (c) 2000-2024 Shiro Kawai <[email protected]>
5 ;;;
6 ;;; Redistribution and use in source and binary forms, with or without
7 ;;; modification, are permitted provided that the following conditions
8 ;;; are met:
9 ;;;
10 ;;; 1. Redistributions of source code must retain the above copyright
11 ;;; notice, this list of conditions and the following disclaimer.
12 ;;;
13 ;;; 2. Redistributions in binary form must reproduce the above copyright
14 ;;; notice, this list of conditions and the following disclaimer in the
15 ;;; documentation and/or other materials provided with the distribution.
16 ;;;
17 ;;; 3. Neither the name of the authors nor the names of its contributors
18 ;;; may be used to endorse or promote products derived from this
19 ;;; software without specific prior written permission.
20 ;;;
21 ;;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 ;;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 ;;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 ;;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 ;;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 ;;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
27 ;;; TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
28 ;;; PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
29 ;;; LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
30 ;;; NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
31 ;;; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 ;;;
33
34 (select-module gauche.charconv)
35 (use scheme.list)
36 (use srfi.13)
37 (use gauche.sequence)
38 (autoload gauche.vport
39 open-input-uvector open-output-uvector get-output-uvector)
40
41 ;; Determine charset compatibility. (ces-equivalent? a b) is true if CES a and
42 ;; CES b refer to the same CES.
43 ;; (ces-upper-compatible? a b) is true if a string in CES b can be a legal
44 ;; string in CES a.
45 ;; There is a case that the compatibility can't be determined. In such
46 ;; case, a value passed as unknown-fallback is returned, whose default is #f.
47
48 (define-values (ces-equivalent? ces-upper-compatible?)
49 (let ()
50 ;; Compatibility table:
51 ;; ((x ...) y ...) means x ... are all equivalent, and includes y ....
52 ;; Each entry is a symbol. The CES name is first stripped off all _ and -
53 ;; then made a lower case symbol before comparison.
54 ;; NB: CES 'none' works as a wildcard.
55 (define ces-compatibility-table
56 '(((usascii ascii iso646))
57 ((iso88591 88591) usascii)
58 ((iso88592 88592) usascii)
59 ((iso88593 88593) usascii)
60 ((iso88594 88594) usascii)
61 ((iso88595 88595) usascii)
62 ((iso88596 88596) usascii)
63 ((iso88597 88597) usascii)
64 ((iso88598 88598) usascii)
65 ((iso88599 88599) usascii)
66 ((iso2022jp csiso2022jp) usascii) ; not exactly
67 ((iso2022jp1 csiso2022jp1) iso2022jp)
68 ((iso2022jp2 csiso2022jp2) iso2022jp)
69 ((iso2022jp3 csiso2022jp3) iso2022jp)
70 ((iso2022kr csiso2022kr) usascii)
71 ((iso2022cnext csiso2022cnext) iso2022cn)
72 ((iso2022cn csiso2022cn) usascii)
73 ((eucjp) usascii)
74 ((euckr) usascii)
75 ((euccn) usascii)
76 ((euctw) usascii)
77 ((big5 bigfive) usascii)
78 ((big5hkscs bigfivehkscs) big5)
79 ((big5plus bigfiveplus) big5)
80 ((johab) usascii)
81 ((shiftjis sjis) usascii) ; not exactly
82 ((unicode iso10646 iso10646/utf8 utf8) usascii)
83 ))
84
85 ;; canonicalize ces name
86 (define (canon-name name)
87 (string->symbol (string-downcase (string-delete (x->string name) #[_-]))))
88
89 ;; find ces entry
90 (define (find-entry ces)
91 (find (^e (memq ces (car e))) ces-compatibility-table))
92
93 (define (ces-equivalent? a b :optional (unknown-fallback #f))
94 (let* ([ces-a (canon-name a)]
95 [ces-b (canon-name b)]
96 [entry-a (find-entry ces-a)]
97 [entry-b (find-entry ces-b)])
98 (cond [(or (eq? ces-a 'none) (eq? ces-b 'none)) #t]
99 [(or (not entry-a) (not entry-b))
100 unknown-fallback]
101 [else (eq? entry-a entry-b)])))
102
103 (define (ces-upper-compatible? a b :optional (unknown-fallback #f))
104 (let* ([ces-a (canon-name a)]
105 [ces-b (canon-name b)]
106 [entry-a (find-entry ces-a)])
107 (cond [(or (eq? ces-a 'none) (eq? ces-b 'none)) #t]
108 [(or (not entry-a) (not (find-entry ces-b)))
109 unknown-fallback]
110 [else (let loop ([entry entry-a])
111 (or (boolean (memq ces-b (car entry)))
112 (any loop (map find-entry (cdr entry)))))])))
113
114 (values ces-equivalent? ces-upper-compatible?)))
115
116 ;; Returns a list of octets for replacement characters.
117 ;; This is called from jconv_set_replacement() internal C function.
118 (define (%ces-replacement ces)
119 (assume-type ces <string>)
120 (cond
121 ;; For unicode, we use U+FFFD Replacement Character.
122 [(ces-equivalent? "utf8" ces) '(#xef #xbf #xbd)]
123 ;; For jis-family, we use U+3013 Geta Mark (JIS 020E)
124 ;; NB: iso-2022-jp is generated via eucjp, so it is suffice to give
125 ;; the replacement char in eucjp.
126 [(ces-equivalent? "eucjp" ces) '(#xa2 #xae)]
127 [(ces-equivalent? "sjis" ces) '(#x81 #xac)]
128 [(ces-equivalent? "iso2022jp" ces) '(#xa2 #xae)]
129 ;; 1-byte encoding, we use '?'
130 [(ces-equivalent? "ascii" ces) '(#x3f)]
131 [(ces-equivalent? "iso88591" ces) '(#x3f)]
132 [(ces-equivalent? "iso88592" ces) '(#x3f)]
133 [(ces-equivalent? "iso88593" ces) '(#x3f)]
134 [(ces-equivalent? "iso88594" ces) '(#x3f)]
135 [(ces-equivalent? "iso88595" ces) '(#x3f)]
136 [(ces-equivalent? "iso88596" ces) '(#x3f)]
137 [(ces-equivalent? "iso88597" ces) '(#x3f)]
138 [(ces-equivalent? "iso88598" ces) '(#x3f)]
139 [(ces-equivalent? "iso88599" ces) '(#x3f)]
140 ;; we should detect some more unicode family to pass U+FFFD, but
141 ;; for now, we return '?'.
142 [else '(#x3f)]))
143
144 ;; returns appropriate input port and size
145 (define (%ces-input input)
146 (assume-type input (</> <string> <u8vector>))
147 (cond [(string? input)
148 (values (open-input-string input :private? #t) (string-length input))]
149 ;; avoid using u8vector? so that we don't depend on gauche.uvector
150 [(is-a? input <u8vector>)
151 (values (open-input-uvector input) (uvector-length input))]))
152
153 ;; returns appropriate output port and get-output-*
154 (define (%ces-output class)
155 (cond
156 [(eq? class <string>)
157 (values (open-output-string :private? #t) get-output-string)]
158 [(eq? class <u8vector>)
159 (values (open-output-uvector) get-output-uvector)]
160 [else (error "Only <string> or <u8vector> is supported, but got:" class)]))
161
162 ;; Convert string or uvector -> string or uvector
163 (define (ces-convert-to class input fromcode
164 :optional (tocode #f)
165 :key illegal-output)
166 (receive (inp isize) (%ces-input input)
167 (receive (out get) (%ces-output class)
168 (let1 in ($ open-input-conversion-port inp fromcode
169 :to-code tocode :buffer-size isize
170 :owner? #t :illegal-output illegal-output)
171 (copy-port in out :unit 'byte)
172 (close-input-port in)
173 (flush out)
174 (begin0 (get out)
175 (close-output-port out))))))
176
177 (define (ces-convert input fromcode
178 :optional (tocode #f)
179 :key illegal-output)
180 (ces-convert-to <string> input fromcode tocode :illegal-output illegal-output))
181
182 ;; "Wrap" the given port for convering to/from native encoding if needed.
183 ;; Unlike open-*-conversion-port, these return port itself if the conversion
184 ;; is not required.
185 (define (wrap-with-input-conversion port from-code
186 :key (to-code (gauche-character-encoding))
187 :allow-other-keys
188 :rest opts)
189 (if (ces-upper-compatible? to-code from-code)
190 port
191 (apply open-input-conversion-port port from-code :owner? #t opts)))
192
193 (define (wrap-with-output-conversion port to-code
194 :key (from-code (gauche-character-encoding))
195 :allow-other-keys
196 :rest opts)
197 (if (ces-upper-compatible? from-code to-code)
198 port
199 (apply open-output-conversion-port port to-code :owner? #t opts)))
200
201 ;; Call with conversion port
202 (define (call-with-input-conversion port proc
203 :key ((:encoding from-code)
204 (gauche-character-encoding))
205 ((:conversion-buffer-size bufsiz) 0)
206 illegal-output)
207 (if (ces-upper-compatible? (gauche-character-encoding) from-code)
208 (proc port)
209 (let1 cvp (open-input-conversion-port port from-code
210 :owner? #f :buffer-size bufsiz
211 :illegal-output illegal-output)
212 (unwind-protect (proc cvp)
213 (close-input-port cvp)))))
214
215 (define (call-with-output-conversion port proc
216 :key ((:encoding to-code)
217 (gauche-character-encoding))
218 ((:conversion-buffer-size bufsiz) 0)
219 illegal-output)
220 (if (ces-upper-compatible? (gauche-character-encoding) to-code)
221 (proc port)
222 (let1 cvp (open-output-conversion-port port to-code
223 :owner? #f :buffer-size bufsiz
224 :illegal-output illegal-output)
225 (unwind-protect (proc cvp)
226 (close-output-port cvp)))))
227
228 (define (with-input-conversion port thunk . opts)
229 (apply call-with-input-conversion port
230 (cut with-input-from-port <> thunk)
231 opts))
232
233 (define (with-output-conversion port thunk . opts)
234 (apply call-with-output-conversion port
235 (cut with-output-to-port <> thunk)
236 opts))
237
238 ;; Inserts conversion port. These are called from system's
239 ;; open-{input|output}-file when :encoding argument is given.
240 (define (%open-input-file/conv name :key (encoding #f)
241 ((:conversion-buffer-size bufsiz) 0)
242 ((:conversion-illegal-output illegal-output) 'raise)
243 :allow-other-keys rest)
244 (if (and (not encoding)
245 (not (eq? (default-file-encoding) (gauche-character-encoding))))
246 (apply %open-input-file/conv name
247 :encoding (default-file-encoding)
248 :conversion-buffer-size bufsiz
249 :conversion-illegal-output illegal-output
250 rest)
251 (and-let* ([port (apply (with-module gauche.internal %open-input-file)
252 name
253 (delete-keyword :encoding rest))])
254 (wrap-with-input-conversion port encoding
255 :buffer-size bufsiz
256 :owner? #t
257 :illegal-output illegal-output))))
258
259 (define (%open-output-file/conv name :key (encoding (default-file-encoding))
260 ((:conversion-buffer-size bufsiz) 0)
261 ((:conversion-illegal-output illegal-output) 'raise)
262 :allow-other-keys rest)
263 (and-let* ([port (apply (with-module gauche.internal %open-output-file)
264 name rest)])
265 (wrap-with-output-conversion port encoding
266 :buffer-size bufsiz :owner? #t
267 :illegal-output illegal-output)))
268
269 ;;
270 ;; Low-level API
271 ;;
272
273 (inline-stub
274 (declcode
275 (.include <gauche/priv/configP.h>
276 "charconv.h"))
277
278 (define-cproc ces-conversion-supported? (from to)
279 ::<boolean>
280 (let* ([cfrom::(const char*) (Scm_GetCESName from "from-code")]
281 [cto ::(const char*) (Scm_GetCESName to "to-code")]
282 [flags::u_long 0])
283 (return (Scm_ConversionSupportedP cfrom cto flags))))
284
285 (define-cproc open-input-conversion-port (source::<input-port>
286 from-code
287 :key (to-code #f)
288 (buffer-size::<fixnum> 0)
289 (owner? #f)
290 illegal-output)
291 (let* ([fc::(const char*) (Scm_GetCESName from_code "from-code")]
292 [tc::(const char*) (Scm_GetCESName to_code "to-code")]
293 [flags::u_long 0])
294 (unless (SCM_FALSEP ownerP)
295 (logior= flags CVPORT_OWNER))
296 (cond [(SCM_EQ illegal-output 'replace) (logior= flags CVPORT_REPLACE)]
297 [(or (SCM_UNBOUNDP illegal-output)
298 (SCM_UNDEFINEDP illegal-output)
299 (SCM_EQ illegal-output 'raise))]
300 [else (Scm_Error ":illegal-output argument must be either raise or \
301 replace, but got: %S" illegal-output)])
302 (return (Scm_MakeInputConversionPort source fc tc buffer_size
303 flags))))
304
305 (define-cproc open-output-conversion-port (sink::<output-port>
306 to-code
307 :key (from-code #f)
308 (buffer-size::<fixnum> 0)
309 (owner? #f)
310 illegal-output)
311 (let* ([fc::(const char*) (Scm_GetCESName from_code "from-code")]
312 [tc::(const char*) (Scm_GetCESName to_code "to-code")]
313 [flags::u_long 0])
314 (unless (SCM_FALSEP ownerP)
315 (logior= flags CVPORT_OWNER))
316 (cond [(SCM_EQ illegal-output 'replace) (logior= flags CVPORT_REPLACE)]
317 [(or (SCM_UNBOUNDP illegal-output)
318 (SCM_UNDEFINEDP illegal-output)
319 (SCM_EQ illegal-output 'raise))]
320 [else (Scm_Error ":illegal-output argument must be either raise or \
321 replace, but got: %S" illegal-output)])
322 (return (Scm_MakeOutputConversionPort sink tc fc buffer_size flags))))
323
324 (define-cproc ces-guess-from-string (string::<string> scheme::<string>)
325 (let* ([size::ScmSmallInt]
326 [s::(const char*) (Scm_GetStringContent string (& size) NULL NULL)]
327 [guessed::(const char*)
328 (Scm_GuessCES (Scm_GetStringConst scheme) s size)])
329 (if guessed
330 (return (SCM_MAKE_STR guessed))
331 (return '#f))))
332 )