Debian Codegraph @

ext/charconv/data/gen-testdata.scm

raw
1 ;;;
2 ;;; Generate some of the data files
3 ;;;
4
5 (use file.util)
6 (use gauche.charconv)
7 (use gauche.uvector)
8 (use srfi.42)
9
10 ;;;
11 ;;; Latin-1 test dataset
12 ;;;
13
14 (define (do-lat1 suffix emitter exclude-b5?)
15 (with-output-to-file #"~(if exclude-b5? 'lat1x 'lat1).~suffix"
16 (^[] (dotimes [n 256]
17 (cond [(<= n #x7f) (write-byte n)]
18 [(and exclude-b5? (= n #xb5))]
19 [(<= #xa0 n) (emitter n)])
20 (when (zero? (modulo (+ n 1) 16)) (newline))))))
21
22 (define (do-lat1* suffix emitter)
23 (do-lat1 suffix emitter #f)
24 (do-lat1 suffix emitter #t))
25
26 (define (lat1.iso8859-1)
27 (do-lat1* "ISO8859-1" write-byte))
28
29 (define (lat1.ascii)
30 (do-lat1 "ASCII" (^b (write-char #\?)) #f))
31
32 (define (lat1.utf-8)
33 (do-lat1* "UTF-8"
34 (^b (write-byte (+ #xc0 (ash b -6)))
35 (write-byte (+ #x80 (logand b #x3f))))))
36
37 ;; NB: U+00B5 (MICRO SIGN) doesn't have a corresponding JIS character.
38 ;; We could've mapped it ot greek mu, but then we'll lose round-trip identity.
39 (define *lat1-eucjp-data* "
40 0xA9A2 U+00A0 # NO-BREAK SPACE [2000]
41 0xA9A3 U+00A1 # INVERTED EXCLAMATION MARK [2000]
42 0xA1F1 U+00A2 # CENT SIGN Windows: U+FFE0
43 0xA1F2 U+00A3 # POUND SIGN Windows: U+FFE1
44 0xA9A4 U+00A4 # CURRENCY SIGN [2000]
45 0xA1EF U+00A5 # YEN SIGN Windows: U+FFE5
46 0xA9A5 U+00A6 # BROKEN BAR [2000]
47 0xA1F8 U+00A7 # SECTION SIGN
48 0xA1AF U+00A8 # DIAERESIS
49 0xA9A6 U+00A9 # COPYRIGHT SIGN [2000]
50 0xA9A7 U+00AA # FEMININE ORDINAL INDICATOR [2000]
51 0xA9A8 U+00AB # LEFT-POINTING DOUBLE ANGLE QUOTATION MARK [2000]
52 0xA2CC U+00AC # NOT SIGN [1983] Windows: U+FFE2
53 0xA9A9 U+00AD # SOFT HYPHEN [2000]
54 0xA9AA U+00AE # REGISTERED SIGN [2000]
55 0xA9AB U+00AF # MACRON [2000]
56 0xA1EB U+00B0 # DEGREE SIGN
57 0xA1DE U+00B1 # PLUS-MINUS SIGN
58 0xA9AC U+00B2 # SUPERSCRIPT TWO [2000]
59 0xA9AD U+00B3 # SUPERSCRIPT THREE [2000]
60 0xA1AD U+00B4 # ACUTE ACCENT
61 0xA2F9 U+00B6 # PILCROW SIGN [1983]
62 0xA9AE U+00B7 # MIDDLE DOT [2000]
63 0xA9AF U+00B8 # CEDILLA [2000]
64 0xA9B0 U+00B9 # SUPERSCRIPT ONE [2000]
65 0xA9B1 U+00BA # MASCULINE ORDINAL INDICATOR [2000]
66 0xA9B2 U+00BB # RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK [2000]
67 0xA9B3 U+00BC # VULGAR FRACTION ONE QUARTER [2000]
68 0xA9B4 U+00BD # VULGAR FRACTION ONE HALF [2000]
69 0xA9B5 U+00BE # VULGAR FRACTION THREE QUARTERS [2000]
70 0xA9B6 U+00BF # INVERTED QUESTION MARK [2000]
71 0xA9B7 U+00C0 # LATIN CAPITAL LETTER A WITH GRAVE [2000]
72 0xA9B8 U+00C1 # LATIN CAPITAL LETTER A WITH ACUTE [2000]
73 0xA9B9 U+00C2 # LATIN CAPITAL LETTER A WITH CIRCUMFLEX [2000]
74 0xA9BA U+00C3 # LATIN CAPITAL LETTER A WITH TILDE [2000]
75 0xA9BB U+00C4 # LATIN CAPITAL LETTER A WITH DIAERESIS [2000]
76 0xA9BC U+00C5 # LATIN CAPITAL LETTER A WITH RING ABOVE [2000]
77 0xA9BD U+00C6 # LATIN CAPITAL LETTER AE [2000]
78 0xA9BE U+00C7 # LATIN CAPITAL LETTER C WITH CEDILLA [2000]
79 0xA9BF U+00C8 # LATIN CAPITAL LETTER E WITH GRAVE [2000]
80 0xA9C0 U+00C9 # LATIN CAPITAL LETTER E WITH ACUTE [2000]
81 0xA9C1 U+00CA # LATIN CAPITAL LETTER E WITH CIRCUMFLEX [2000]
82 0xA9C2 U+00CB # LATIN CAPITAL LETTER E WITH DIAERESIS [2000]
83 0xA9C3 U+00CC # LATIN CAPITAL LETTER I WITH GRAVE [2000]
84 0xA9C4 U+00CD # LATIN CAPITAL LETTER I WITH ACUTE [2000]
85 0xA9C5 U+00CE # LATIN CAPITAL LETTER I WITH CIRCUMFLEX [2000]
86 0xA9C6 U+00CF # LATIN CAPITAL LETTER I WITH DIAERESIS [2000]
87 0xA9C7 U+00D0 # LATIN CAPITAL LETTER ETH [2000]
88 0xA9C8 U+00D1 # LATIN CAPITAL LETTER N WITH TILDE [2000]
89 0xA9C9 U+00D2 # LATIN CAPITAL LETTER O WITH GRAVE [2000]
90 0xA9CA U+00D3 # LATIN CAPITAL LETTER O WITH ACUTE [2000]
91 0xA9CB U+00D4 # LATIN CAPITAL LETTER O WITH CIRCUMFLEX [2000]
92 0xA9CC U+00D5 # LATIN CAPITAL LETTER O WITH TILDE [2000]
93 0xA9CD U+00D6 # LATIN CAPITAL LETTER O WITH DIAERESIS [2000]
94 0xA1DF U+00D7 # MULTIPLICATION SIGN
95 0xA9CE U+00D8 # LATIN CAPITAL LETTER O WITH STROKE [2000]
96 0xA9CF U+00D9 # LATIN CAPITAL LETTER U WITH GRAVE [2000]
97 0xA9D0 U+00DA # LATIN CAPITAL LETTER U WITH ACUTE [2000]
98 0xA9D1 U+00DB # LATIN CAPITAL LETTER U WITH CIRCUMFLEX [2000]
99 0xA9D2 U+00DC # LATIN CAPITAL LETTER U WITH DIAERESIS [2000]
100 0xA9D3 U+00DD # LATIN CAPITAL LETTER Y WITH ACUTE [2000]
101 0xA9D4 U+00DE # LATIN CAPITAL LETTER THORN [2000]
102 0xA9D5 U+00DF # LATIN SMALL LETTER SHARP S [2000]
103 0xA9D6 U+00E0 # LATIN SMALL LETTER A WITH GRAVE [2000]
104 0xA9D7 U+00E1 # LATIN SMALL LETTER A WITH ACUTE [2000]
105 0xA9D8 U+00E2 # LATIN SMALL LETTER A WITH CIRCUMFLEX [2000]
106 0xA9D9 U+00E3 # LATIN SMALL LETTER A WITH TILDE [2000]
107 0xA9DA U+00E4 # LATIN SMALL LETTER A WITH DIAERESIS [2000]
108 0xA9DB U+00E5 # LATIN SMALL LETTER A WITH RING ABOVE [2000]
109 0xA9DC U+00E6 # LATIN SMALL LETTER AE [2000]
110 0xA9DD U+00E7 # LATIN SMALL LETTER C WITH CEDILLA [2000]
111 0xA9DE U+00E8 # LATIN SMALL LETTER E WITH GRAVE [2000]
112 0xA9DF U+00E9 # LATIN SMALL LETTER E WITH ACUTE [2000]
113 0xA9E0 U+00EA # LATIN SMALL LETTER E WITH CIRCUMFLEX [2000]
114 0xA9E1 U+00EB # LATIN SMALL LETTER E WITH DIAERESIS [2000]
115 0xA9E2 U+00EC # LATIN SMALL LETTER I WITH GRAVE [2000]
116 0xA9E3 U+00ED # LATIN SMALL LETTER I WITH ACUTE [2000]
117 0xA9E4 U+00EE # LATIN SMALL LETTER I WITH CIRCUMFLEX [2000]
118 0xA9E5 U+00EF # LATIN SMALL LETTER I WITH DIAERESIS [2000]
119 0xA9E6 U+00F0 # LATIN SMALL LETTER ETH [2000]
120 0xA9E7 U+00F1 # LATIN SMALL LETTER N WITH TILDE [2000]
121 0xA9E8 U+00F2 # LATIN SMALL LETTER O WITH GRAVE [2000]
122 0xA9E9 U+00F3 # LATIN SMALL LETTER O WITH ACUTE [2000]
123 0xA9EA U+00F4 # LATIN SMALL LETTER O WITH CIRCUMFLEX [2000]
124 0xA9EB U+00F5 # LATIN SMALL LETTER O WITH TILDE [2000]
125 0xA9EC U+00F6 # LATIN SMALL LETTER O WITH DIAERESIS [2000]
126 0xA1E0 U+00F7 # DIVISION SIGN
127 0xA9ED U+00F8 # LATIN SMALL LETTER O WITH STROKE [2000]
128 0xA9EE U+00F9 # LATIN SMALL LETTER U WITH GRAVE [2000]
129 0xA9EF U+00FA # LATIN SMALL LETTER U WITH ACUTE [2000]
130 0xA9F0 U+00FB # LATIN SMALL LETTER U WITH CIRCUMFLEX [2000]
131 0xA9F1 U+00FC # LATIN SMALL LETTER U WITH DIAERESIS [2000]
132 0xA9F2 U+00FD # LATIN SMALL LETTER Y WITH ACUTE [2000]
133 0xA9F3 U+00FE # LATIN SMALL LETTER THORN [2000]
134 0xA9F4 U+00FF # LATIN SMALL LETTER Y WITH DIAERESIS [2000]")
135
136 (define *lat1-eucjp-table*
137 ;; NB: We use U+3013 (EUC A2AE, "Geta Mark") as the replacement
138 ;; character respecting the tradition.
139 (rlet1 tab (make-vector (- #x100 #xa0) #xa2ae)
140 (map (^[line]
141 (rxmatch-case line
142 [#/0x([[:xdigit:]]+)\s+U\+([[:xdigit:]]+)/ (_ e u)
143 (vector-set! tab
144 (- (string->number u 16) #xa0)
145 (string->number e 16))]))
146 (string-split *lat1-eucjp-data* #\newline))))
147
148 (define (lat1.eucjp)
149 (do-lat1* "EUCJP"
150 (^b (let1 c (vector-ref *lat1-eucjp-table* (- b #xa0))
151 (if (<= c #xff)
152 (write-byte c)
153 (begin
154 (write-byte (ash c -8))
155 (write-byte (logand c #xff))))))))
156
157 (define (lat1.sjis)
158 (do-lat1* "SJIS"
159 (^b (let1 c (vector-ref *lat1-eucjp-table* (- b #xa0))
160 (if (<= c #xff)
161 (write-byte c)
162 (let1 m (open-output-string)
163 (write-byte (ash c -8) m)
164 (write-byte (logand c #xff) m)
165 (display (ces-convert (get-output-string m)
166 'eucjp 'sjis))))))))
167
168 (define (lat1.iso2022-jp)
169 (with-output-to-file #"lat1.ISO2022JP"
170 (^[] (display
171 (ces-convert (file->string "lat1.EUCJP") 'eucjp 'iso2022-jp)))))
172
173 (define (lat1.utf-16)
174 (define (expand vec be?)
175 (do-ec (: b vec)
176 (if be?
177 (begin (write-byte 0) (write-byte b))
178 (begin (write-byte b) (write-byte 0)))))
179 (define (generate name)
180 (let1 vec (call-with-input-file #"~|name|.ISO8859-1" port->uvector)
181 (with-output-to-file #"~|name|.UTF-16BE"
182 (cut expand vec #t))
183 (with-output-to-file #"~|name|.UTF-16LE"
184 (cut expand vec #f))
185 (with-output-to-file #"~|name|.UTF-16"
186 (^[] (write-byte #xfe) (write-byte #xff) (expand vec #t)))))
187 (generate "lat1")
188 (generate "lat1x"))
189
190 (define (lat1.utf-32)
191 (define (expand vec be?)
192 (do-ec (: b vec)
193 (if be?
194 (begin (dotimes [3] (write-byte 0)) (write-byte b))
195 (begin (write-byte b) (dotimes [3] (write-byte 0))))))
196 (define (generate name)
197 (let1 vec (call-with-input-file #"~|name|.ISO8859-1" port->uvector)
198 (with-output-to-file #"~|name|.UTF-32BE"
199 (cut expand vec #t))
200 (with-output-to-file #"~|name|.UTF-32LE"
201 (cut expand vec #f))
202 (with-output-to-file #"~|name|.UTF-32"
203 (^[] (write-byte 0) (write-byte 0) (write-byte #xfe) (write-byte #xff)
204 (expand vec #t)))))
205 (generate "lat1")
206 (generate "lat1x"))
207
208 (define (gen-lat1)
209 (lat1.iso8859-1)
210 (lat1.ascii)
211 (lat1.utf-8)
212 (lat1.utf-16)
213 (lat1.utf-32)
214 (lat1.eucjp)
215 (lat1.sjis)
216 (lat1.iso2022-jp))
217
218 ;;;
219 ;;; Latin-N test dataset
220 ;;;
221
222 ;; We just test latin-N <-> utf16
223 (define (do-latN n)
224 (define alis
225 (filter-map (^l (rxmatch-if (#/0x([[:xdigit:]]+)\s+0x([[:xdigit:]]+)/ l)
226 [_ code ucs]
227 (cons (string->number code 16) (string->number ucs 16))
228 #f))
229 (file->string-list #"8859-~|n|.TXT")))
230 (when (= n 7)
231 (pprint alis :level #f :length #f
232 :controls (make-write-controls :base 16 :radix #t)))
233 (with-output-to-file #"lat~|n|.ISO8859-~|n|"
234 (^[] (dolist [p alis] (write-byte (car p)))))
235 (with-output-to-file #"lat~|n|.UTF-16BE"
236 (^[] (dolist [p alis]
237 (when (= n 7)
238 (format (current-error-port) "~4,'0x\n" (cdr p)))
239 (write-byte (ash (cdr p) -8))
240 (write-byte (logand (cdr p) #xff))))))
241
242 (define (gen-latN)
243 (for-each do-latN '(2 3 4 5 6 7 8 9 10 11 13 14 15 16)))
244
245 ;;;
246
247 (define (main args)
248 (gen-lat1)
249 (gen-latN))