| 1 |
;;; |
| 2 |
;;; Generate some of the data files |
| 3 |
;;; |
| 4 |
|
| 5 |
(use file.util) |
| 6 |
(use gauche.charconv) |
| 7 |
(use gauche.uvector) |
| 8 |
(use srfi.42) |
| 9 |
|
| 10 |
;;; |
| 11 |
;;; Latin-1 test dataset |
| 12 |
;;; |
| 13 |
|
| 14 |
(define (do-lat1 suffix emitter exclude-b5?) |
| 15 |
(with-output-to-file #"~(if exclude-b5? 'lat1x 'lat1).~suffix" |
| 16 |
(^[] (dotimes [n 256] |
| 17 |
(cond [(<= n #x7f) (write-byte n)] |
| 18 |
[(and exclude-b5? (= n #xb5))] |
| 19 |
[(<= #xa0 n) (emitter n)]) |
| 20 |
(when (zero? (modulo (+ n 1) 16)) (newline)))))) |
| 21 |
|
| 22 |
(define (do-lat1* suffix emitter) |
| 23 |
(do-lat1 suffix emitter #f) |
| 24 |
(do-lat1 suffix emitter #t)) |
| 25 |
|
| 26 |
(define (lat1.iso8859-1) |
| 27 |
(do-lat1* "ISO8859-1" write-byte)) |
| 28 |
|
| 29 |
(define (lat1.ascii) |
| 30 |
(do-lat1 "ASCII" (^b (write-char #\?)) #f)) |
| 31 |
|
| 32 |
(define (lat1.utf-8) |
| 33 |
(do-lat1* "UTF-8" |
| 34 |
(^b (write-byte (+ #xc0 (ash b -6))) |
| 35 |
(write-byte (+ #x80 (logand b #x3f)))))) |
| 36 |
|
| 37 |
;; NB: U+00B5 (MICRO SIGN) doesn't have a corresponding JIS character. |
| 38 |
;; We could've mapped it ot greek mu, but then we'll lose round-trip identity. |
| 39 |
(define *lat1-eucjp-data* " |
| 40 |
0xA9A2 U+00A0 # NO-BREAK SPACE [2000] |
| 41 |
0xA9A3 U+00A1 # INVERTED EXCLAMATION MARK [2000] |
| 42 |
0xA1F1 U+00A2 # CENT SIGN Windows: U+FFE0 |
| 43 |
0xA1F2 U+00A3 # POUND SIGN Windows: U+FFE1 |
| 44 |
0xA9A4 U+00A4 # CURRENCY SIGN [2000] |
| 45 |
0xA1EF U+00A5 # YEN SIGN Windows: U+FFE5 |
| 46 |
0xA9A5 U+00A6 # BROKEN BAR [2000] |
| 47 |
0xA1F8 U+00A7 # SECTION SIGN |
| 48 |
0xA1AF U+00A8 # DIAERESIS |
| 49 |
0xA9A6 U+00A9 # COPYRIGHT SIGN [2000] |
| 50 |
0xA9A7 U+00AA # FEMININE ORDINAL INDICATOR [2000] |
| 51 |
0xA9A8 U+00AB # LEFT-POINTING DOUBLE ANGLE QUOTATION MARK [2000] |
| 52 |
0xA2CC U+00AC # NOT SIGN [1983] Windows: U+FFE2 |
| 53 |
0xA9A9 U+00AD # SOFT HYPHEN [2000] |
| 54 |
0xA9AA U+00AE # REGISTERED SIGN [2000] |
| 55 |
0xA9AB U+00AF # MACRON [2000] |
| 56 |
0xA1EB U+00B0 # DEGREE SIGN |
| 57 |
0xA1DE U+00B1 # PLUS-MINUS SIGN |
| 58 |
0xA9AC U+00B2 # SUPERSCRIPT TWO [2000] |
| 59 |
0xA9AD U+00B3 # SUPERSCRIPT THREE [2000] |
| 60 |
0xA1AD U+00B4 # ACUTE ACCENT |
| 61 |
0xA2F9 U+00B6 # PILCROW SIGN [1983] |
| 62 |
0xA9AE U+00B7 # MIDDLE DOT [2000] |
| 63 |
0xA9AF U+00B8 # CEDILLA [2000] |
| 64 |
0xA9B0 U+00B9 # SUPERSCRIPT ONE [2000] |
| 65 |
0xA9B1 U+00BA # MASCULINE ORDINAL INDICATOR [2000] |
| 66 |
0xA9B2 U+00BB # RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK [2000] |
| 67 |
0xA9B3 U+00BC # VULGAR FRACTION ONE QUARTER [2000] |
| 68 |
0xA9B4 U+00BD # VULGAR FRACTION ONE HALF [2000] |
| 69 |
0xA9B5 U+00BE # VULGAR FRACTION THREE QUARTERS [2000] |
| 70 |
0xA9B6 U+00BF # INVERTED QUESTION MARK [2000] |
| 71 |
0xA9B7 U+00C0 # LATIN CAPITAL LETTER A WITH GRAVE [2000] |
| 72 |
0xA9B8 U+00C1 # LATIN CAPITAL LETTER A WITH ACUTE [2000] |
| 73 |
0xA9B9 U+00C2 # LATIN CAPITAL LETTER A WITH CIRCUMFLEX [2000] |
| 74 |
0xA9BA U+00C3 # LATIN CAPITAL LETTER A WITH TILDE [2000] |
| 75 |
0xA9BB U+00C4 # LATIN CAPITAL LETTER A WITH DIAERESIS [2000] |
| 76 |
0xA9BC U+00C5 # LATIN CAPITAL LETTER A WITH RING ABOVE [2000] |
| 77 |
0xA9BD U+00C6 # LATIN CAPITAL LETTER AE [2000] |
| 78 |
0xA9BE U+00C7 # LATIN CAPITAL LETTER C WITH CEDILLA [2000] |
| 79 |
0xA9BF U+00C8 # LATIN CAPITAL LETTER E WITH GRAVE [2000] |
| 80 |
0xA9C0 U+00C9 # LATIN CAPITAL LETTER E WITH ACUTE [2000] |
| 81 |
0xA9C1 U+00CA # LATIN CAPITAL LETTER E WITH CIRCUMFLEX [2000] |
| 82 |
0xA9C2 U+00CB # LATIN CAPITAL LETTER E WITH DIAERESIS [2000] |
| 83 |
0xA9C3 U+00CC # LATIN CAPITAL LETTER I WITH GRAVE [2000] |
| 84 |
0xA9C4 U+00CD # LATIN CAPITAL LETTER I WITH ACUTE [2000] |
| 85 |
0xA9C5 U+00CE # LATIN CAPITAL LETTER I WITH CIRCUMFLEX [2000] |
| 86 |
0xA9C6 U+00CF # LATIN CAPITAL LETTER I WITH DIAERESIS [2000] |
| 87 |
0xA9C7 U+00D0 # LATIN CAPITAL LETTER ETH [2000] |
| 88 |
0xA9C8 U+00D1 # LATIN CAPITAL LETTER N WITH TILDE [2000] |
| 89 |
0xA9C9 U+00D2 # LATIN CAPITAL LETTER O WITH GRAVE [2000] |
| 90 |
0xA9CA U+00D3 # LATIN CAPITAL LETTER O WITH ACUTE [2000] |
| 91 |
0xA9CB U+00D4 # LATIN CAPITAL LETTER O WITH CIRCUMFLEX [2000] |
| 92 |
0xA9CC U+00D5 # LATIN CAPITAL LETTER O WITH TILDE [2000] |
| 93 |
0xA9CD U+00D6 # LATIN CAPITAL LETTER O WITH DIAERESIS [2000] |
| 94 |
0xA1DF U+00D7 # MULTIPLICATION SIGN |
| 95 |
0xA9CE U+00D8 # LATIN CAPITAL LETTER O WITH STROKE [2000] |
| 96 |
0xA9CF U+00D9 # LATIN CAPITAL LETTER U WITH GRAVE [2000] |
| 97 |
0xA9D0 U+00DA # LATIN CAPITAL LETTER U WITH ACUTE [2000] |
| 98 |
0xA9D1 U+00DB # LATIN CAPITAL LETTER U WITH CIRCUMFLEX [2000] |
| 99 |
0xA9D2 U+00DC # LATIN CAPITAL LETTER U WITH DIAERESIS [2000] |
| 100 |
0xA9D3 U+00DD # LATIN CAPITAL LETTER Y WITH ACUTE [2000] |
| 101 |
0xA9D4 U+00DE # LATIN CAPITAL LETTER THORN [2000] |
| 102 |
0xA9D5 U+00DF # LATIN SMALL LETTER SHARP S [2000] |
| 103 |
0xA9D6 U+00E0 # LATIN SMALL LETTER A WITH GRAVE [2000] |
| 104 |
0xA9D7 U+00E1 # LATIN SMALL LETTER A WITH ACUTE [2000] |
| 105 |
0xA9D8 U+00E2 # LATIN SMALL LETTER A WITH CIRCUMFLEX [2000] |
| 106 |
0xA9D9 U+00E3 # LATIN SMALL LETTER A WITH TILDE [2000] |
| 107 |
0xA9DA U+00E4 # LATIN SMALL LETTER A WITH DIAERESIS [2000] |
| 108 |
0xA9DB U+00E5 # LATIN SMALL LETTER A WITH RING ABOVE [2000] |
| 109 |
0xA9DC U+00E6 # LATIN SMALL LETTER AE [2000] |
| 110 |
0xA9DD U+00E7 # LATIN SMALL LETTER C WITH CEDILLA [2000] |
| 111 |
0xA9DE U+00E8 # LATIN SMALL LETTER E WITH GRAVE [2000] |
| 112 |
0xA9DF U+00E9 # LATIN SMALL LETTER E WITH ACUTE [2000] |
| 113 |
0xA9E0 U+00EA # LATIN SMALL LETTER E WITH CIRCUMFLEX [2000] |
| 114 |
0xA9E1 U+00EB # LATIN SMALL LETTER E WITH DIAERESIS [2000] |
| 115 |
0xA9E2 U+00EC # LATIN SMALL LETTER I WITH GRAVE [2000] |
| 116 |
0xA9E3 U+00ED # LATIN SMALL LETTER I WITH ACUTE [2000] |
| 117 |
0xA9E4 U+00EE # LATIN SMALL LETTER I WITH CIRCUMFLEX [2000] |
| 118 |
0xA9E5 U+00EF # LATIN SMALL LETTER I WITH DIAERESIS [2000] |
| 119 |
0xA9E6 U+00F0 # LATIN SMALL LETTER ETH [2000] |
| 120 |
0xA9E7 U+00F1 # LATIN SMALL LETTER N WITH TILDE [2000] |
| 121 |
0xA9E8 U+00F2 # LATIN SMALL LETTER O WITH GRAVE [2000] |
| 122 |
0xA9E9 U+00F3 # LATIN SMALL LETTER O WITH ACUTE [2000] |
| 123 |
0xA9EA U+00F4 # LATIN SMALL LETTER O WITH CIRCUMFLEX [2000] |
| 124 |
0xA9EB U+00F5 # LATIN SMALL LETTER O WITH TILDE [2000] |
| 125 |
0xA9EC U+00F6 # LATIN SMALL LETTER O WITH DIAERESIS [2000] |
| 126 |
0xA1E0 U+00F7 # DIVISION SIGN |
| 127 |
0xA9ED U+00F8 # LATIN SMALL LETTER O WITH STROKE [2000] |
| 128 |
0xA9EE U+00F9 # LATIN SMALL LETTER U WITH GRAVE [2000] |
| 129 |
0xA9EF U+00FA # LATIN SMALL LETTER U WITH ACUTE [2000] |
| 130 |
0xA9F0 U+00FB # LATIN SMALL LETTER U WITH CIRCUMFLEX [2000] |
| 131 |
0xA9F1 U+00FC # LATIN SMALL LETTER U WITH DIAERESIS [2000] |
| 132 |
0xA9F2 U+00FD # LATIN SMALL LETTER Y WITH ACUTE [2000] |
| 133 |
0xA9F3 U+00FE # LATIN SMALL LETTER THORN [2000] |
| 134 |
0xA9F4 U+00FF # LATIN SMALL LETTER Y WITH DIAERESIS [2000]") |
| 135 |
|
| 136 |
(define *lat1-eucjp-table* |
| 137 |
;; NB: We use U+3013 (EUC A2AE, "Geta Mark") as the replacement |
| 138 |
;; character respecting the tradition. |
| 139 |
(rlet1 tab (make-vector (- #x100 #xa0) #xa2ae) |
| 140 |
(map (^[line] |
| 141 |
(rxmatch-case line |
| 142 |
[#/0x([[:xdigit:]]+)\s+U\+([[:xdigit:]]+)/ (_ e u) |
| 143 |
(vector-set! tab |
| 144 |
(- (string->number u 16) #xa0) |
| 145 |
(string->number e 16))])) |
| 146 |
(string-split *lat1-eucjp-data* #\newline)))) |
| 147 |
|
| 148 |
(define (lat1.eucjp) |
| 149 |
(do-lat1* "EUCJP" |
| 150 |
(^b (let1 c (vector-ref *lat1-eucjp-table* (- b #xa0)) |
| 151 |
(if (<= c #xff) |
| 152 |
(write-byte c) |
| 153 |
(begin |
| 154 |
(write-byte (ash c -8)) |
| 155 |
(write-byte (logand c #xff)))))))) |
| 156 |
|
| 157 |
(define (lat1.sjis) |
| 158 |
(do-lat1* "SJIS" |
| 159 |
(^b (let1 c (vector-ref *lat1-eucjp-table* (- b #xa0)) |
| 160 |
(if (<= c #xff) |
| 161 |
(write-byte c) |
| 162 |
(let1 m (open-output-string) |
| 163 |
(write-byte (ash c -8) m) |
| 164 |
(write-byte (logand c #xff) m) |
| 165 |
(display (ces-convert (get-output-string m) |
| 166 |
'eucjp 'sjis)))))))) |
| 167 |
|
| 168 |
(define (lat1.iso2022-jp) |
| 169 |
(with-output-to-file #"lat1.ISO2022JP" |
| 170 |
(^[] (display |
| 171 |
(ces-convert (file->string "lat1.EUCJP") 'eucjp 'iso2022-jp))))) |
| 172 |
|
| 173 |
(define (lat1.utf-16) |
| 174 |
(define (expand vec be?) |
| 175 |
(do-ec (: b vec) |
| 176 |
(if be? |
| 177 |
(begin (write-byte 0) (write-byte b)) |
| 178 |
(begin (write-byte b) (write-byte 0))))) |
| 179 |
(define (generate name) |
| 180 |
(let1 vec (call-with-input-file #"~|name|.ISO8859-1" port->uvector) |
| 181 |
(with-output-to-file #"~|name|.UTF-16BE" |
| 182 |
(cut expand vec #t)) |
| 183 |
(with-output-to-file #"~|name|.UTF-16LE" |
| 184 |
(cut expand vec #f)) |
| 185 |
(with-output-to-file #"~|name|.UTF-16" |
| 186 |
(^[] (write-byte #xfe) (write-byte #xff) (expand vec #t))))) |
| 187 |
(generate "lat1") |
| 188 |
(generate "lat1x")) |
| 189 |
|
| 190 |
(define (lat1.utf-32) |
| 191 |
(define (expand vec be?) |
| 192 |
(do-ec (: b vec) |
| 193 |
(if be? |
| 194 |
(begin (dotimes [3] (write-byte 0)) (write-byte b)) |
| 195 |
(begin (write-byte b) (dotimes [3] (write-byte 0)))))) |
| 196 |
(define (generate name) |
| 197 |
(let1 vec (call-with-input-file #"~|name|.ISO8859-1" port->uvector) |
| 198 |
(with-output-to-file #"~|name|.UTF-32BE" |
| 199 |
(cut expand vec #t)) |
| 200 |
(with-output-to-file #"~|name|.UTF-32LE" |
| 201 |
(cut expand vec #f)) |
| 202 |
(with-output-to-file #"~|name|.UTF-32" |
| 203 |
(^[] (write-byte 0) (write-byte 0) (write-byte #xfe) (write-byte #xff) |
| 204 |
(expand vec #t))))) |
| 205 |
(generate "lat1") |
| 206 |
(generate "lat1x")) |
| 207 |
|
| 208 |
(define (gen-lat1) |
| 209 |
(lat1.iso8859-1) |
| 210 |
(lat1.ascii) |
| 211 |
(lat1.utf-8) |
| 212 |
(lat1.utf-16) |
| 213 |
(lat1.utf-32) |
| 214 |
(lat1.eucjp) |
| 215 |
(lat1.sjis) |
| 216 |
(lat1.iso2022-jp)) |
| 217 |
|
| 218 |
;;; |
| 219 |
;;; Latin-N test dataset |
| 220 |
;;; |
| 221 |
|
| 222 |
;; We just test latin-N <-> utf16 |
| 223 |
(define (do-latN n) |
| 224 |
(define alis |
| 225 |
(filter-map (^l (rxmatch-if (#/0x([[:xdigit:]]+)\s+0x([[:xdigit:]]+)/ l) |
| 226 |
[_ code ucs] |
| 227 |
(cons (string->number code 16) (string->number ucs 16)) |
| 228 |
#f)) |
| 229 |
(file->string-list #"8859-~|n|.TXT"))) |
| 230 |
(when (= n 7) |
| 231 |
(pprint alis :level #f :length #f |
| 232 |
:controls (make-write-controls :base 16 :radix #t))) |
| 233 |
(with-output-to-file #"lat~|n|.ISO8859-~|n|" |
| 234 |
(^[] (dolist [p alis] (write-byte (car p))))) |
| 235 |
(with-output-to-file #"lat~|n|.UTF-16BE" |
| 236 |
(^[] (dolist [p alis] |
| 237 |
(when (= n 7) |
| 238 |
(format (current-error-port) "~4,'0x\n" (cdr p))) |
| 239 |
(write-byte (ash (cdr p) -8)) |
| 240 |
(write-byte (logand (cdr p) #xff)))))) |
| 241 |
|
| 242 |
(define (gen-latN) |
| 243 |
(for-each do-latN '(2 3 4 5 6 7 8 9 10 11 13 14 15 16))) |
| 244 |
|
| 245 |
;;; |
| 246 |
|
| 247 |
(define (main args) |
| 248 |
(gen-lat1) |
| 249 |
(gen-latN)) |