;;; ;;; Generate some of the data files ;;; (use file.util) (use gauche.charconv) (use gauche.uvector) (use srfi.42) ;;; ;;; Latin-1 test dataset ;;; (define (do-lat1 suffix emitter exclude-b5?) (with-output-to-file #"~(if exclude-b5? 'lat1x 'lat1).~suffix" (^[] (dotimes [n 256] (cond [(<= n #x7f) (write-byte n)] [(and exclude-b5? (= n #xb5))] [(<= #xa0 n) (emitter n)]) (when (zero? (modulo (+ n 1) 16)) (newline)))))) (define (do-lat1* suffix emitter) (do-lat1 suffix emitter #f) (do-lat1 suffix emitter #t)) (define (lat1.iso8859-1) (do-lat1* "ISO8859-1" write-byte)) (define (lat1.ascii) (do-lat1 "ASCII" (^b (write-char #\?)) #f)) (define (lat1.utf-8) (do-lat1* "UTF-8" (^b (write-byte (+ #xc0 (ash b -6))) (write-byte (+ #x80 (logand b #x3f)))))) ;; NB: U+00B5 (MICRO SIGN) doesn't have a corresponding JIS character. ;; We could've mapped it ot greek mu, but then we'll lose round-trip identity. (define *lat1-eucjp-data* " 0xA9A2 U+00A0 # NO-BREAK SPACE [2000] 0xA9A3 U+00A1 # INVERTED EXCLAMATION MARK [2000] 0xA1F1 U+00A2 # CENT SIGN Windows: U+FFE0 0xA1F2 U+00A3 # POUND SIGN Windows: U+FFE1 0xA9A4 U+00A4 # CURRENCY SIGN [2000] 0xA1EF U+00A5 # YEN SIGN Windows: U+FFE5 0xA9A5 U+00A6 # BROKEN BAR [2000] 0xA1F8 U+00A7 # SECTION SIGN 0xA1AF U+00A8 # DIAERESIS 0xA9A6 U+00A9 # COPYRIGHT SIGN [2000] 0xA9A7 U+00AA # FEMININE ORDINAL INDICATOR [2000] 0xA9A8 U+00AB # LEFT-POINTING DOUBLE ANGLE QUOTATION MARK [2000] 0xA2CC U+00AC # NOT SIGN [1983] Windows: U+FFE2 0xA9A9 U+00AD # SOFT HYPHEN [2000] 0xA9AA U+00AE # REGISTERED SIGN [2000] 0xA9AB U+00AF # MACRON [2000] 0xA1EB U+00B0 # DEGREE SIGN 0xA1DE U+00B1 # PLUS-MINUS SIGN 0xA9AC U+00B2 # SUPERSCRIPT TWO [2000] 0xA9AD U+00B3 # SUPERSCRIPT THREE [2000] 0xA1AD U+00B4 # ACUTE ACCENT 0xA2F9 U+00B6 # PILCROW SIGN [1983] 0xA9AE U+00B7 # MIDDLE DOT [2000] 0xA9AF U+00B8 # CEDILLA [2000] 0xA9B0 U+00B9 # SUPERSCRIPT ONE [2000] 0xA9B1 U+00BA # MASCULINE ORDINAL INDICATOR [2000] 0xA9B2 U+00BB # RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK [2000] 0xA9B3 U+00BC # VULGAR FRACTION ONE QUARTER [2000] 0xA9B4 U+00BD # VULGAR FRACTION ONE HALF [2000] 0xA9B5 U+00BE # VULGAR FRACTION THREE QUARTERS [2000] 0xA9B6 U+00BF # INVERTED QUESTION MARK [2000] 0xA9B7 U+00C0 # LATIN CAPITAL LETTER A WITH GRAVE [2000] 0xA9B8 U+00C1 # LATIN CAPITAL LETTER A WITH ACUTE [2000] 0xA9B9 U+00C2 # LATIN CAPITAL LETTER A WITH CIRCUMFLEX [2000] 0xA9BA U+00C3 # LATIN CAPITAL LETTER A WITH TILDE [2000] 0xA9BB U+00C4 # LATIN CAPITAL LETTER A WITH DIAERESIS [2000] 0xA9BC U+00C5 # LATIN CAPITAL LETTER A WITH RING ABOVE [2000] 0xA9BD U+00C6 # LATIN CAPITAL LETTER AE [2000] 0xA9BE U+00C7 # LATIN CAPITAL LETTER C WITH CEDILLA [2000] 0xA9BF U+00C8 # LATIN CAPITAL LETTER E WITH GRAVE [2000] 0xA9C0 U+00C9 # LATIN CAPITAL LETTER E WITH ACUTE [2000] 0xA9C1 U+00CA # LATIN CAPITAL LETTER E WITH CIRCUMFLEX [2000] 0xA9C2 U+00CB # LATIN CAPITAL LETTER E WITH DIAERESIS [2000] 0xA9C3 U+00CC # LATIN CAPITAL LETTER I WITH GRAVE [2000] 0xA9C4 U+00CD # LATIN CAPITAL LETTER I WITH ACUTE [2000] 0xA9C5 U+00CE # LATIN CAPITAL LETTER I WITH CIRCUMFLEX [2000] 0xA9C6 U+00CF # LATIN CAPITAL LETTER I WITH DIAERESIS [2000] 0xA9C7 U+00D0 # LATIN CAPITAL LETTER ETH [2000] 0xA9C8 U+00D1 # LATIN CAPITAL LETTER N WITH TILDE [2000] 0xA9C9 U+00D2 # LATIN CAPITAL LETTER O WITH GRAVE [2000] 0xA9CA U+00D3 # LATIN CAPITAL LETTER O WITH ACUTE [2000] 0xA9CB U+00D4 # LATIN CAPITAL LETTER O WITH CIRCUMFLEX [2000] 0xA9CC U+00D5 # LATIN CAPITAL LETTER O WITH TILDE [2000] 0xA9CD U+00D6 # LATIN CAPITAL LETTER O WITH DIAERESIS [2000] 0xA1DF U+00D7 # MULTIPLICATION SIGN 0xA9CE U+00D8 # LATIN CAPITAL LETTER O WITH STROKE [2000] 0xA9CF U+00D9 # LATIN CAPITAL LETTER U WITH GRAVE [2000] 0xA9D0 U+00DA # LATIN CAPITAL LETTER U WITH ACUTE [2000] 0xA9D1 U+00DB # LATIN CAPITAL LETTER U WITH CIRCUMFLEX [2000] 0xA9D2 U+00DC # LATIN CAPITAL LETTER U WITH DIAERESIS [2000] 0xA9D3 U+00DD # LATIN CAPITAL LETTER Y WITH ACUTE [2000] 0xA9D4 U+00DE # LATIN CAPITAL LETTER THORN [2000] 0xA9D5 U+00DF # LATIN SMALL LETTER SHARP S [2000] 0xA9D6 U+00E0 # LATIN SMALL LETTER A WITH GRAVE [2000] 0xA9D7 U+00E1 # LATIN SMALL LETTER A WITH ACUTE [2000] 0xA9D8 U+00E2 # LATIN SMALL LETTER A WITH CIRCUMFLEX [2000] 0xA9D9 U+00E3 # LATIN SMALL LETTER A WITH TILDE [2000] 0xA9DA U+00E4 # LATIN SMALL LETTER A WITH DIAERESIS [2000] 0xA9DB U+00E5 # LATIN SMALL LETTER A WITH RING ABOVE [2000] 0xA9DC U+00E6 # LATIN SMALL LETTER AE [2000] 0xA9DD U+00E7 # LATIN SMALL LETTER C WITH CEDILLA [2000] 0xA9DE U+00E8 # LATIN SMALL LETTER E WITH GRAVE [2000] 0xA9DF U+00E9 # LATIN SMALL LETTER E WITH ACUTE [2000] 0xA9E0 U+00EA # LATIN SMALL LETTER E WITH CIRCUMFLEX [2000] 0xA9E1 U+00EB # LATIN SMALL LETTER E WITH DIAERESIS [2000] 0xA9E2 U+00EC # LATIN SMALL LETTER I WITH GRAVE [2000] 0xA9E3 U+00ED # LATIN SMALL LETTER I WITH ACUTE [2000] 0xA9E4 U+00EE # LATIN SMALL LETTER I WITH CIRCUMFLEX [2000] 0xA9E5 U+00EF # LATIN SMALL LETTER I WITH DIAERESIS [2000] 0xA9E6 U+00F0 # LATIN SMALL LETTER ETH [2000] 0xA9E7 U+00F1 # LATIN SMALL LETTER N WITH TILDE [2000] 0xA9E8 U+00F2 # LATIN SMALL LETTER O WITH GRAVE [2000] 0xA9E9 U+00F3 # LATIN SMALL LETTER O WITH ACUTE [2000] 0xA9EA U+00F4 # LATIN SMALL LETTER O WITH CIRCUMFLEX [2000] 0xA9EB U+00F5 # LATIN SMALL LETTER O WITH TILDE [2000] 0xA9EC U+00F6 # LATIN SMALL LETTER O WITH DIAERESIS [2000] 0xA1E0 U+00F7 # DIVISION SIGN 0xA9ED U+00F8 # LATIN SMALL LETTER O WITH STROKE [2000] 0xA9EE U+00F9 # LATIN SMALL LETTER U WITH GRAVE [2000] 0xA9EF U+00FA # LATIN SMALL LETTER U WITH ACUTE [2000] 0xA9F0 U+00FB # LATIN SMALL LETTER U WITH CIRCUMFLEX [2000] 0xA9F1 U+00FC # LATIN SMALL LETTER U WITH DIAERESIS [2000] 0xA9F2 U+00FD # LATIN SMALL LETTER Y WITH ACUTE [2000] 0xA9F3 U+00FE # LATIN SMALL LETTER THORN [2000] 0xA9F4 U+00FF # LATIN SMALL LETTER Y WITH DIAERESIS [2000]") (define *lat1-eucjp-table* ;; NB: We use U+3013 (EUC A2AE, "Geta Mark") as the replacement ;; character respecting the tradition. (rlet1 tab (make-vector (- #x100 #xa0) #xa2ae) (map (^[line] (rxmatch-case line [#/0x([[:xdigit:]]+)\s+U\+([[:xdigit:]]+)/ (_ e u) (vector-set! tab (- (string->number u 16) #xa0) (string->number e 16))])) (string-split *lat1-eucjp-data* #\newline)))) (define (lat1.eucjp) (do-lat1* "EUCJP" (^b (let1 c (vector-ref *lat1-eucjp-table* (- b #xa0)) (if (<= c #xff) (write-byte c) (begin (write-byte (ash c -8)) (write-byte (logand c #xff)))))))) (define (lat1.sjis) (do-lat1* "SJIS" (^b (let1 c (vector-ref *lat1-eucjp-table* (- b #xa0)) (if (<= c #xff) (write-byte c) (let1 m (open-output-string) (write-byte (ash c -8) m) (write-byte (logand c #xff) m) (display (ces-convert (get-output-string m) 'eucjp 'sjis)))))))) (define (lat1.iso2022-jp) (with-output-to-file #"lat1.ISO2022JP" (^[] (display (ces-convert (file->string "lat1.EUCJP") 'eucjp 'iso2022-jp))))) (define (lat1.utf-16) (define (expand vec be?) (do-ec (: b vec) (if be? (begin (write-byte 0) (write-byte b)) (begin (write-byte b) (write-byte 0))))) (define (generate name) (let1 vec (call-with-input-file #"~|name|.ISO8859-1" port->uvector) (with-output-to-file #"~|name|.UTF-16BE" (cut expand vec #t)) (with-output-to-file #"~|name|.UTF-16LE" (cut expand vec #f)) (with-output-to-file #"~|name|.UTF-16" (^[] (write-byte #xfe) (write-byte #xff) (expand vec #t))))) (generate "lat1") (generate "lat1x")) (define (lat1.utf-32) (define (expand vec be?) (do-ec (: b vec) (if be? (begin (dotimes [3] (write-byte 0)) (write-byte b)) (begin (write-byte b) (dotimes [3] (write-byte 0)))))) (define (generate name) (let1 vec (call-with-input-file #"~|name|.ISO8859-1" port->uvector) (with-output-to-file #"~|name|.UTF-32BE" (cut expand vec #t)) (with-output-to-file #"~|name|.UTF-32LE" (cut expand vec #f)) (with-output-to-file #"~|name|.UTF-32" (^[] (write-byte 0) (write-byte 0) (write-byte #xfe) (write-byte #xff) (expand vec #t))))) (generate "lat1") (generate "lat1x")) (define (gen-lat1) (lat1.iso8859-1) (lat1.ascii) (lat1.utf-8) (lat1.utf-16) (lat1.utf-32) (lat1.eucjp) (lat1.sjis) (lat1.iso2022-jp)) ;;; ;;; Latin-N test dataset ;;; ;; We just test latin-N <-> utf16 (define (do-latN n) (define alis (filter-map (^l (rxmatch-if (#/0x([[:xdigit:]]+)\s+0x([[:xdigit:]]+)/ l) [_ code ucs] (cons (string->number code 16) (string->number ucs 16)) #f)) (file->string-list #"8859-~|n|.TXT"))) (when (= n 7) (pprint alis :level #f :length #f :controls (make-write-controls :base 16 :radix #t))) (with-output-to-file #"lat~|n|.ISO8859-~|n|" (^[] (dolist [p alis] (write-byte (car p))))) (with-output-to-file #"lat~|n|.UTF-16BE" (^[] (dolist [p alis] (when (= n 7) (format (current-error-port) "~4,'0x\n" (cdr p))) (write-byte (ash (cdr p) -8)) (write-byte (logand (cdr p) #xff)))))) (define (gen-latN) (for-each do-latN '(2 3 4 5 6 7 8 9 10 11 13 14 15 16))) ;;; (define (main args) (gen-lat1) (gen-latN))