| 1 |
;;; |
| 2 |
;;; csv.scm - read and write CSV (actually, xSV) format. |
| 3 |
;;; |
| 4 |
;;; Copyright (c) 2000-2024 Shiro Kawai <[email protected]> |
| 5 |
;;; |
| 6 |
;;; Redistribution and use in source and binary forms, with or without |
| 7 |
;;; modification, are permitted provided that the following conditions |
| 8 |
;;; are met: |
| 9 |
;;; |
| 10 |
;;; 1. Redistributions of source code must retain the above copyright |
| 11 |
;;; notice, this list of conditions and the following disclaimer. |
| 12 |
;;; |
| 13 |
;;; 2. Redistributions in binary form must reproduce the above copyright |
| 14 |
;;; notice, this list of conditions and the following disclaimer in the |
| 15 |
;;; documentation and/or other materials provided with the distribution. |
| 16 |
;;; |
| 17 |
;;; 3. Neither the name of the authors nor the names of its contributors |
| 18 |
;;; may be used to endorse or promote products derived from this |
| 19 |
;;; software without specific prior written permission. |
| 20 |
;;; |
| 21 |
;;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 22 |
;;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 23 |
;;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 24 |
;;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 25 |
;;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 26 |
;;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED |
| 27 |
;;; TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR |
| 28 |
;;; PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF |
| 29 |
;;; LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING |
| 30 |
;;; NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| 31 |
;;; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 32 |
;;; |
| 33 |
|
| 34 |
(define-module text.csv |
| 35 |
(use scheme.list) |
| 36 |
(use scheme.charset) |
| 37 |
(use srfi.13) |
| 38 |
(use srfi.42) |
| 39 |
(use gauche.sequence) |
| 40 |
(export make-csv-reader |
| 41 |
make-csv-writer |
| 42 |
make-csv-header-parser |
| 43 |
make-csv-record-parser |
| 44 |
csv-rows->tuples) |
| 45 |
) |
| 46 |
(select-module text.csv) |
| 47 |
|
| 48 |
;;; |
| 49 |
;;;Low-level API - convert text into nested lists |
| 50 |
;;; |
| 51 |
|
| 52 |
;; API |
| 53 |
(define (make-csv-reader separator :optional (quote-char #\")) |
| 54 |
(^[:optional (port (current-input-port))] |
| 55 |
(csv-reader separator quote-char port))) |
| 56 |
|
| 57 |
(define (csv-reader sep quo port) |
| 58 |
(define (eor? ch) (or (eqv? ch #\newline) (eof-object? ch))) |
| 59 |
|
| 60 |
(define (start fields) |
| 61 |
(let1 ch (read-char port) |
| 62 |
(cond [(eor? ch) (reverse! (cons "" fields))] |
| 63 |
[(eqv? ch sep) (start (cons "" fields))] |
| 64 |
[(eqv? ch quo) (quoted fields)] |
| 65 |
[(char-whitespace? ch) (start fields)] |
| 66 |
[else (unquoted (list ch) fields)]))) |
| 67 |
|
| 68 |
(define (unquoted chs fields) |
| 69 |
(let loop ([ch (read-char port)] [last chs] [chs chs]) |
| 70 |
(cond [(eor? ch) (reverse! (cons (finish last) fields))] |
| 71 |
[(eqv? ch sep) (start (cons (finish last) fields))] |
| 72 |
[(char-whitespace? ch) (loop (read-char port) last (cons ch chs))] |
| 73 |
[else (let1 chs (cons ch chs) |
| 74 |
(loop (read-char port) chs chs))]))) |
| 75 |
|
| 76 |
(define (finish rchrs) (list->string (reverse! rchrs))) |
| 77 |
|
| 78 |
(define (quoted fields) |
| 79 |
(let loop ([ch (read-char port)] [chs '()]) |
| 80 |
(cond [(eof-object? ch) (error "unterminated quoted field")] |
| 81 |
[(eqv? ch quo) |
| 82 |
(if (eqv? (peek-char port) quo) |
| 83 |
(begin (read-char port) (loop (read-char port) (cons quo chs))) |
| 84 |
(quoted-tail (cons (finish chs) fields)))] |
| 85 |
[else (loop (read-char port) (cons ch chs))]))) |
| 86 |
|
| 87 |
(define (quoted-tail fields) |
| 88 |
(let loop ([ch (read-char port)]) |
| 89 |
(cond [(eor? ch) (reverse! fields)] |
| 90 |
[(eqv? ch sep) (start fields)] |
| 91 |
[else (loop (read-char port))]))) |
| 92 |
|
| 93 |
(if (eof-object? (peek-char port)) |
| 94 |
(eof-object) |
| 95 |
(start '()))) |
| 96 |
|
| 97 |
;; API |
| 98 |
(define (make-csv-writer separator :optional |
| 99 |
(newline "\n") (quote-char #\") |
| 100 |
(special-char-set #[\;\s])) |
| 101 |
(let* ([quote-string (string quote-char)] |
| 102 |
[quote-escape (string-append quote-string quote-string)] |
| 103 |
[quote-rx (string->regexp (regexp-quote quote-string))] |
| 104 |
[separator-chars (if (string? separator) |
| 105 |
(string->list separator) |
| 106 |
(list separator))] |
| 107 |
[newline-chars (if (string? newline) |
| 108 |
(string->list newline) |
| 109 |
(list newline))] |
| 110 |
[special-chars (apply char-set-adjoin special-char-set quote-char |
| 111 |
(append newline-chars separator-chars))]) |
| 112 |
(^[port fields] |
| 113 |
(define (write-a-field field) |
| 114 |
(if (string-index field special-chars) |
| 115 |
(begin (display quote-char port) |
| 116 |
(display (regexp-replace-all quote-rx field quote-escape) port) |
| 117 |
(display quote-char port)) |
| 118 |
(display field port))) |
| 119 |
|
| 120 |
(unless (null? fields) |
| 121 |
(write-a-field (car fields)) |
| 122 |
(dolist [field (cdr fields)] |
| 123 |
(display separator port) |
| 124 |
(write-a-field field))) |
| 125 |
(display newline port)))) |
| 126 |
|
| 127 |
;;; |
| 128 |
;;;Middle-level API |
| 129 |
;;; |
| 130 |
|
| 131 |
;; Occasionally, CVS files generated from spreadsheet contains |
| 132 |
;; superfluous rows/columns and we need to make sense of them. |
| 133 |
;; Here are some utilities to help them. |
| 134 |
|
| 135 |
;; Header spec is a string, a regexp or a predicate on a string. It |
| 136 |
;; is used to find the column in a header row. |
| 137 |
(define (%header-spec->pred spec) |
| 138 |
(cond [(string? spec) (^c (equal? spec c))] |
| 139 |
[(regexp? spec) (^c (rxmatch spec c))] |
| 140 |
[(applicable? spec <string>) spec] |
| 141 |
[else (error "Invalid header slot spec. Must be a string, an regexp or a predicate, but got:" spec)])) |
| 142 |
|
| 143 |
;; API |
| 144 |
;; Create a procedure that detects a row that contains the specified |
| 145 |
;; slots. Returns a permuter vector, which is a vector of integers, |
| 146 |
;; where K-th element being I to mean the K-th slot value should be |
| 147 |
;; taken from I-th column. |
| 148 |
;; |
| 149 |
;; Header-slots is a list of slot spec, which can be either a string, |
| 150 |
;; a regexp, or a predicate that takes a string. |
| 151 |
;; |
| 152 |
;; Example: |
| 153 |
;; input data: ("" "" "Year" "Country" "" "Population" "GDP") |
| 154 |
;; header-slots: ("Country" "Year" "GDP" "Population") |
| 155 |
;; result: #(3 2 6 5) |
| 156 |
(define (make-csv-header-parser header-slots) |
| 157 |
(define num-slots (length header-slots)) |
| 158 |
(define header-finder ; String -> Maybe Int |
| 159 |
(if (every string? header-slots) |
| 160 |
(let1 tab (make-hash-table 'equal?) ; Quick way |
| 161 |
(do-ec (: s (index k) header-slots) |
| 162 |
(hash-table-put! tab s k)) |
| 163 |
(^[column] (hash-table-get tab column #f))) |
| 164 |
(let1 mappers (map-with-index |
| 165 |
(^[i spec] |
| 166 |
(let1 p (%header-spec->pred spec) |
| 167 |
(^[column] (and (p column) i)))) |
| 168 |
header-slots) |
| 169 |
(^[column] (any (^m (m column)) mappers))))) |
| 170 |
(^[row] |
| 171 |
(and-let1 maps ($ filter identity |
| 172 |
(map-with-index (^[i c] (and-let1 k (header-finder c) |
| 173 |
(cons k i))) |
| 174 |
row)) |
| 175 |
(and (= num-slots (length maps)) |
| 176 |
(rlet1 permuter (make-vector num-slots) |
| 177 |
(do-ec (: p maps) |
| 178 |
(set! (~ permuter (car p)) (cdr p)))))))) |
| 179 |
|
| 180 |
;; API |
| 181 |
;; Create a procedure that converts one input row into a list of slot |
| 182 |
;; values, orderd in the same way as header-slots. |
| 183 |
;; The permuter is the vector returned by make-csv-header-parser. |
| 184 |
;; Required-slots determines if the input row is valid or not. If |
| 185 |
;; not, #f is returned. |
| 186 |
;; |
| 187 |
;; required-slots : (<spec> ...) |
| 188 |
;; <spec> : slot-spec | (slot-spec predicate) |
| 189 |
;; |
| 190 |
;; A single slot-spec in <spec> means `(,slot-spec ,(complement string-null?)) |
| 191 |
;; If required-slots is omitted or (), a row is regarded as valid if |
| 192 |
;; there's at least one non-null slots. |
| 193 |
(define (make-csv-record-parser header-slots permuter |
| 194 |
:optional (required-slots '())) |
| 195 |
(define (make-requirement-checker spec) |
| 196 |
(let* ([slot (if (pair? spec) (car spec) spec)] |
| 197 |
[pred (if (pair? spec) (cadr spec) (complement string-null?))] |
| 198 |
[ind (find-index (cut equal? slot <>) header-slots)]) |
| 199 |
(unless ind |
| 200 |
(error "make-csv-record-parser: invalid slot name in required-slots:" |
| 201 |
spec)) |
| 202 |
(^[slot-values] (pred (list-ref slot-values ind))))) |
| 203 |
(define check |
| 204 |
(if (null? required-slots) |
| 205 |
(^[slot-values] (not (every string-null? slot-values))) |
| 206 |
(apply every-pred (map make-requirement-checker required-slots)))) |
| 207 |
(^[row] |
| 208 |
(let1 slot-values (permute row permuter) |
| 209 |
(and (check slot-values) slot-values)))) |
| 210 |
|
| 211 |
;; API |
| 212 |
;; Convert input rows to a list of tuples (A tuple is a list of slot values) |
| 213 |
;; If no header is found, #f is returned. |
| 214 |
;; If allow-gap? is #t, it keeps reading rows until the end, skipping |
| 215 |
;; invalid rows. If allow-gap? is #f, it stops reading once it sees |
| 216 |
;; an invalid row after headers. |
| 217 |
(define (csv-rows->tuples rows header-slots |
| 218 |
:key (required-slots '()) |
| 219 |
(allow-gap? #f)) |
| 220 |
(define header-parser (make-csv-header-parser header-slots)) |
| 221 |
(let header-loop ([rows rows]) |
| 222 |
(cond [(null? rows) #f] ; no header found |
| 223 |
[(header-parser (car rows)) |
| 224 |
=> (^[permuter] |
| 225 |
(define record-parser |
| 226 |
(make-csv-record-parser header-slots |
| 227 |
permuter |
| 228 |
required-slots)) |
| 229 |
(let record-loop ([rows (cdr rows)] [r '()]) |
| 230 |
(cond [(null? rows) (reverse r)] |
| 231 |
[(record-parser (car rows)) |
| 232 |
=> (^[tuple] (record-loop (cdr rows) (cons tuple r)))] |
| 233 |
[allow-gap? (record-loop (cdr rows) r)] |
| 234 |
[else (reverse r)])))] |
| 235 |
[else (header-loop (cdr rows))]))) |