Debian Codegraph @

lib/text/csv.scm

raw
1 ;;;
2 ;;; csv.scm - read and write CSV (actually, xSV) format.
3 ;;;
4 ;;; Copyright (c) 2000-2024 Shiro Kawai <[email protected]>
5 ;;;
6 ;;; Redistribution and use in source and binary forms, with or without
7 ;;; modification, are permitted provided that the following conditions
8 ;;; are met:
9 ;;;
10 ;;; 1. Redistributions of source code must retain the above copyright
11 ;;; notice, this list of conditions and the following disclaimer.
12 ;;;
13 ;;; 2. Redistributions in binary form must reproduce the above copyright
14 ;;; notice, this list of conditions and the following disclaimer in the
15 ;;; documentation and/or other materials provided with the distribution.
16 ;;;
17 ;;; 3. Neither the name of the authors nor the names of its contributors
18 ;;; may be used to endorse or promote products derived from this
19 ;;; software without specific prior written permission.
20 ;;;
21 ;;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 ;;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 ;;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 ;;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 ;;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 ;;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
27 ;;; TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
28 ;;; PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
29 ;;; LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
30 ;;; NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
31 ;;; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 ;;;
33
34 (define-module text.csv
35 (use scheme.list)
36 (use scheme.charset)
37 (use srfi.13)
38 (use srfi.42)
39 (use gauche.sequence)
40 (export make-csv-reader
41 make-csv-writer
42 make-csv-header-parser
43 make-csv-record-parser
44 csv-rows->tuples)
45 )
46 (select-module text.csv)
47
48 ;;;
49 ;;;Low-level API - convert text into nested lists
50 ;;;
51
52 ;; API
53 (define (make-csv-reader separator :optional (quote-char #\"))
54 (^[:optional (port (current-input-port))]
55 (csv-reader separator quote-char port)))
56
57 (define (csv-reader sep quo port)
58 (define (eor? ch) (or (eqv? ch #\newline) (eof-object? ch)))
59
60 (define (start fields)
61 (let1 ch (read-char port)
62 (cond [(eor? ch) (reverse! (cons "" fields))]
63 [(eqv? ch sep) (start (cons "" fields))]
64 [(eqv? ch quo) (quoted fields)]
65 [(char-whitespace? ch) (start fields)]
66 [else (unquoted (list ch) fields)])))
67
68 (define (unquoted chs fields)
69 (let loop ([ch (read-char port)] [last chs] [chs chs])
70 (cond [(eor? ch) (reverse! (cons (finish last) fields))]
71 [(eqv? ch sep) (start (cons (finish last) fields))]
72 [(char-whitespace? ch) (loop (read-char port) last (cons ch chs))]
73 [else (let1 chs (cons ch chs)
74 (loop (read-char port) chs chs))])))
75
76 (define (finish rchrs) (list->string (reverse! rchrs)))
77
78 (define (quoted fields)
79 (let loop ([ch (read-char port)] [chs '()])
80 (cond [(eof-object? ch) (error "unterminated quoted field")]
81 [(eqv? ch quo)
82 (if (eqv? (peek-char port) quo)
83 (begin (read-char port) (loop (read-char port) (cons quo chs)))
84 (quoted-tail (cons (finish chs) fields)))]
85 [else (loop (read-char port) (cons ch chs))])))
86
87 (define (quoted-tail fields)
88 (let loop ([ch (read-char port)])
89 (cond [(eor? ch) (reverse! fields)]
90 [(eqv? ch sep) (start fields)]
91 [else (loop (read-char port))])))
92
93 (if (eof-object? (peek-char port))
94 (eof-object)
95 (start '())))
96
97 ;; API
98 (define (make-csv-writer separator :optional
99 (newline "\n") (quote-char #\")
100 (special-char-set #[\;\s]))
101 (let* ([quote-string (string quote-char)]
102 [quote-escape (string-append quote-string quote-string)]
103 [quote-rx (string->regexp (regexp-quote quote-string))]
104 [separator-chars (if (string? separator)
105 (string->list separator)
106 (list separator))]
107 [newline-chars (if (string? newline)
108 (string->list newline)
109 (list newline))]
110 [special-chars (apply char-set-adjoin special-char-set quote-char
111 (append newline-chars separator-chars))])
112 (^[port fields]
113 (define (write-a-field field)
114 (if (string-index field special-chars)
115 (begin (display quote-char port)
116 (display (regexp-replace-all quote-rx field quote-escape) port)
117 (display quote-char port))
118 (display field port)))
119
120 (unless (null? fields)
121 (write-a-field (car fields))
122 (dolist [field (cdr fields)]
123 (display separator port)
124 (write-a-field field)))
125 (display newline port))))
126
127 ;;;
128 ;;;Middle-level API
129 ;;;
130
131 ;; Occasionally, CVS files generated from spreadsheet contains
132 ;; superfluous rows/columns and we need to make sense of them.
133 ;; Here are some utilities to help them.
134
135 ;; Header spec is a string, a regexp or a predicate on a string. It
136 ;; is used to find the column in a header row.
137 (define (%header-spec->pred spec)
138 (cond [(string? spec) (^c (equal? spec c))]
139 [(regexp? spec) (^c (rxmatch spec c))]
140 [(applicable? spec <string>) spec]
141 [else (error "Invalid header slot spec. Must be a string, an regexp or a predicate, but got:" spec)]))
142
143 ;; API
144 ;; Create a procedure that detects a row that contains the specified
145 ;; slots. Returns a permuter vector, which is a vector of integers,
146 ;; where K-th element being I to mean the K-th slot value should be
147 ;; taken from I-th column.
148 ;;
149 ;; Header-slots is a list of slot spec, which can be either a string,
150 ;; a regexp, or a predicate that takes a string.
151 ;;
152 ;; Example:
153 ;; input data: ("" "" "Year" "Country" "" "Population" "GDP")
154 ;; header-slots: ("Country" "Year" "GDP" "Population")
155 ;; result: #(3 2 6 5)
156 (define (make-csv-header-parser header-slots)
157 (define num-slots (length header-slots))
158 (define header-finder ; String -> Maybe Int
159 (if (every string? header-slots)
160 (let1 tab (make-hash-table 'equal?) ; Quick way
161 (do-ec (: s (index k) header-slots)
162 (hash-table-put! tab s k))
163 (^[column] (hash-table-get tab column #f)))
164 (let1 mappers (map-with-index
165 (^[i spec]
166 (let1 p (%header-spec->pred spec)
167 (^[column] (and (p column) i))))
168 header-slots)
169 (^[column] (any (^m (m column)) mappers)))))
170 (^[row]
171 (and-let1 maps ($ filter identity
172 (map-with-index (^[i c] (and-let1 k (header-finder c)
173 (cons k i)))
174 row))
175 (and (= num-slots (length maps))
176 (rlet1 permuter (make-vector num-slots)
177 (do-ec (: p maps)
178 (set! (~ permuter (car p)) (cdr p))))))))
179
180 ;; API
181 ;; Create a procedure that converts one input row into a list of slot
182 ;; values, orderd in the same way as header-slots.
183 ;; The permuter is the vector returned by make-csv-header-parser.
184 ;; Required-slots determines if the input row is valid or not. If
185 ;; not, #f is returned.
186 ;;
187 ;; required-slots : (<spec> ...)
188 ;; <spec> : slot-spec | (slot-spec predicate)
189 ;;
190 ;; A single slot-spec in <spec> means `(,slot-spec ,(complement string-null?))
191 ;; If required-slots is omitted or (), a row is regarded as valid if
192 ;; there's at least one non-null slots.
193 (define (make-csv-record-parser header-slots permuter
194 :optional (required-slots '()))
195 (define (make-requirement-checker spec)
196 (let* ([slot (if (pair? spec) (car spec) spec)]
197 [pred (if (pair? spec) (cadr spec) (complement string-null?))]
198 [ind (find-index (cut equal? slot <>) header-slots)])
199 (unless ind
200 (error "make-csv-record-parser: invalid slot name in required-slots:"
201 spec))
202 (^[slot-values] (pred (list-ref slot-values ind)))))
203 (define check
204 (if (null? required-slots)
205 (^[slot-values] (not (every string-null? slot-values)))
206 (apply every-pred (map make-requirement-checker required-slots))))
207 (^[row]
208 (let1 slot-values (permute row permuter)
209 (and (check slot-values) slot-values))))
210
211 ;; API
212 ;; Convert input rows to a list of tuples (A tuple is a list of slot values)
213 ;; If no header is found, #f is returned.
214 ;; If allow-gap? is #t, it keeps reading rows until the end, skipping
215 ;; invalid rows. If allow-gap? is #f, it stops reading once it sees
216 ;; an invalid row after headers.
217 (define (csv-rows->tuples rows header-slots
218 :key (required-slots '())
219 (allow-gap? #f))
220 (define header-parser (make-csv-header-parser header-slots))
221 (let header-loop ([rows rows])
222 (cond [(null? rows) #f] ; no header found
223 [(header-parser (car rows))
224 => (^[permuter]
225 (define record-parser
226 (make-csv-record-parser header-slots
227 permuter
228 required-slots))
229 (let record-loop ([rows (cdr rows)] [r '()])
230 (cond [(null? rows) (reverse r)]
231 [(record-parser (car rows))
232 => (^[tuple] (record-loop (cdr rows) (cons tuple r)))]
233 [allow-gap? (record-loop (cdr rows) r)]
234 [else (reverse r)])))]
235 [else (header-loop (cdr rows))])))