Debian Codegraph @

ext/misc/csv.c

raw
1 /*
2 ** 2016-05-28
3 **
4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
6 **
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
10 **
11 ******************************************************************************
12 **
13 ** This file contains the implementation of an SQLite virtual table for
14 ** reading CSV files.
15 **
16 ** Usage:
17 **
18 ** .load ./csv
19 ** CREATE VIRTUAL TABLE temp.csv USING csv(filename=FILENAME);
20 ** SELECT * FROM csv;
21 **
22 ** The columns are named "c1", "c2", "c3", ... by default. Or the
23 ** application can define its own CREATE TABLE statement using the
24 ** schema= parameter, like this:
25 **
26 ** CREATE VIRTUAL TABLE temp.csv2 USING csv(
27 ** filename = '../http.log',
28 ** schema = 'CREATE TABLE x(date,ipaddr,url,referrer,userAgent)'
29 ** );
30 **
31 ** Instead of specifying a file, the text of the CSV can be loaded using
32 ** the data= parameter.
33 **
34 ** If the columns=N parameter is supplied, then the CSV file is assumed to have
35 ** N columns. If both the columns= and schema= parameters are omitted, then
36 ** the number and names of the columns is determined by the first line of
37 ** the CSV input.
38 **
39 ** Some extra debugging features (used for testing virtual tables) are available
40 ** if this module is compiled with -DSQLITE_TEST.
41 */
42 #include <sqlite3ext.h>
43 SQLITE_EXTENSION_INIT1
44 #include <string.h>
45 #include <stdlib.h>
46 #include <assert.h>
47 #include <stdarg.h>
48 #include <ctype.h>
49 #include <stdio.h>
50
51 #ifndef SQLITE_OMIT_VIRTUALTABLE
52
53 /*
54 ** A macro to hint to the compiler that a function should not be
55 ** inlined.
56 */
57 #if defined(__GNUC__)
58 # define CSV_NOINLINE __attribute__((noinline))
59 #elif defined(_MSC_VER) && _MSC_VER>=1310
60 # define CSV_NOINLINE __declspec(noinline)
61 #else
62 # define CSV_NOINLINE
63 #endif
64
65 #ifndef SQLITEINT_H
66 typedef sqlite3_int64 i64;
67 typedef sqlite3_uint64 u64;
68 #endif
69
70 /* Max size of the error message in a CsvReader */
71 #define CSV_MXERR 200
72
73 /* Size of the CsvReader input buffer */
74 #define CSV_INBUFSZ 1024
75
76 /* A context object used when read a CSV file. */
77 typedef struct CsvReader CsvReader;
78 struct CsvReader {
79 FILE *in; /* Read the CSV text from this input stream */
80 char *z; /* Accumulated text for a field */
81 i64 n; /* Number of bytes in z */
82 i64 nAlloc; /* Space allocated for z[] */
83 i64 nLine; /* Current line number */
84 int bNotFirst; /* True if prior text has been seen */
85 int cTerm; /* Character that terminated the most recent field */
86 size_t iIn; /* Next unread character in the input buffer */
87 size_t nIn; /* Number of characters in the input buffer */
88 char *zIn; /* The input buffer */
89 char zErr[CSV_MXERR]; /* Error message */
90 };
91
92 /* Initialize a CsvReader object */
93 static void csv_reader_init(CsvReader *p){
94 p->in = 0;
95 p->z = 0;
96 p->n = 0;
97 p->nAlloc = 0;
98 p->nLine = 0;
99 p->bNotFirst = 0;
100 p->nIn = 0;
101 p->zIn = 0;
102 p->zErr[0] = 0;
103 }
104
105 /* Close and reset a CsvReader object */
106 static void csv_reader_reset(CsvReader *p){
107 if( p->in ){
108 fclose(p->in);
109 sqlite3_free(p->zIn);
110 }
111 sqlite3_free(p->z);
112 csv_reader_init(p);
113 }
114
115 /* Report an error on a CsvReader */
116 static void csv_errmsg(CsvReader *p, const char *zFormat, ...){
117 va_list ap;
118 va_start(ap, zFormat);
119 sqlite3_vsnprintf(CSV_MXERR, p->zErr, zFormat, ap);
120 va_end(ap);
121 }
122
123 /* Open the file associated with a CsvReader
124 ** Return the number of errors.
125 */
126 static int csv_reader_open(
127 CsvReader *p, /* The reader to open */
128 const char *zFilename, /* Read from this filename */
129 const char *zData /* ... or use this data */
130 ){
131 if( zFilename ){
132 p->zIn = sqlite3_malloc64( CSV_INBUFSZ );
133 if( p->zIn==0 ){
134 csv_errmsg(p, "out of memory");
135 return 1;
136 }
137 p->in = fopen(zFilename, "rb");
138 if( p->in==0 ){
139 sqlite3_free(p->zIn);
140 csv_reader_reset(p);
141 csv_errmsg(p, "cannot open '%s' for reading", zFilename);
142 return 1;
143 }
144 }else{
145 assert( p->in==0 );
146 p->zIn = (char*)zData;
147 p->nIn = strlen(zData);
148 }
149 return 0;
150 }
151
152 /* The input buffer has overflowed. Refill the input buffer, then
153 ** return the next character
154 */
155 static CSV_NOINLINE int csv_getc_refill(CsvReader *p){
156 size_t got;
157
158 assert( p->iIn>=p->nIn ); /* Only called on an empty input buffer */
159 assert( p->in!=0 ); /* Only called if reading froma file */
160
161 got = fread(p->zIn, 1, CSV_INBUFSZ, p->in);
162 if( got==0 ) return EOF;
163 p->nIn = got;
164 p->iIn = 1;
165 return p->zIn[0];
166 }
167
168 /* Return the next character of input. Return EOF at end of input. */
169 static int csv_getc(CsvReader *p){
170 if( p->iIn >= p->nIn ){
171 if( p->in!=0 ) return csv_getc_refill(p);
172 return EOF;
173 }
174 return ((unsigned char*)p->zIn)[p->iIn++];
175 }
176
177 /* Increase the size of p->z and append character c to the end.
178 ** Return 0 on success and non-zero if there is an OOM error */
179 static CSV_NOINLINE int csv_resize_and_append(CsvReader *p, char c){
180 char *zNew;
181 i64 nNew = p->nAlloc*2 + 100;
182 zNew = sqlite3_realloc64(p->z, nNew);
183 if( zNew ){
184 p->z = zNew;
185 p->nAlloc = nNew;
186 p->z[p->n++] = c;
187 return 0;
188 }else{
189 csv_errmsg(p, "out of memory");
190 return 1;
191 }
192 }
193
194 /* Append a single character to the CsvReader.z[] array.
195 ** Return 0 on success and non-zero if there is an OOM error */
196 static int csv_append(CsvReader *p, char c){
197 if( p->n>=p->nAlloc-1 ) return csv_resize_and_append(p, c);
198 p->z[p->n++] = c;
199 return 0;
200 }
201
202 /* Read a single field of CSV text. Compatible with rfc4180 and extended
203 ** with the option of having a separator other than ",".
204 **
205 ** + Input comes from p->in.
206 ** + Store results in p->z of length p->n. Space to hold p->z comes
207 ** from sqlite3_malloc64().
208 ** + Keep track of the line number in p->nLine.
209 ** + Store the character that terminates the field in p->cTerm. Store
210 ** EOF on end-of-file.
211 **
212 ** Return 0 at EOF or on OOM. On EOF, the p->cTerm character will have
213 ** been set to EOF.
214 */
215 static char *csv_read_one_field(CsvReader *p){
216 int c;
217 p->n = 0;
218 c = csv_getc(p);
219 if( c==EOF ){
220 p->cTerm = EOF;
221 return 0;
222 }
223 if( c=='"' ){
224 int pc, ppc;
225 i64 startLine = p->nLine;
226 pc = ppc = 0;
227 while( 1 ){
228 c = csv_getc(p);
229 if( c<='"' || pc=='"' ){
230 if( c=='\n' ) p->nLine++;
231 if( c=='"' ){
232 if( pc=='"' ){
233 pc = 0;
234 continue;
235 }
236 }
237 if( (c==',' && pc=='"')
238 || (c=='\n' && pc=='"')
239 || (c=='\n' && pc=='\r' && ppc=='"')
240 || (c==EOF && pc=='"')
241 ){
242 do{ p->n--; }while( p->z[p->n]!='"' );
243 p->cTerm = (char)c;
244 break;
245 }
246 if( pc=='"' && c!='\r' ){
247 csv_errmsg(p, "line %d: unescaped %c character", p->nLine, '"');
248 break;
249 }
250 if( c==EOF ){
251 csv_errmsg(p, "line %d: unterminated %c-quoted field\n",
252 startLine, '"');
253 p->cTerm = (char)c;
254 break;
255 }
256 }
257 if( csv_append(p, (char)c) ) return 0;
258 ppc = pc;
259 pc = c;
260 }
261 }else{
262 /* If this is the first field being parsed and it begins with the
263 ** UTF-8 BOM (0xEF BB BF) then skip the BOM */
264 if( (c&0xff)==0xef && p->bNotFirst==0 ){
265 csv_append(p, (char)c);
266 c = csv_getc(p);
267 if( (c&0xff)==0xbb ){
268 csv_append(p, (char)c);
269 c = csv_getc(p);
270 if( (c&0xff)==0xbf ){
271 p->bNotFirst = 1;
272 p->n = 0;
273 return csv_read_one_field(p);
274 }
275 }
276 }
277 while( c>',' || (c!=EOF && c!=',' && c!='\n') ){
278 if( csv_append(p, (char)c) ) return 0;
279 c = csv_getc(p);
280 }
281 if( c=='\n' ){
282 p->nLine++;
283 if( p->n>0 && p->z[p->n-1]=='\r' ) p->n--;
284 }
285 p->cTerm = (char)c;
286 }
287 assert( p->z==0 || p->n<p->nAlloc );
288 if( p->z ) p->z[p->n] = 0;
289 p->bNotFirst = 1;
290 return p->z;
291 }
292
293
294 /* Forward references to the various virtual table methods implemented
295 ** in this file. */
296 static int csvtabCreate(sqlite3*, void*, int, const char*const*,
297 sqlite3_vtab**,char**);
298 static int csvtabConnect(sqlite3*, void*, int, const char*const*,
299 sqlite3_vtab**,char**);
300 static int csvtabBestIndex(sqlite3_vtab*,sqlite3_index_info*);
301 static int csvtabDisconnect(sqlite3_vtab*);
302 static int csvtabOpen(sqlite3_vtab*, sqlite3_vtab_cursor**);
303 static int csvtabClose(sqlite3_vtab_cursor*);
304 static int csvtabFilter(sqlite3_vtab_cursor*, int idxNum, const char *idxStr,
305 int argc, sqlite3_value **argv);
306 static int csvtabNext(sqlite3_vtab_cursor*);
307 static int csvtabEof(sqlite3_vtab_cursor*);
308 static int csvtabColumn(sqlite3_vtab_cursor*,sqlite3_context*,int);
309 static int csvtabRowid(sqlite3_vtab_cursor*,sqlite3_int64*);
310
311 /* An instance of the CSV virtual table */
312 typedef struct CsvTable {
313 sqlite3_vtab base; /* Base class. Must be first */
314 char *zFilename; /* Name of the CSV file */
315 char *zData; /* Raw CSV data in lieu of zFilename */
316 long iStart; /* Offset to start of data in zFilename */
317 int nCol; /* Number of columns in the CSV file */
318 unsigned int tstFlags; /* Bit values used for testing */
319 } CsvTable;
320
321 /* Allowed values for tstFlags */
322 #define CSVTEST_FIDX 0x0001 /* Pretend that constrained search cost less*/
323
324 /* A cursor for the CSV virtual table */
325 typedef struct CsvCursor {
326 sqlite3_vtab_cursor base; /* Base class. Must be first */
327 CsvReader rdr; /* The CsvReader object */
328 char **azVal; /* Value of the current row */
329 i64 *aLen; /* Length of each entry */
330 sqlite3_int64 iRowid; /* The current rowid. Negative for EOF */
331 } CsvCursor;
332
333 /* Transfer error message text from a reader into a CsvTable */
334 static void csv_xfer_error(CsvTable *pTab, CsvReader *pRdr){
335 sqlite3_free(pTab->base.zErrMsg);
336 pTab->base.zErrMsg = sqlite3_mprintf("%s", pRdr->zErr);
337 }
338
339 /*
340 ** This method is the destructor fo a CsvTable object.
341 */
342 static int csvtabDisconnect(sqlite3_vtab *pVtab){
343 CsvTable *p = (CsvTable*)pVtab;
344 sqlite3_free(p->zFilename);
345 sqlite3_free(p->zData);
346 sqlite3_free(p);
347 return SQLITE_OK;
348 }
349
350 /* Skip leading whitespace. Return a pointer to the first non-whitespace
351 ** character, or to the zero terminator if the string has only whitespace */
352 static const char *csv_skip_whitespace(const char *z){
353 while( isspace((unsigned char)z[0]) ) z++;
354 return z;
355 }
356
357 /* Remove trailing whitespace from the end of string z[] */
358 static void csv_trim_whitespace(char *z){
359 size_t n = strlen(z);
360 while( n>0 && isspace((unsigned char)z[n]) ) n--;
361 z[n] = 0;
362 }
363
364 /* Dequote the string */
365 static void csv_dequote(char *z){
366 int j;
367 char cQuote = z[0];
368 size_t i, n;
369
370 if( cQuote!='\'' && cQuote!='"' ) return;
371 n = strlen(z);
372 if( n<2 || z[n-1]!=z[0] ) return;
373 for(i=1, j=0; i<n-1; i++){
374 if( z[i]==cQuote && z[i+1]==cQuote ) i++;
375 z[j++] = z[i];
376 }
377 z[j] = 0;
378 }
379
380 /* Check to see if the string is of the form: "TAG = VALUE" with optional
381 ** whitespace before and around tokens. If it is, return a pointer to the
382 ** first character of VALUE. If it is not, return NULL.
383 */
384 static const char *csv_parameter(const char *zTag, int nTag, const char *z){
385 z = csv_skip_whitespace(z);
386 if( strncmp(zTag, z, nTag)!=0 ) return 0;
387 z = csv_skip_whitespace(z+nTag);
388 if( z[0]!='=' ) return 0;
389 return csv_skip_whitespace(z+1);
390 }
391
392 /* Decode a parameter that requires a dequoted string.
393 **
394 ** Return 1 if the parameter is seen, or 0 if not. 1 is returned
395 ** even if there is an error. If an error occurs, then an error message
396 ** is left in p->zErr. If there are no errors, p->zErr[0]==0.
397 */
398 static int csv_string_parameter(
399 CsvReader *p, /* Leave the error message here, if there is one */
400 const char *zParam, /* Parameter we are checking for */
401 const char *zArg, /* Raw text of the virtual table argment */
402 char **pzVal /* Write the dequoted string value here */
403 ){
404 const char *zValue;
405 zValue = csv_parameter(zParam,(int)strlen(zParam),zArg);
406 if( zValue==0 ) return 0;
407 p->zErr[0] = 0;
408 if( *pzVal ){
409 csv_errmsg(p, "more than one '%s' parameter", zParam);
410 return 1;
411 }
412 *pzVal = sqlite3_mprintf("%s", zValue);
413 if( *pzVal==0 ){
414 csv_errmsg(p, "out of memory");
415 return 1;
416 }
417 csv_trim_whitespace(*pzVal);
418 csv_dequote(*pzVal);
419 return 1;
420 }
421
422
423 /* Return 0 if the argument is false and 1 if it is true. Return -1 if
424 ** we cannot really tell.
425 */
426 static int csv_boolean(const char *z){
427 if( sqlite3_stricmp("yes",z)==0
428 || sqlite3_stricmp("on",z)==0
429 || sqlite3_stricmp("true",z)==0
430 || (z[0]=='1' && z[1]==0)
431 ){
432 return 1;
433 }
434 if( sqlite3_stricmp("no",z)==0
435 || sqlite3_stricmp("off",z)==0
436 || sqlite3_stricmp("false",z)==0
437 || (z[0]=='0' && z[1]==0)
438 ){
439 return 0;
440 }
441 return -1;
442 }
443
444 /* Check to see if the string is of the form: "TAG = BOOLEAN" or just "TAG".
445 ** If it is, set *pValue to be the value of the boolean ("true" if there is
446 ** not "= BOOLEAN" component) and return non-zero. If the input string
447 ** does not begin with TAG, return zero.
448 */
449 static int csv_boolean_parameter(
450 const char *zTag, /* Tag we are looking for */
451 int nTag, /* Size of the tag in bytes */
452 const char *z, /* Input parameter */
453 int *pValue /* Write boolean value here */
454 ){
455 int b;
456 z = csv_skip_whitespace(z);
457 if( strncmp(zTag, z, nTag)!=0 ) return 0;
458 z = csv_skip_whitespace(z + nTag);
459 if( z[0]==0 ){
460 *pValue = 1;
461 return 1;
462 }
463 if( z[0]!='=' ) return 0;
464 z = csv_skip_whitespace(z+1);
465 b = csv_boolean(z);
466 if( b>=0 ){
467 *pValue = b;
468 return 1;
469 }
470 return 0;
471 }
472
473 /*
474 ** Parameters:
475 ** filename=FILENAME Name of file containing CSV content
476 ** data=TEXT Direct CSV content.
477 ** schema=SCHEMA Alternative CSV schema.
478 ** header=YES|NO First row of CSV defines the names of
479 ** columns if "yes". Default "no".
480 ** columns=N Assume the CSV file contains N columns.
481 **
482 ** Only available if compiled with SQLITE_TEST:
483 **
484 ** testflags=N Bitmask of test flags. Optional
485 **
486 ** If schema= is omitted, then the columns are named "c0", "c1", "c2",
487 ** and so forth. If columns=N is omitted, then the file is opened and
488 ** the number of columns in the first row is counted to determine the
489 ** column count. If header=YES, then the first row is skipped.
490 */
491 static int csvtabConnect(
492 sqlite3 *db,
493 void *pAux,
494 int argc, const char *const*argv,
495 sqlite3_vtab **ppVtab,
496 char **pzErr
497 ){
498 CsvTable *pNew = 0; /* The CsvTable object to construct */
499 int bHeader = -1; /* header= flags. -1 means not seen yet */
500 int rc = SQLITE_OK; /* Result code from this routine */
501 u64 i, j; /* Loop counters */
502 #ifdef SQLITE_TEST
503 int tstFlags = 0; /* Value for testflags=N parameter */
504 #endif
505 int b; /* Value of a boolean parameter */
506 int nCol = -99; /* Value of the columns= parameter */
507 CsvReader sRdr; /* A CSV file reader used to store an error
508 ** message and/or to count the number of columns */
509 static const char *azParam[] = {
510 "filename", "data", "schema",
511 };
512 char *azPValue[3]; /* Parameter values */
513 # define CSV_FILENAME (azPValue[0])
514 # define CSV_DATA (azPValue[1])
515 # define CSV_SCHEMA (azPValue[2])
516
517 assert( sizeof(azPValue)==sizeof(azParam) );
518 memset(&sRdr, 0, sizeof(sRdr));
519 memset(azPValue, 0, sizeof(azPValue));
520 for(i=3; i<(u64)argc; i++){
521 const char *z = argv[i];
522 const char *zValue;
523 for(j=0; j<sizeof(azParam)/sizeof(azParam[0]); j++){
524 if( csv_string_parameter(&sRdr, azParam[j], z, &azPValue[j]) ) break;
525 }
526 if( j<sizeof(azParam)/sizeof(azParam[0]) ){
527 if( sRdr.zErr[0] ) goto csvtab_connect_error;
528 }else
529 if( csv_boolean_parameter("header",6,z,&b) ){
530 if( bHeader>=0 ){
531 csv_errmsg(&sRdr, "more than one 'header' parameter");
532 goto csvtab_connect_error;
533 }
534 bHeader = b;
535 }else
536 #ifdef SQLITE_TEST
537 if( (zValue = csv_parameter("testflags",9,z))!=0 ){
538 tstFlags = (unsigned int)atoi(zValue);
539 }else
540 #endif
541 if( (zValue = csv_parameter("columns",7,z))!=0 ){
542 if( nCol>0 ){
543 csv_errmsg(&sRdr, "more than one 'columns' parameter");
544 goto csvtab_connect_error;
545 }
546 nCol = atoi(zValue);
547 if( nCol<=0 ){
548 csv_errmsg(&sRdr, "column= value must be positive");
549 goto csvtab_connect_error;
550 }
551 }else
552 {
553 csv_errmsg(&sRdr, "bad parameter: '%s'", z);
554 goto csvtab_connect_error;
555 }
556 }
557 if( (CSV_FILENAME==0)==(CSV_DATA==0) ){
558 csv_errmsg(&sRdr, "must specify either filename= or data= but not both");
559 goto csvtab_connect_error;
560 }
561
562 if( (nCol<=0 || bHeader==1)
563 && csv_reader_open(&sRdr, CSV_FILENAME, CSV_DATA)
564 ){
565 goto csvtab_connect_error;
566 }
567 pNew = sqlite3_malloc64( sizeof(*pNew) );
568 *ppVtab = (sqlite3_vtab*)pNew;
569 if( pNew==0 ) goto csvtab_connect_oom;
570 memset(pNew, 0, sizeof(*pNew));
571 if( CSV_SCHEMA==0 ){
572 sqlite3_str *pStr = sqlite3_str_new(0);
573 char *zSep = "";
574 int iCol = 0;
575 sqlite3_str_appendf(pStr, "CREATE TABLE x(");
576 if( nCol<0 && bHeader<1 ){
577 nCol = 0;
578 do{
579 csv_read_one_field(&sRdr);
580 nCol++;
581 }while( sRdr.cTerm==',' );
582 }
583 if( nCol>0 && bHeader<1 ){
584 for(iCol=0; iCol<nCol; iCol++){
585 sqlite3_str_appendf(pStr, "%sc%d TEXT", zSep, iCol);
586 zSep = ",";
587 }
588 }else{
589 do{
590 char *z = csv_read_one_field(&sRdr);
591 if( (nCol>0 && iCol<nCol) || (nCol<0 && bHeader) ){
592 sqlite3_str_appendf(pStr,"%s\"%w\" TEXT", zSep, z);
593 zSep = ",";
594 iCol++;
595 }
596 }while( sRdr.cTerm==',' );
597 if( nCol<0 ){
598 nCol = iCol;
599 }else{
600 while( iCol<nCol ){
601 sqlite3_str_appendf(pStr,"%sc%d TEXT", zSep, ++iCol);
602 zSep = ",";
603 }
604 }
605 }
606 pNew->nCol = nCol;
607 sqlite3_str_appendf(pStr, ")");
608 CSV_SCHEMA = sqlite3_str_finish(pStr);
609 if( CSV_SCHEMA==0 ) goto csvtab_connect_oom;
610 }else if( nCol<0 ){
611 do{
612 csv_read_one_field(&sRdr);
613 pNew->nCol++;
614 }while( sRdr.cTerm==',' );
615 }else{
616 pNew->nCol = nCol;
617 }
618 pNew->zFilename = CSV_FILENAME; CSV_FILENAME = 0;
619 pNew->zData = CSV_DATA; CSV_DATA = 0;
620 #ifdef SQLITE_TEST
621 pNew->tstFlags = tstFlags;
622 #endif
623 if( bHeader!=1 ){
624 pNew->iStart = 0;
625 }else if( pNew->zData ){
626 pNew->iStart = (int)sRdr.iIn;
627 }else{
628 pNew->iStart = (int)(ftell(sRdr.in) - sRdr.nIn + sRdr.iIn);
629 }
630 csv_reader_reset(&sRdr);
631 rc = sqlite3_declare_vtab(db, CSV_SCHEMA);
632 if( rc ){
633 csv_errmsg(&sRdr, "bad schema: '%s' - %s", CSV_SCHEMA, sqlite3_errmsg(db));
634 goto csvtab_connect_error;
635 }
636 for(i=0; i<sizeof(azPValue)/sizeof(azPValue[0]); i++){
637 sqlite3_free(azPValue[i]);
638 }
639 /* Rationale for DIRECTONLY:
640 ** An attacker who controls a database schema could use this vtab
641 ** to exfiltrate sensitive data from other files in the filesystem.
642 ** And, recommended practice is to put all CSV virtual tables in the
643 ** TEMP namespace, so they should still be usable from within TEMP
644 ** views, so there shouldn't be a serious loss of functionality by
645 ** prohibiting the use of this vtab from persistent triggers and views.
646 */
647 sqlite3_vtab_config(db, SQLITE_VTAB_DIRECTONLY);
648 return SQLITE_OK;
649
650 csvtab_connect_oom:
651 rc = SQLITE_NOMEM;
652 csv_errmsg(&sRdr, "out of memory");
653
654 csvtab_connect_error:
655 if( pNew ) csvtabDisconnect(&pNew->base);
656 for(i=0; i<sizeof(azPValue)/sizeof(azPValue[0]); i++){
657 sqlite3_free(azPValue[i]);
658 }
659 if( sRdr.zErr[0] ){
660 sqlite3_free(*pzErr);
661 *pzErr = sqlite3_mprintf("%s", sRdr.zErr);
662 }
663 csv_reader_reset(&sRdr);
664 if( rc==SQLITE_OK ) rc = SQLITE_ERROR;
665 return rc;
666 }
667
668 /*
669 ** Reset the current row content held by a CsvCursor.
670 */
671 static void csvtabCursorRowReset(CsvCursor *pCur){
672 CsvTable *pTab = (CsvTable*)pCur->base.pVtab;
673 int i;
674 for(i=0; i<pTab->nCol; i++){
675 sqlite3_free(pCur->azVal[i]);
676 pCur->azVal[i] = 0;
677 pCur->aLen[i] = 0;
678 }
679 }
680
681 /*
682 ** The xConnect and xCreate methods do the same thing, but they must be
683 ** different so that the virtual table is not an eponymous virtual table.
684 */
685 static int csvtabCreate(
686 sqlite3 *db,
687 void *pAux,
688 int argc, const char *const*argv,
689 sqlite3_vtab **ppVtab,
690 char **pzErr
691 ){
692 return csvtabConnect(db, pAux, argc, argv, ppVtab, pzErr);
693 }
694
695 /*
696 ** Destructor for a CsvCursor.
697 */
698 static int csvtabClose(sqlite3_vtab_cursor *cur){
699 CsvCursor *pCur = (CsvCursor*)cur;
700 csvtabCursorRowReset(pCur);
701 csv_reader_reset(&pCur->rdr);
702 sqlite3_free(cur);
703 return SQLITE_OK;
704 }
705
706 /*
707 ** Constructor for a new CsvTable cursor object.
708 */
709 static int csvtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
710 CsvTable *pTab = (CsvTable*)p;
711 CsvCursor *pCur;
712 size_t nByte;
713 nByte = sizeof(*pCur) + (sizeof(char*)+sizeof(i64))*pTab->nCol;
714 pCur = sqlite3_malloc64( nByte );
715 if( pCur==0 ) return SQLITE_NOMEM;
716 memset(pCur, 0, nByte);
717 pCur->azVal = (char**)&pCur[1];
718 pCur->aLen = (i64*)&pCur->azVal[pTab->nCol];
719 *ppCursor = &pCur->base;
720 if( csv_reader_open(&pCur->rdr, pTab->zFilename, pTab->zData) ){
721 csv_xfer_error(pTab, &pCur->rdr);
722 return SQLITE_ERROR;
723 }
724 return SQLITE_OK;
725 }
726
727
728 /*
729 ** Advance a CsvCursor to its next row of input.
730 ** Set the EOF marker if we reach the end of input.
731 */
732 static int csvtabNext(sqlite3_vtab_cursor *cur){
733 CsvCursor *pCur = (CsvCursor*)cur;
734 CsvTable *pTab = (CsvTable*)cur->pVtab;
735 int i = 0;
736 char *z;
737 do{
738 z = csv_read_one_field(&pCur->rdr);
739 if( z==0 ){
740 break;
741 }
742 if( i<pTab->nCol ){
743 if( pCur->aLen[i] < pCur->rdr.n+1 ){
744 char *zNew = sqlite3_realloc64(pCur->azVal[i], pCur->rdr.n+1);
745 if( zNew==0 ){
746 csv_errmsg(&pCur->rdr, "out of memory");
747 csv_xfer_error(pTab, &pCur->rdr);
748 break;
749 }
750 pCur->azVal[i] = zNew;
751 pCur->aLen[i] = pCur->rdr.n+1;
752 }
753 memcpy(pCur->azVal[i], z, pCur->rdr.n+1);
754 i++;
755 }
756 }while( pCur->rdr.cTerm==',' );
757 if( z==0 && i==0 ){
758 pCur->iRowid = -1;
759 }else{
760 pCur->iRowid++;
761 while( i<pTab->nCol ){
762 sqlite3_free(pCur->azVal[i]);
763 pCur->azVal[i] = 0;
764 pCur->aLen[i] = 0;
765 i++;
766 }
767 }
768 return SQLITE_OK;
769 }
770
771 /*
772 ** Return values of columns for the row at which the CsvCursor
773 ** is currently pointing.
774 */
775 static int csvtabColumn(
776 sqlite3_vtab_cursor *cur, /* The cursor */
777 sqlite3_context *ctx, /* First argument to sqlite3_result_...() */
778 int i /* Which column to return */
779 ){
780 CsvCursor *pCur = (CsvCursor*)cur;
781 CsvTable *pTab = (CsvTable*)cur->pVtab;
782 if( i>=0 && i<pTab->nCol && pCur->azVal[i]!=0 ){
783 sqlite3_result_text(ctx, pCur->azVal[i], -1, SQLITE_TRANSIENT);
784 }
785 return SQLITE_OK;
786 }
787
788 /*
789 ** Return the rowid for the current row.
790 */
791 static int csvtabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
792 CsvCursor *pCur = (CsvCursor*)cur;
793 *pRowid = pCur->iRowid;
794 return SQLITE_OK;
795 }
796
797 /*
798 ** Return TRUE if the cursor has been moved off of the last
799 ** row of output.
800 */
801 static int csvtabEof(sqlite3_vtab_cursor *cur){
802 CsvCursor *pCur = (CsvCursor*)cur;
803 return pCur->iRowid<0;
804 }
805
806 /*
807 ** Only a full table scan is supported. So xFilter simply rewinds to
808 ** the beginning.
809 */
810 static int csvtabFilter(
811 sqlite3_vtab_cursor *pVtabCursor,
812 int idxNum, const char *idxStr,
813 int argc, sqlite3_value **argv
814 ){
815 CsvCursor *pCur = (CsvCursor*)pVtabCursor;
816 CsvTable *pTab = (CsvTable*)pVtabCursor->pVtab;
817 pCur->iRowid = 0;
818
819 /* Ensure the field buffer is always allocated. Otherwise, if the
820 ** first field is zero bytes in size, this may be mistaken for an OOM
821 ** error in csvtabNext(). */
822 if( csv_append(&pCur->rdr, 0) ) return SQLITE_NOMEM;
823
824 if( pCur->rdr.in==0 ){
825 assert( pCur->rdr.zIn==pTab->zData );
826 assert( pTab->iStart>=0 );
827 assert( (size_t)pTab->iStart<=pCur->rdr.nIn );
828 pCur->rdr.iIn = pTab->iStart;
829 }else{
830 fseek(pCur->rdr.in, pTab->iStart, SEEK_SET);
831 pCur->rdr.iIn = 0;
832 pCur->rdr.nIn = 0;
833 }
834 return csvtabNext(pVtabCursor);
835 }
836
837 /*
838 ** Only a forward full table scan is supported. xBestIndex is mostly
839 ** a no-op. If CSVTEST_FIDX is set, then the presence of equality
840 ** constraints lowers the estimated cost, which is fiction, but is useful
841 ** for testing certain kinds of virtual table behavior.
842 */
843 static int csvtabBestIndex(
844 sqlite3_vtab *tab,
845 sqlite3_index_info *pIdxInfo
846 ){
847 pIdxInfo->estimatedCost = 1000000;
848 #ifdef SQLITE_TEST
849 if( (((CsvTable*)tab)->tstFlags & CSVTEST_FIDX)!=0 ){
850 /* The usual (and sensible) case is to always do a full table scan.
851 ** The code in this branch only runs when testflags=1. This code
852 ** generates an artifical and unrealistic plan which is useful
853 ** for testing virtual table logic but is not helpful to real applications.
854 **
855 ** Any ==, LIKE, or GLOB constraint is marked as usable by the virtual
856 ** table (even though it is not) and the cost of running the virtual table
857 ** is reduced from 1 million to just 10. The constraints are *not* marked
858 ** as omittable, however, so the query planner should still generate a
859 ** plan that gives a correct answer, even if they plan is not optimal.
860 */
861 int i;
862 int nConst = 0;
863 for(i=0; i<pIdxInfo->nConstraint; i++){
864 unsigned char op;
865 if( pIdxInfo->aConstraint[i].usable==0 ) continue;
866 op = pIdxInfo->aConstraint[i].op;
867 if( op==SQLITE_INDEX_CONSTRAINT_EQ
868 || op==SQLITE_INDEX_CONSTRAINT_LIKE
869 || op==SQLITE_INDEX_CONSTRAINT_GLOB
870 ){
871 pIdxInfo->estimatedCost = 10;
872 pIdxInfo->aConstraintUsage[nConst].argvIndex = nConst+1;
873 nConst++;
874 }
875 }
876 }
877 #endif
878 return SQLITE_OK;
879 }
880
881
882 static sqlite3_module CsvModule = {
883 0, /* iVersion */
884 csvtabCreate, /* xCreate */
885 csvtabConnect, /* xConnect */
886 csvtabBestIndex, /* xBestIndex */
887 csvtabDisconnect, /* xDisconnect */
888 csvtabDisconnect, /* xDestroy */
889 csvtabOpen, /* xOpen - open a cursor */
890 csvtabClose, /* xClose - close a cursor */
891 csvtabFilter, /* xFilter - configure scan constraints */
892 csvtabNext, /* xNext - advance a cursor */
893 csvtabEof, /* xEof - check for end of scan */
894 csvtabColumn, /* xColumn - read data */
895 csvtabRowid, /* xRowid - read data */
896 0, /* xUpdate */
897 0, /* xBegin */
898 0, /* xSync */
899 0, /* xCommit */
900 0, /* xRollback */
901 0, /* xFindMethod */
902 0, /* xRename */
903 0, /* xSavepoint */
904 0, /* xRelease */
905 0, /* xRollbackTo */
906 0, /* xShadowName */
907 0 /* xIntegrity */
908 };
909
910 #ifdef SQLITE_TEST
911 /*
912 ** For virtual table testing, make a version of the CSV virtual table
913 ** available that has an xUpdate function. But the xUpdate always returns
914 ** SQLITE_READONLY since the CSV file is not really writable.
915 */
916 static int csvtabUpdate(sqlite3_vtab *p,int n,sqlite3_value**v,sqlite3_int64*x){
917 return SQLITE_READONLY;
918 }
919 static sqlite3_module CsvModuleFauxWrite = {
920 0, /* iVersion */
921 csvtabCreate, /* xCreate */
922 csvtabConnect, /* xConnect */
923 csvtabBestIndex, /* xBestIndex */
924 csvtabDisconnect, /* xDisconnect */
925 csvtabDisconnect, /* xDestroy */
926 csvtabOpen, /* xOpen - open a cursor */
927 csvtabClose, /* xClose - close a cursor */
928 csvtabFilter, /* xFilter - configure scan constraints */
929 csvtabNext, /* xNext - advance a cursor */
930 csvtabEof, /* xEof - check for end of scan */
931 csvtabColumn, /* xColumn - read data */
932 csvtabRowid, /* xRowid - read data */
933 csvtabUpdate, /* xUpdate */
934 0, /* xBegin */
935 0, /* xSync */
936 0, /* xCommit */
937 0, /* xRollback */
938 0, /* xFindMethod */
939 0, /* xRename */
940 0, /* xSavepoint */
941 0, /* xRelease */
942 0, /* xRollbackTo */
943 0, /* xShadowName */
944 0 /* xIntegrity */
945 };
946 #endif /* SQLITE_TEST */
947
948 #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */
949
950
951 #ifdef _WIN32
952 __declspec(dllexport)
953 #endif
954 /*
955 ** This routine is called when the extension is loaded. The new
956 ** CSV virtual table module is registered with the calling database
957 ** connection.
958 */
959 int sqlite3_csv_init(
960 sqlite3 *db,
961 char **pzErrMsg,
962 const sqlite3_api_routines *pApi
963 ){
964 #ifndef SQLITE_OMIT_VIRTUALTABLE
965 int rc;
966 SQLITE_EXTENSION_INIT2(pApi);
967 rc = sqlite3_create_module(db, "csv", &CsvModule, 0);
968 #ifdef SQLITE_TEST
969 if( rc==SQLITE_OK ){
970 rc = sqlite3_create_module(db, "csv_wr", &CsvModuleFauxWrite, 0);
971 }
972 #endif
973 return rc;
974 #else
975 return SQLITE_OK;
976 #endif
977 }