Debian Codegraph @

ext/rbu/sqlite3rbu.c

raw
1 /*
2 ** 2014 August 30
3 **
4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
6 **
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
10 **
11 *************************************************************************
12 **
13 **
14 ** OVERVIEW
15 **
16 ** The RBU extension requires that the RBU update be packaged as an
17 ** SQLite database. The tables it expects to find are described in
18 ** sqlite3rbu.h. Essentially, for each table xyz in the target database
19 ** that the user wishes to write to, a corresponding data_xyz table is
20 ** created in the RBU database and populated with one row for each row to
21 ** update, insert or delete from the target table.
22 **
23 ** The update proceeds in three stages:
24 **
25 ** 1) The database is updated. The modified database pages are written
26 ** to a *-oal file. A *-oal file is just like a *-wal file, except
27 ** that it is named "<database>-oal" instead of "<database>-wal".
28 ** Because regular SQLite clients do not look for file named
29 ** "<database>-oal", they go on using the original database in
30 ** rollback mode while the *-oal file is being generated.
31 **
32 ** During this stage RBU does not update the database by writing
33 ** directly to the target tables. Instead it creates "imposter"
34 ** tables using the SQLITE_TESTCTRL_IMPOSTER interface that it uses
35 ** to update each b-tree individually. All updates required by each
36 ** b-tree are completed before moving on to the next, and all
37 ** updates are done in sorted key order.
38 **
39 ** 2) The "<database>-oal" file is moved to the equivalent "<database>-wal"
40 ** location using a call to rename(2). Before doing this the RBU
41 ** module takes an EXCLUSIVE lock on the database file, ensuring
42 ** that there are no other active readers.
43 **
44 ** Once the EXCLUSIVE lock is released, any other database readers
45 ** detect the new *-wal file and read the database in wal mode. At
46 ** this point they see the new version of the database - including
47 ** the updates made as part of the RBU update.
48 **
49 ** 3) The new *-wal file is checkpointed. This proceeds in the same way
50 ** as a regular database checkpoint, except that a single frame is
51 ** checkpointed each time sqlite3rbu_step() is called. If the RBU
52 ** handle is closed before the entire *-wal file is checkpointed,
53 ** the checkpoint progress is saved in the RBU database and the
54 ** checkpoint can be resumed by another RBU client at some point in
55 ** the future.
56 **
57 ** POTENTIAL PROBLEMS
58 **
59 ** The rename() call might not be portable. And RBU is not currently
60 ** syncing the directory after renaming the file.
61 **
62 ** When state is saved, any commit to the *-oal file and the commit to
63 ** the RBU update database are not atomic. So if the power fails at the
64 ** wrong moment they might get out of sync. As the main database will be
65 ** committed before the RBU update database this will likely either just
66 ** pass unnoticed, or result in SQLITE_CONSTRAINT errors (due to UNIQUE
67 ** constraint violations).
68 **
69 ** If some client does modify the target database mid RBU update, or some
70 ** other error occurs, the RBU extension will keep throwing errors. It's
71 ** not really clear how to get out of this state. The system could just
72 ** by delete the RBU update database and *-oal file and have the device
73 ** download the update again and start over.
74 **
75 ** At present, for an UPDATE, both the new.* and old.* records are
76 ** collected in the rbu_xyz table. And for both UPDATEs and DELETEs all
77 ** fields are collected. This means we're probably writing a lot more
78 ** data to disk when saving the state of an ongoing update to the RBU
79 ** update database than is strictly necessary.
80 **
81 */
82
83 #include <assert.h>
84 #include <string.h>
85 #include <stdio.h>
86
87 #include "sqlite3.h"
88
89 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU)
90 #include "sqlite3rbu.h"
91
92 #if defined(_WIN32_WCE)
93 #include "windows.h"
94 #endif
95
96 /* Maximum number of prepared UPDATE statements held by this module */
97 #define SQLITE_RBU_UPDATE_CACHESIZE 16
98
99 /* Delta checksums disabled by default. Compile with -DRBU_ENABLE_DELTA_CKSUM
100 ** to enable checksum verification.
101 */
102 #ifndef RBU_ENABLE_DELTA_CKSUM
103 # define RBU_ENABLE_DELTA_CKSUM 0
104 #endif
105
106 /*
107 ** Swap two objects of type TYPE.
108 */
109 #if !defined(SQLITE_AMALGAMATION)
110 # define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;}
111 #endif
112
113 /*
114 ** Name of the URI option that causes RBU to take an exclusive lock as
115 ** part of the incremental checkpoint operation.
116 */
117 #define RBU_EXCLUSIVE_CHECKPOINT "rbu_exclusive_checkpoint"
118
119
120 /*
121 ** The rbu_state table is used to save the state of a partially applied
122 ** update so that it can be resumed later. The table consists of integer
123 ** keys mapped to values as follows:
124 **
125 ** RBU_STATE_STAGE:
126 ** May be set to integer values 1, 2, 4 or 5. As follows:
127 ** 1: the *-rbu file is currently under construction.
128 ** 2: the *-rbu file has been constructed, but not yet moved
129 ** to the *-wal path.
130 ** 4: the checkpoint is underway.
131 ** 5: the rbu update has been checkpointed.
132 **
133 ** RBU_STATE_TBL:
134 ** Only valid if STAGE==1. The target database name of the table
135 ** currently being written.
136 **
137 ** RBU_STATE_IDX:
138 ** Only valid if STAGE==1. The target database name of the index
139 ** currently being written, or NULL if the main table is currently being
140 ** updated.
141 **
142 ** RBU_STATE_ROW:
143 ** Only valid if STAGE==1. Number of rows already processed for the current
144 ** table/index.
145 **
146 ** RBU_STATE_PROGRESS:
147 ** Trbul number of sqlite3rbu_step() calls made so far as part of this
148 ** rbu update.
149 **
150 ** RBU_STATE_CKPT:
151 ** Valid if STAGE==4. The 64-bit checksum associated with the wal-index
152 ** header created by recovering the *-wal file. This is used to detect
153 ** cases when another client appends frames to the *-wal file in the
154 ** middle of an incremental checkpoint (an incremental checkpoint cannot
155 ** be continued if this happens).
156 **
157 ** RBU_STATE_COOKIE:
158 ** Valid if STAGE==1. The current change-counter cookie value in the
159 ** target db file.
160 **
161 ** RBU_STATE_OALSZ:
162 ** Valid if STAGE==1. The size in bytes of the *-oal file.
163 **
164 ** RBU_STATE_DATATBL:
165 ** Only valid if STAGE==1. The RBU database name of the table
166 ** currently being read.
167 */
168 #define RBU_STATE_STAGE 1
169 #define RBU_STATE_TBL 2
170 #define RBU_STATE_IDX 3
171 #define RBU_STATE_ROW 4
172 #define RBU_STATE_PROGRESS 5
173 #define RBU_STATE_CKPT 6
174 #define RBU_STATE_COOKIE 7
175 #define RBU_STATE_OALSZ 8
176 #define RBU_STATE_PHASEONESTEP 9
177 #define RBU_STATE_DATATBL 10
178
179 #define RBU_STAGE_OAL 1
180 #define RBU_STAGE_MOVE 2
181 #define RBU_STAGE_CAPTURE 3
182 #define RBU_STAGE_CKPT 4
183 #define RBU_STAGE_DONE 5
184
185
186 #define RBU_CREATE_STATE \
187 "CREATE TABLE IF NOT EXISTS %s.rbu_state(k INTEGER PRIMARY KEY, v)"
188
189 typedef struct RbuFrame RbuFrame;
190 typedef struct RbuObjIter RbuObjIter;
191 typedef struct RbuState RbuState;
192 typedef struct RbuSpan RbuSpan;
193 typedef struct rbu_vfs rbu_vfs;
194 typedef struct rbu_file rbu_file;
195 typedef struct RbuUpdateStmt RbuUpdateStmt;
196
197 #if !defined(SQLITE_AMALGAMATION)
198 typedef unsigned int u32;
199 typedef unsigned short u16;
200 typedef unsigned char u8;
201 typedef sqlite3_int64 i64;
202 typedef sqlite3_uint64 u64;
203 #endif
204
205 /*
206 ** These values must match the values defined in wal.c for the equivalent
207 ** locks. These are not magic numbers as they are part of the SQLite file
208 ** format.
209 */
210 #define WAL_LOCK_WRITE 0
211 #define WAL_LOCK_CKPT 1
212 #define WAL_LOCK_READ0 3
213
214 #define SQLITE_FCNTL_RBUCNT 5149216
215
216 /*
217 ** A structure to store values read from the rbu_state table in memory.
218 */
219 struct RbuState {
220 int eStage;
221 char *zTbl;
222 char *zDataTbl;
223 char *zIdx;
224 i64 iWalCksum;
225 int nRow;
226 i64 nProgress;
227 u32 iCookie;
228 i64 iOalSz;
229 i64 nPhaseOneStep;
230 };
231
232 struct RbuUpdateStmt {
233 char *zMask; /* Copy of update mask used with pUpdate */
234 sqlite3_stmt *pUpdate; /* Last update statement (or NULL) */
235 RbuUpdateStmt *pNext;
236 };
237
238 struct RbuSpan {
239 const char *zSpan;
240 int nSpan;
241 };
242
243 /*
244 ** An iterator of this type is used to iterate through all objects in
245 ** the target database that require updating. For each such table, the
246 ** iterator visits, in order:
247 **
248 ** * the table itself,
249 ** * each index of the table (zero or more points to visit), and
250 ** * a special "cleanup table" state.
251 **
252 ** abIndexed:
253 ** If the table has no indexes on it, abIndexed is set to NULL. Otherwise,
254 ** it points to an array of flags nTblCol elements in size. The flag is
255 ** set for each column that is either a part of the PK or a part of an
256 ** index. Or clear otherwise.
257 **
258 ** If there are one or more partial indexes on the table, all fields of
259 ** this array set set to 1. This is because in that case, the module has
260 ** no way to tell which fields will be required to add and remove entries
261 ** from the partial indexes.
262 **
263 */
264 struct RbuObjIter {
265 sqlite3_stmt *pTblIter; /* Iterate through tables */
266 sqlite3_stmt *pIdxIter; /* Index iterator */
267 int nTblCol; /* Size of azTblCol[] array */
268 char **azTblCol; /* Array of unquoted target column names */
269 char **azTblType; /* Array of target column types */
270 int *aiSrcOrder; /* src table col -> target table col */
271 u8 *abTblPk; /* Array of flags, set on target PK columns */
272 u8 *abNotNull; /* Array of flags, set on NOT NULL columns */
273 u8 *abIndexed; /* Array of flags, set on indexed & PK cols */
274 int eType; /* Table type - an RBU_PK_XXX value */
275
276 /* Output variables. zTbl==0 implies EOF. */
277 int bCleanup; /* True in "cleanup" state */
278 const char *zTbl; /* Name of target db table */
279 const char *zDataTbl; /* Name of rbu db table (or null) */
280 const char *zIdx; /* Name of target db index (or null) */
281 int iTnum; /* Root page of current object */
282 int iPkTnum; /* If eType==EXTERNAL, root of PK index */
283 int bUnique; /* Current index is unique */
284 int nIndex; /* Number of aux. indexes on table zTbl */
285
286 /* Statements created by rbuObjIterPrepareAll() */
287 int nCol; /* Number of columns in current object */
288 sqlite3_stmt *pSelect; /* Source data */
289 sqlite3_stmt *pInsert; /* Statement for INSERT operations */
290 sqlite3_stmt *pDelete; /* Statement for DELETE ops */
291 sqlite3_stmt *pTmpInsert; /* Insert into rbu_tmp_$zDataTbl */
292 int nIdxCol;
293 RbuSpan *aIdxCol;
294 char *zIdxSql;
295
296 /* Last UPDATE used (for PK b-tree updates only), or NULL. */
297 RbuUpdateStmt *pRbuUpdate;
298 };
299
300 /*
301 ** Values for RbuObjIter.eType
302 **
303 ** 0: Table does not exist (error)
304 ** 1: Table has an implicit rowid.
305 ** 2: Table has an explicit IPK column.
306 ** 3: Table has an external PK index.
307 ** 4: Table is WITHOUT ROWID.
308 ** 5: Table is a virtual table.
309 */
310 #define RBU_PK_NOTABLE 0
311 #define RBU_PK_NONE 1
312 #define RBU_PK_IPK 2
313 #define RBU_PK_EXTERNAL 3
314 #define RBU_PK_WITHOUT_ROWID 4
315 #define RBU_PK_VTAB 5
316
317
318 /*
319 ** Within the RBU_STAGE_OAL stage, each call to sqlite3rbu_step() performs
320 ** one of the following operations.
321 */
322 #define RBU_INSERT 1 /* Insert on a main table b-tree */
323 #define RBU_DELETE 2 /* Delete a row from a main table b-tree */
324 #define RBU_REPLACE 3 /* Delete and then insert a row */
325 #define RBU_IDX_DELETE 4 /* Delete a row from an aux. index b-tree */
326 #define RBU_IDX_INSERT 5 /* Insert on an aux. index b-tree */
327
328 #define RBU_UPDATE 6 /* Update a row in a main table b-tree */
329
330 /*
331 ** A single step of an incremental checkpoint - frame iWalFrame of the wal
332 ** file should be copied to page iDbPage of the database file.
333 */
334 struct RbuFrame {
335 u32 iDbPage;
336 u32 iWalFrame;
337 };
338
339 #ifndef UNUSED_PARAMETER
340 /*
341 ** The following macros are used to suppress compiler warnings and to
342 ** make it clear to human readers when a function parameter is deliberately
343 ** left unused within the body of a function. This usually happens when
344 ** a function is called via a function pointer. For example the
345 ** implementation of an SQL aggregate step callback may not use the
346 ** parameter indicating the number of arguments passed to the aggregate,
347 ** if it knows that this is enforced elsewhere.
348 **
349 ** When a function parameter is not used at all within the body of a function,
350 ** it is generally named "NotUsed" or "NotUsed2" to make things even clearer.
351 ** However, these macros may also be used to suppress warnings related to
352 ** parameters that may or may not be used depending on compilation options.
353 ** For example those parameters only used in assert() statements. In these
354 ** cases the parameters are named as per the usual conventions.
355 */
356 #define UNUSED_PARAMETER(x) (void)(x)
357 #define UNUSED_PARAMETER2(x,y) UNUSED_PARAMETER(x),UNUSED_PARAMETER(y)
358 #endif
359
360 /*
361 ** RBU handle.
362 **
363 ** nPhaseOneStep:
364 ** If the RBU database contains an rbu_count table, this value is set to
365 ** a running estimate of the number of b-tree operations required to
366 ** finish populating the *-oal file. This allows the sqlite3_bp_progress()
367 ** API to calculate the permyriadage progress of populating the *-oal file
368 ** using the formula:
369 **
370 ** permyriadage = (10000 * nProgress) / nPhaseOneStep
371 **
372 ** nPhaseOneStep is initialized to the sum of:
373 **
374 ** nRow * (nIndex + 1)
375 **
376 ** for all source tables in the RBU database, where nRow is the number
377 ** of rows in the source table and nIndex the number of indexes on the
378 ** corresponding target database table.
379 **
380 ** This estimate is accurate if the RBU update consists entirely of
381 ** INSERT operations. However, it is inaccurate if:
382 **
383 ** * the RBU update contains any UPDATE operations. If the PK specified
384 ** for an UPDATE operation does not exist in the target table, then
385 ** no b-tree operations are required on index b-trees. Or if the
386 ** specified PK does exist, then (nIndex*2) such operations are
387 ** required (one delete and one insert on each index b-tree).
388 **
389 ** * the RBU update contains any DELETE operations for which the specified
390 ** PK does not exist. In this case no operations are required on index
391 ** b-trees.
392 **
393 ** * the RBU update contains REPLACE operations. These are similar to
394 ** UPDATE operations.
395 **
396 ** nPhaseOneStep is updated to account for the conditions above during the
397 ** first pass of each source table. The updated nPhaseOneStep value is
398 ** stored in the rbu_state table if the RBU update is suspended.
399 */
400 struct sqlite3rbu {
401 int eStage; /* Value of RBU_STATE_STAGE field */
402 sqlite3 *dbMain; /* target database handle */
403 sqlite3 *dbRbu; /* rbu database handle */
404 char *zTarget; /* Path to target db */
405 char *zRbu; /* Path to rbu db */
406 char *zState; /* Path to state db (or NULL if zRbu) */
407 char zStateDb[5]; /* Db name for state ("stat" or "main") */
408 int rc; /* Value returned by last rbu_step() call */
409 char *zErrmsg; /* Error message if rc!=SQLITE_OK */
410 int nStep; /* Rows processed for current object */
411 sqlite3_int64 nProgress; /* Rows processed for all objects */
412 RbuObjIter objiter; /* Iterator for skipping through tbl/idx */
413 const char *zVfsName; /* Name of automatically created rbu vfs */
414 rbu_file *pTargetFd; /* File handle open on target db */
415 int nPagePerSector; /* Pages per sector for pTargetFd */
416 i64 iOalSz;
417 i64 nPhaseOneStep;
418 void *pRenameArg;
419 int (*xRename)(void*, const char*, const char*);
420
421 /* The following state variables are used as part of the incremental
422 ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding
423 ** function rbuSetupCheckpoint() for details. */
424 u32 iMaxFrame; /* Largest iWalFrame value in aFrame[] */
425 u32 mLock;
426 int nFrame; /* Entries in aFrame[] array */
427 int nFrameAlloc; /* Allocated size of aFrame[] array */
428 RbuFrame *aFrame;
429 int pgsz;
430 u8 *aBuf;
431 i64 iWalCksum;
432 i64 szTemp; /* Current size of all temp files in use */
433 i64 szTempLimit; /* Total size limit for temp files */
434
435 /* Used in RBU vacuum mode only */
436 int nRbu; /* Number of RBU VFS in the stack */
437 rbu_file *pRbuFd; /* Fd for main db of dbRbu */
438 };
439
440 /*
441 ** An rbu VFS is implemented using an instance of this structure.
442 **
443 ** Variable pRbu is only non-NULL for automatically created RBU VFS objects.
444 ** It is NULL for RBU VFS objects created explicitly using
445 ** sqlite3rbu_create_vfs(). It is used to track the total amount of temp
446 ** space used by the RBU handle.
447 */
448 struct rbu_vfs {
449 sqlite3_vfs base; /* rbu VFS shim methods */
450 sqlite3_vfs *pRealVfs; /* Underlying VFS */
451 sqlite3_mutex *mutex; /* Mutex to protect pMain */
452 sqlite3rbu *pRbu; /* Owner RBU object */
453 rbu_file *pMain; /* List of main db files */
454 rbu_file *pMainRbu; /* List of main db files with pRbu!=0 */
455 };
456
457 /*
458 ** Each file opened by an rbu VFS is represented by an instance of
459 ** the following structure.
460 **
461 ** If this is a temporary file (pRbu!=0 && flags&DELETE_ON_CLOSE), variable
462 ** "sz" is set to the current size of the database file.
463 */
464 struct rbu_file {
465 sqlite3_file base; /* sqlite3_file methods */
466 sqlite3_file *pReal; /* Underlying file handle */
467 rbu_vfs *pRbuVfs; /* Pointer to the rbu_vfs object */
468 sqlite3rbu *pRbu; /* Pointer to rbu object (rbu target only) */
469 i64 sz; /* Size of file in bytes (temp only) */
470
471 int openFlags; /* Flags this file was opened with */
472 u32 iCookie; /* Cookie value for main db files */
473 u8 iWriteVer; /* "write-version" value for main db files */
474 u8 bNolock; /* True to fail EXCLUSIVE locks */
475
476 int nShm; /* Number of entries in apShm[] array */
477 char **apShm; /* Array of mmap'd *-shm regions */
478 char *zDel; /* Delete this when closing file */
479
480 const char *zWal; /* Wal filename for this main db file */
481 rbu_file *pWalFd; /* Wal file descriptor for this main db */
482 rbu_file *pMainNext; /* Next MAIN_DB file */
483 rbu_file *pMainRbuNext; /* Next MAIN_DB file with pRbu!=0 */
484 };
485
486 /*
487 ** True for an RBU vacuum handle, or false otherwise.
488 */
489 #define rbuIsVacuum(p) ((p)->zTarget==0)
490
491
492 /*************************************************************************
493 ** The following three functions, found below:
494 **
495 ** rbuDeltaGetInt()
496 ** rbuDeltaChecksum()
497 ** rbuDeltaApply()
498 **
499 ** are lifted from the fossil source code (http://fossil-scm.org). They
500 ** are used to implement the scalar SQL function rbu_fossil_delta().
501 */
502
503 /*
504 ** Read bytes from *pz and convert them into a positive integer. When
505 ** finished, leave *pz pointing to the first character past the end of
506 ** the integer. The *pLen parameter holds the length of the string
507 ** in *pz and is decremented once for each character in the integer.
508 */
509 static unsigned int rbuDeltaGetInt(const char **pz, int *pLen){
510 static const signed char zValue[] = {
511 -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
512 -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
513 -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
514 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1,
515 -1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
516 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, -1, -1, -1, -1, 36,
517 -1, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51,
518 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, -1, -1, -1, 63, -1,
519 };
520 unsigned int v = 0;
521 int c;
522 unsigned char *z = (unsigned char*)*pz;
523 unsigned char *zStart = z;
524 while( (c = zValue[0x7f&*(z++)])>=0 ){
525 v = (v<<6) + c;
526 }
527 z--;
528 *pLen -= (int)(z - zStart);
529 *pz = (char*)z;
530 return v;
531 }
532
533 #if RBU_ENABLE_DELTA_CKSUM
534 /*
535 ** Compute a 32-bit checksum on the N-byte buffer. Return the result.
536 */
537 static unsigned int rbuDeltaChecksum(const char *zIn, size_t N){
538 const unsigned char *z = (const unsigned char *)zIn;
539 unsigned sum0 = 0;
540 unsigned sum1 = 0;
541 unsigned sum2 = 0;
542 unsigned sum3 = 0;
543 while(N >= 16){
544 sum0 += ((unsigned)z[0] + z[4] + z[8] + z[12]);
545 sum1 += ((unsigned)z[1] + z[5] + z[9] + z[13]);
546 sum2 += ((unsigned)z[2] + z[6] + z[10]+ z[14]);
547 sum3 += ((unsigned)z[3] + z[7] + z[11]+ z[15]);
548 z += 16;
549 N -= 16;
550 }
551 while(N >= 4){
552 sum0 += z[0];
553 sum1 += z[1];
554 sum2 += z[2];
555 sum3 += z[3];
556 z += 4;
557 N -= 4;
558 }
559 sum3 += (sum2 << 8) + (sum1 << 16) + (sum0 << 24);
560 switch(N){
561 case 3: sum3 += (z[2] << 8);
562 case 2: sum3 += (z[1] << 16);
563 case 1: sum3 += (z[0] << 24);
564 default: ;
565 }
566 return sum3;
567 }
568 #endif
569
570 /*
571 ** Apply a delta.
572 **
573 ** The output buffer should be big enough to hold the whole output
574 ** file and a NUL terminator at the end. The delta_output_size()
575 ** routine will determine this size for you.
576 **
577 ** The delta string should be null-terminated. But the delta string
578 ** may contain embedded NUL characters (if the input and output are
579 ** binary files) so we also have to pass in the length of the delta in
580 ** the lenDelta parameter.
581 **
582 ** This function returns the size of the output file in bytes (excluding
583 ** the final NUL terminator character). Except, if the delta string is
584 ** malformed or intended for use with a source file other than zSrc,
585 ** then this routine returns -1.
586 **
587 ** Refer to the delta_create() documentation above for a description
588 ** of the delta file format.
589 */
590 static int rbuDeltaApply(
591 const char *zSrc, /* The source or pattern file */
592 int lenSrc, /* Length of the source file */
593 const char *zDelta, /* Delta to apply to the pattern */
594 int lenDelta, /* Length of the delta */
595 char *zOut /* Write the output into this preallocated buffer */
596 ){
597 unsigned int limit;
598 unsigned int total = 0;
599 #if RBU_ENABLE_DELTA_CKSUM
600 char *zOrigOut = zOut;
601 #endif
602
603 limit = rbuDeltaGetInt(&zDelta, &lenDelta);
604 if( *zDelta!='\n' ){
605 /* ERROR: size integer not terminated by "\n" */
606 return -1;
607 }
608 zDelta++; lenDelta--;
609 while( *zDelta && lenDelta>0 ){
610 unsigned int cnt, ofst;
611 cnt = rbuDeltaGetInt(&zDelta, &lenDelta);
612 switch( zDelta[0] ){
613 case '@': {
614 zDelta++; lenDelta--;
615 ofst = rbuDeltaGetInt(&zDelta, &lenDelta);
616 if( lenDelta>0 && zDelta[0]!=',' ){
617 /* ERROR: copy command not terminated by ',' */
618 return -1;
619 }
620 zDelta++; lenDelta--;
621 total += cnt;
622 if( total>limit ){
623 /* ERROR: copy exceeds output file size */
624 return -1;
625 }
626 if( (u64)ofst+(u64)cnt > (u64)lenSrc ){
627 /* ERROR: copy extends past end of input */
628 return -1;
629 }
630 memcpy(zOut, &zSrc[ofst], cnt);
631 zOut += cnt;
632 break;
633 }
634 case ':': {
635 zDelta++; lenDelta--;
636 total += cnt;
637 if( total>limit ){
638 /* ERROR: insert command gives an output larger than predicted */
639 return -1;
640 }
641 if( (int)cnt>lenDelta ){
642 /* ERROR: insert count exceeds size of delta */
643 return -1;
644 }
645 memcpy(zOut, zDelta, cnt);
646 zOut += cnt;
647 zDelta += cnt;
648 lenDelta -= cnt;
649 break;
650 }
651 case ';': {
652 zDelta++; lenDelta--;
653 zOut[0] = 0;
654 #if RBU_ENABLE_DELTA_CKSUM
655 if( cnt!=rbuDeltaChecksum(zOrigOut, total) ){
656 /* ERROR: bad checksum */
657 return -1;
658 }
659 #endif
660 if( total!=limit ){
661 /* ERROR: generated size does not match predicted size */
662 return -1;
663 }
664 return total;
665 }
666 default: {
667 /* ERROR: unknown delta operator */
668 return -1;
669 }
670 }
671 }
672 /* ERROR: unterminated delta */
673 return -1;
674 }
675
676 static int rbuDeltaOutputSize(const char *zDelta, int lenDelta){
677 int size;
678 size = rbuDeltaGetInt(&zDelta, &lenDelta);
679 if( *zDelta!='\n' ){
680 /* ERROR: size integer not terminated by "\n" */
681 return -1;
682 }
683 return size;
684 }
685
686 /*
687 ** End of code taken from fossil.
688 *************************************************************************/
689
690 /*
691 ** Implementation of SQL scalar function rbu_fossil_delta().
692 **
693 ** This function applies a fossil delta patch to a blob. Exactly two
694 ** arguments must be passed to this function. The first is the blob to
695 ** patch and the second the patch to apply. If no error occurs, this
696 ** function returns the patched blob.
697 */
698 static void rbuFossilDeltaFunc(
699 sqlite3_context *context,
700 int argc,
701 sqlite3_value **argv
702 ){
703 const char *aDelta;
704 int nDelta;
705 const char *aOrig;
706 int nOrig;
707
708 int nOut;
709 int nOut2;
710 char *aOut;
711
712 assert( argc==2 );
713 UNUSED_PARAMETER(argc);
714
715 nOrig = sqlite3_value_bytes(argv[0]);
716 aOrig = (const char*)sqlite3_value_blob(argv[0]);
717 nDelta = sqlite3_value_bytes(argv[1]);
718 aDelta = (const char*)sqlite3_value_blob(argv[1]);
719
720 /* Figure out the size of the output */
721 nOut = rbuDeltaOutputSize(aDelta, nDelta);
722 if( nOut<0 ){
723 sqlite3_result_error(context, "corrupt fossil delta", -1);
724 return;
725 }
726
727 aOut = sqlite3_malloc(nOut+1);
728 if( aOut==0 ){
729 sqlite3_result_error_nomem(context);
730 }else{
731 nOut2 = rbuDeltaApply(aOrig, nOrig, aDelta, nDelta, aOut);
732 if( nOut2!=nOut ){
733 sqlite3_free(aOut);
734 sqlite3_result_error(context, "corrupt fossil delta", -1);
735 }else{
736 sqlite3_result_blob(context, aOut, nOut, sqlite3_free);
737 }
738 }
739 }
740
741
742 /*
743 ** Prepare the SQL statement in buffer zSql against database handle db.
744 ** If successful, set *ppStmt to point to the new statement and return
745 ** SQLITE_OK.
746 **
747 ** Otherwise, if an error does occur, set *ppStmt to NULL and return
748 ** an SQLite error code. Additionally, set output variable *pzErrmsg to
749 ** point to a buffer containing an error message. It is the responsibility
750 ** of the caller to (eventually) free this buffer using sqlite3_free().
751 */
752 static int prepareAndCollectError(
753 sqlite3 *db,
754 sqlite3_stmt **ppStmt,
755 char **pzErrmsg,
756 const char *zSql
757 ){
758 int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
759 if( rc!=SQLITE_OK ){
760 *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
761 *ppStmt = 0;
762 }
763 return rc;
764 }
765
766 /*
767 ** Reset the SQL statement passed as the first argument. Return a copy
768 ** of the value returned by sqlite3_reset().
769 **
770 ** If an error has occurred, then set *pzErrmsg to point to a buffer
771 ** containing an error message. It is the responsibility of the caller
772 ** to eventually free this buffer using sqlite3_free().
773 */
774 static int resetAndCollectError(sqlite3_stmt *pStmt, char **pzErrmsg){
775 int rc = sqlite3_reset(pStmt);
776 if( rc!=SQLITE_OK ){
777 *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(sqlite3_db_handle(pStmt)));
778 }
779 return rc;
780 }
781
782 /*
783 ** Unless it is NULL, argument zSql points to a buffer allocated using
784 ** sqlite3_malloc containing an SQL statement. This function prepares the SQL
785 ** statement against database db and frees the buffer. If statement
786 ** compilation is successful, *ppStmt is set to point to the new statement
787 ** handle and SQLITE_OK is returned.
788 **
789 ** Otherwise, if an error occurs, *ppStmt is set to NULL and an error code
790 ** returned. In this case, *pzErrmsg may also be set to point to an error
791 ** message. It is the responsibility of the caller to free this error message
792 ** buffer using sqlite3_free().
793 **
794 ** If argument zSql is NULL, this function assumes that an OOM has occurred.
795 ** In this case SQLITE_NOMEM is returned and *ppStmt set to NULL.
796 */
797 static int prepareFreeAndCollectError(
798 sqlite3 *db,
799 sqlite3_stmt **ppStmt,
800 char **pzErrmsg,
801 char *zSql
802 ){
803 int rc;
804 assert( *pzErrmsg==0 );
805 if( zSql==0 ){
806 rc = SQLITE_NOMEM;
807 *ppStmt = 0;
808 }else{
809 rc = prepareAndCollectError(db, ppStmt, pzErrmsg, zSql);
810 sqlite3_free(zSql);
811 }
812 return rc;
813 }
814
815 /*
816 ** Free the RbuObjIter.azTblCol[] and RbuObjIter.abTblPk[] arrays allocated
817 ** by an earlier call to rbuObjIterCacheTableInfo().
818 */
819 static void rbuObjIterFreeCols(RbuObjIter *pIter){
820 int i;
821 for(i=0; i<pIter->nTblCol; i++){
822 sqlite3_free(pIter->azTblCol[i]);
823 sqlite3_free(pIter->azTblType[i]);
824 }
825 sqlite3_free(pIter->azTblCol);
826 pIter->azTblCol = 0;
827 pIter->azTblType = 0;
828 pIter->aiSrcOrder = 0;
829 pIter->abTblPk = 0;
830 pIter->abNotNull = 0;
831 pIter->nTblCol = 0;
832 pIter->eType = 0; /* Invalid value */
833 }
834
835 /*
836 ** Finalize all statements and free all allocations that are specific to
837 ** the current object (table/index pair).
838 */
839 static void rbuObjIterClearStatements(RbuObjIter *pIter){
840 RbuUpdateStmt *pUp;
841
842 sqlite3_finalize(pIter->pSelect);
843 sqlite3_finalize(pIter->pInsert);
844 sqlite3_finalize(pIter->pDelete);
845 sqlite3_finalize(pIter->pTmpInsert);
846 pUp = pIter->pRbuUpdate;
847 while( pUp ){
848 RbuUpdateStmt *pTmp = pUp->pNext;
849 sqlite3_finalize(pUp->pUpdate);
850 sqlite3_free(pUp);
851 pUp = pTmp;
852 }
853 sqlite3_free(pIter->aIdxCol);
854 sqlite3_free(pIter->zIdxSql);
855
856 pIter->pSelect = 0;
857 pIter->pInsert = 0;
858 pIter->pDelete = 0;
859 pIter->pRbuUpdate = 0;
860 pIter->pTmpInsert = 0;
861 pIter->nCol = 0;
862 pIter->nIdxCol = 0;
863 pIter->aIdxCol = 0;
864 pIter->zIdxSql = 0;
865 }
866
867 /*
868 ** Clean up any resources allocated as part of the iterator object passed
869 ** as the only argument.
870 */
871 static void rbuObjIterFinalize(RbuObjIter *pIter){
872 rbuObjIterClearStatements(pIter);
873 sqlite3_finalize(pIter->pTblIter);
874 sqlite3_finalize(pIter->pIdxIter);
875 rbuObjIterFreeCols(pIter);
876 memset(pIter, 0, sizeof(RbuObjIter));
877 }
878
879 /*
880 ** Advance the iterator to the next position.
881 **
882 ** If no error occurs, SQLITE_OK is returned and the iterator is left
883 ** pointing to the next entry. Otherwise, an error code and message is
884 ** left in the RBU handle passed as the first argument. A copy of the
885 ** error code is returned.
886 */
887 static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){
888 int rc = p->rc;
889 if( rc==SQLITE_OK ){
890
891 /* Free any SQLite statements used while processing the previous object */
892 rbuObjIterClearStatements(pIter);
893 if( pIter->zIdx==0 ){
894 rc = sqlite3_exec(p->dbMain,
895 "DROP TRIGGER IF EXISTS temp.rbu_insert_tr;"
896 "DROP TRIGGER IF EXISTS temp.rbu_update1_tr;"
897 "DROP TRIGGER IF EXISTS temp.rbu_update2_tr;"
898 "DROP TRIGGER IF EXISTS temp.rbu_delete_tr;"
899 , 0, 0, &p->zErrmsg
900 );
901 }
902
903 if( rc==SQLITE_OK ){
904 if( pIter->bCleanup ){
905 rbuObjIterFreeCols(pIter);
906 pIter->bCleanup = 0;
907 rc = sqlite3_step(pIter->pTblIter);
908 if( rc!=SQLITE_ROW ){
909 rc = resetAndCollectError(pIter->pTblIter, &p->zErrmsg);
910 pIter->zTbl = 0;
911 pIter->zDataTbl = 0;
912 }else{
913 pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0);
914 pIter->zDataTbl = (const char*)sqlite3_column_text(pIter->pTblIter,1);
915 rc = (pIter->zDataTbl && pIter->zTbl) ? SQLITE_OK : SQLITE_NOMEM;
916 }
917 }else{
918 if( pIter->zIdx==0 ){
919 sqlite3_stmt *pIdx = pIter->pIdxIter;
920 rc = sqlite3_bind_text(pIdx, 1, pIter->zTbl, -1, SQLITE_STATIC);
921 }
922 if( rc==SQLITE_OK ){
923 rc = sqlite3_step(pIter->pIdxIter);
924 if( rc!=SQLITE_ROW ){
925 rc = resetAndCollectError(pIter->pIdxIter, &p->zErrmsg);
926 pIter->bCleanup = 1;
927 pIter->zIdx = 0;
928 }else{
929 pIter->zIdx = (const char*)sqlite3_column_text(pIter->pIdxIter, 0);
930 pIter->iTnum = sqlite3_column_int(pIter->pIdxIter, 1);
931 pIter->bUnique = sqlite3_column_int(pIter->pIdxIter, 2);
932 rc = pIter->zIdx ? SQLITE_OK : SQLITE_NOMEM;
933 }
934 }
935 }
936 }
937 }
938
939 if( rc!=SQLITE_OK ){
940 rbuObjIterFinalize(pIter);
941 p->rc = rc;
942 }
943 return rc;
944 }
945
946
947 /*
948 ** The implementation of the rbu_target_name() SQL function. This function
949 ** accepts one or two arguments. The first argument is the name of a table -
950 ** the name of a table in the RBU database. The second, if it is present, is 1
951 ** for a view or 0 for a table.
952 **
953 ** For a non-vacuum RBU handle, if the table name matches the pattern:
954 **
955 ** data[0-9]_<name>
956 **
957 ** where <name> is any sequence of 1 or more characters, <name> is returned.
958 ** Otherwise, if the only argument does not match the above pattern, an SQL
959 ** NULL is returned.
960 **
961 ** "data_t1" -> "t1"
962 ** "data0123_t2" -> "t2"
963 ** "dataAB_t3" -> NULL
964 **
965 ** For an rbu vacuum handle, a copy of the first argument is returned if
966 ** the second argument is either missing or 0 (not a view).
967 */
968 static void rbuTargetNameFunc(
969 sqlite3_context *pCtx,
970 int argc,
971 sqlite3_value **argv
972 ){
973 sqlite3rbu *p = sqlite3_user_data(pCtx);
974 const char *zIn;
975 assert( argc==1 || argc==2 );
976
977 zIn = (const char*)sqlite3_value_text(argv[0]);
978 if( zIn ){
979 if( rbuIsVacuum(p) ){
980 assert( argc==2 || argc==1 );
981 if( argc==1 || 0==sqlite3_value_int(argv[1]) ){
982 sqlite3_result_text(pCtx, zIn, -1, SQLITE_STATIC);
983 }
984 }else{
985 if( strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){
986 int i;
987 for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++);
988 if( zIn[i]=='_' && zIn[i+1] ){
989 sqlite3_result_text(pCtx, &zIn[i+1], -1, SQLITE_STATIC);
990 }
991 }
992 }
993 }
994 }
995
996 /*
997 ** Initialize the iterator structure passed as the second argument.
998 **
999 ** If no error occurs, SQLITE_OK is returned and the iterator is left
1000 ** pointing to the first entry. Otherwise, an error code and message is
1001 ** left in the RBU handle passed as the first argument. A copy of the
1002 ** error code is returned.
1003 */
1004 static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){
1005 int rc;
1006 memset(pIter, 0, sizeof(RbuObjIter));
1007
1008 rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg,
1009 sqlite3_mprintf(
1010 "SELECT rbu_target_name(name, type='view') AS target, name "
1011 "FROM sqlite_schema "
1012 "WHERE type IN ('table', 'view') AND target IS NOT NULL "
1013 " %s "
1014 "ORDER BY name"
1015 , rbuIsVacuum(p) ? "AND rootpage!=0 AND rootpage IS NOT NULL" : ""));
1016
1017 if( rc==SQLITE_OK ){
1018 rc = prepareAndCollectError(p->dbMain, &pIter->pIdxIter, &p->zErrmsg,
1019 "SELECT name, rootpage, sql IS NULL OR substr(8, 6)=='UNIQUE' "
1020 " FROM main.sqlite_schema "
1021 " WHERE type='index' AND tbl_name = ?"
1022 );
1023 }
1024
1025 pIter->bCleanup = 1;
1026 p->rc = rc;
1027 return rbuObjIterNext(p, pIter);
1028 }
1029
1030 /*
1031 ** This is a wrapper around "sqlite3_mprintf(zFmt, ...)". If an OOM occurs,
1032 ** an error code is stored in the RBU handle passed as the first argument.
1033 **
1034 ** If an error has already occurred (p->rc is already set to something other
1035 ** than SQLITE_OK), then this function returns NULL without modifying the
1036 ** stored error code. In this case it still calls sqlite3_free() on any
1037 ** printf() parameters associated with %z conversions.
1038 */
1039 static char *rbuMPrintf(sqlite3rbu *p, const char *zFmt, ...){
1040 char *zSql = 0;
1041 va_list ap;
1042 va_start(ap, zFmt);
1043 zSql = sqlite3_vmprintf(zFmt, ap);
1044 if( p->rc==SQLITE_OK ){
1045 if( zSql==0 ) p->rc = SQLITE_NOMEM;
1046 }else{
1047 sqlite3_free(zSql);
1048 zSql = 0;
1049 }
1050 va_end(ap);
1051 return zSql;
1052 }
1053
1054 /*
1055 ** Argument zFmt is a sqlite3_mprintf() style format string. The trailing
1056 ** arguments are the usual subsitution values. This function performs
1057 ** the printf() style substitutions and executes the result as an SQL
1058 ** statement on the RBU handles database.
1059 **
1060 ** If an error occurs, an error code and error message is stored in the
1061 ** RBU handle. If an error has already occurred when this function is
1062 ** called, it is a no-op.
1063 */
1064 static int rbuMPrintfExec(sqlite3rbu *p, sqlite3 *db, const char *zFmt, ...){
1065 va_list ap;
1066 char *zSql;
1067 va_start(ap, zFmt);
1068 zSql = sqlite3_vmprintf(zFmt, ap);
1069 if( p->rc==SQLITE_OK ){
1070 if( zSql==0 ){
1071 p->rc = SQLITE_NOMEM;
1072 }else{
1073 p->rc = sqlite3_exec(db, zSql, 0, 0, &p->zErrmsg);
1074 }
1075 }
1076 sqlite3_free(zSql);
1077 va_end(ap);
1078 return p->rc;
1079 }
1080
1081 /*
1082 ** Attempt to allocate and return a pointer to a zeroed block of nByte
1083 ** bytes.
1084 **
1085 ** If an error (i.e. an OOM condition) occurs, return NULL and leave an
1086 ** error code in the rbu handle passed as the first argument. Or, if an
1087 ** error has already occurred when this function is called, return NULL
1088 ** immediately without attempting the allocation or modifying the stored
1089 ** error code.
1090 */
1091 static void *rbuMalloc(sqlite3rbu *p, sqlite3_int64 nByte){
1092 void *pRet = 0;
1093 if( p->rc==SQLITE_OK ){
1094 assert( nByte>0 );
1095 pRet = sqlite3_malloc64(nByte);
1096 if( pRet==0 ){
1097 p->rc = SQLITE_NOMEM;
1098 }else{
1099 memset(pRet, 0, nByte);
1100 }
1101 }
1102 return pRet;
1103 }
1104
1105
1106 /*
1107 ** Allocate and zero the pIter->azTblCol[] and abTblPk[] arrays so that
1108 ** there is room for at least nCol elements. If an OOM occurs, store an
1109 ** error code in the RBU handle passed as the first argument.
1110 */
1111 static void rbuAllocateIterArrays(sqlite3rbu *p, RbuObjIter *pIter, int nCol){
1112 sqlite3_int64 nByte = (2*sizeof(char*) + sizeof(int) + 3*sizeof(u8)) * nCol;
1113 char **azNew;
1114
1115 azNew = (char**)rbuMalloc(p, nByte);
1116 if( azNew ){
1117 pIter->azTblCol = azNew;
1118 pIter->azTblType = &azNew[nCol];
1119 pIter->aiSrcOrder = (int*)&pIter->azTblType[nCol];
1120 pIter->abTblPk = (u8*)&pIter->aiSrcOrder[nCol];
1121 pIter->abNotNull = (u8*)&pIter->abTblPk[nCol];
1122 pIter->abIndexed = (u8*)&pIter->abNotNull[nCol];
1123 }
1124 }
1125
1126 /*
1127 ** The first argument must be a nul-terminated string. This function
1128 ** returns a copy of the string in memory obtained from sqlite3_malloc().
1129 ** It is the responsibility of the caller to eventually free this memory
1130 ** using sqlite3_free().
1131 **
1132 ** If an OOM condition is encountered when attempting to allocate memory,
1133 ** output variable (*pRc) is set to SQLITE_NOMEM before returning. Otherwise,
1134 ** if the allocation succeeds, (*pRc) is left unchanged.
1135 */
1136 static char *rbuStrndup(const char *zStr, int *pRc){
1137 char *zRet = 0;
1138
1139 if( *pRc==SQLITE_OK ){
1140 if( zStr ){
1141 size_t nCopy = strlen(zStr) + 1;
1142 zRet = (char*)sqlite3_malloc64(nCopy);
1143 if( zRet ){
1144 memcpy(zRet, zStr, nCopy);
1145 }else{
1146 *pRc = SQLITE_NOMEM;
1147 }
1148 }
1149 }
1150
1151 return zRet;
1152 }
1153
1154 /*
1155 ** Finalize the statement passed as the second argument.
1156 **
1157 ** If the sqlite3_finalize() call indicates that an error occurs, and the
1158 ** rbu handle error code is not already set, set the error code and error
1159 ** message accordingly.
1160 */
1161 static void rbuFinalize(sqlite3rbu *p, sqlite3_stmt *pStmt){
1162 sqlite3 *db = sqlite3_db_handle(pStmt);
1163 int rc = sqlite3_finalize(pStmt);
1164 if( p->rc==SQLITE_OK && rc!=SQLITE_OK ){
1165 p->rc = rc;
1166 p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
1167 }
1168 }
1169
1170 /* Determine the type of a table.
1171 **
1172 ** peType is of type (int*), a pointer to an output parameter of type
1173 ** (int). This call sets the output parameter as follows, depending
1174 ** on the type of the table specified by parameters dbName and zTbl.
1175 **
1176 ** RBU_PK_NOTABLE: No such table.
1177 ** RBU_PK_NONE: Table has an implicit rowid.
1178 ** RBU_PK_IPK: Table has an explicit IPK column.
1179 ** RBU_PK_EXTERNAL: Table has an external PK index.
1180 ** RBU_PK_WITHOUT_ROWID: Table is WITHOUT ROWID.
1181 ** RBU_PK_VTAB: Table is a virtual table.
1182 **
1183 ** Argument *piPk is also of type (int*), and also points to an output
1184 ** parameter. Unless the table has an external primary key index
1185 ** (i.e. unless *peType is set to 3), then *piPk is set to zero. Or,
1186 ** if the table does have an external primary key index, then *piPk
1187 ** is set to the root page number of the primary key index before
1188 ** returning.
1189 **
1190 ** ALGORITHM:
1191 **
1192 ** if( no entry exists in sqlite_schema ){
1193 ** return RBU_PK_NOTABLE
1194 ** }else if( sql for the entry starts with "CREATE VIRTUAL" ){
1195 ** return RBU_PK_VTAB
1196 ** }else if( "PRAGMA index_list()" for the table contains a "pk" index ){
1197 ** if( the index that is the pk exists in sqlite_schema ){
1198 ** *piPK = rootpage of that index.
1199 ** return RBU_PK_EXTERNAL
1200 ** }else{
1201 ** return RBU_PK_WITHOUT_ROWID
1202 ** }
1203 ** }else if( "PRAGMA table_info()" lists one or more "pk" columns ){
1204 ** return RBU_PK_IPK
1205 ** }else{
1206 ** return RBU_PK_NONE
1207 ** }
1208 */
1209 static void rbuTableType(
1210 sqlite3rbu *p,
1211 const char *zTab,
1212 int *peType,
1213 int *piTnum,
1214 int *piPk
1215 ){
1216 /*
1217 ** 0) SELECT count(*) FROM sqlite_schema where name=%Q AND IsVirtual(%Q)
1218 ** 1) PRAGMA index_list = ?
1219 ** 2) SELECT count(*) FROM sqlite_schema where name=%Q
1220 ** 3) PRAGMA table_info = ?
1221 */
1222 sqlite3_stmt *aStmt[4] = {0, 0, 0, 0};
1223
1224 *peType = RBU_PK_NOTABLE;
1225 *piPk = 0;
1226
1227 assert( p->rc==SQLITE_OK );
1228 p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[0], &p->zErrmsg,
1229 sqlite3_mprintf(
1230 "SELECT "
1231 " (sql COLLATE nocase BETWEEN 'CREATE VIRTUAL' AND 'CREATE VIRTUAM'),"
1232 " rootpage"
1233 " FROM sqlite_schema"
1234 " WHERE name=%Q", zTab
1235 ));
1236 if( p->rc!=SQLITE_OK || sqlite3_step(aStmt[0])!=SQLITE_ROW ){
1237 /* Either an error, or no such table. */
1238 goto rbuTableType_end;
1239 }
1240 if( sqlite3_column_int(aStmt[0], 0) ){
1241 *peType = RBU_PK_VTAB; /* virtual table */
1242 goto rbuTableType_end;
1243 }
1244 *piTnum = sqlite3_column_int(aStmt[0], 1);
1245
1246 p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[1], &p->zErrmsg,
1247 sqlite3_mprintf("PRAGMA index_list=%Q",zTab)
1248 );
1249 if( p->rc ) goto rbuTableType_end;
1250 while( sqlite3_step(aStmt[1])==SQLITE_ROW ){
1251 const u8 *zOrig = sqlite3_column_text(aStmt[1], 3);
1252 const u8 *zIdx = sqlite3_column_text(aStmt[1], 1);
1253 if( zOrig && zIdx && zOrig[0]=='p' ){
1254 p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[2], &p->zErrmsg,
1255 sqlite3_mprintf(
1256 "SELECT rootpage FROM sqlite_schema WHERE name = %Q", zIdx
1257 ));
1258 if( p->rc==SQLITE_OK ){
1259 if( sqlite3_step(aStmt[2])==SQLITE_ROW ){
1260 *piPk = sqlite3_column_int(aStmt[2], 0);
1261 *peType = RBU_PK_EXTERNAL;
1262 }else{
1263 *peType = RBU_PK_WITHOUT_ROWID;
1264 }
1265 }
1266 goto rbuTableType_end;
1267 }
1268 }
1269
1270 p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[3], &p->zErrmsg,
1271 sqlite3_mprintf("PRAGMA table_info=%Q",zTab)
1272 );
1273 if( p->rc==SQLITE_OK ){
1274 while( sqlite3_step(aStmt[3])==SQLITE_ROW ){
1275 if( sqlite3_column_int(aStmt[3],5)>0 ){
1276 *peType = RBU_PK_IPK; /* explicit IPK column */
1277 goto rbuTableType_end;
1278 }
1279 }
1280 *peType = RBU_PK_NONE;
1281 }
1282
1283 rbuTableType_end: {
1284 unsigned int i;
1285 for(i=0; i<sizeof(aStmt)/sizeof(aStmt[0]); i++){
1286 rbuFinalize(p, aStmt[i]);
1287 }
1288 }
1289 }
1290
1291 /*
1292 ** This is a helper function for rbuObjIterCacheTableInfo(). It populates
1293 ** the pIter->abIndexed[] array.
1294 */
1295 static void rbuObjIterCacheIndexedCols(sqlite3rbu *p, RbuObjIter *pIter){
1296 sqlite3_stmt *pList = 0;
1297 int bIndex = 0;
1298
1299 if( p->rc==SQLITE_OK ){
1300 memcpy(pIter->abIndexed, pIter->abTblPk, sizeof(u8)*pIter->nTblCol);
1301 p->rc = prepareFreeAndCollectError(p->dbMain, &pList, &p->zErrmsg,
1302 sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl)
1303 );
1304 }
1305
1306 pIter->nIndex = 0;
1307 while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pList) ){
1308 const char *zIdx = (const char*)sqlite3_column_text(pList, 1);
1309 int bPartial = sqlite3_column_int(pList, 4);
1310 sqlite3_stmt *pXInfo = 0;
1311 if( zIdx==0 ) break;
1312 if( bPartial ){
1313 memset(pIter->abIndexed, 0x01, sizeof(u8)*pIter->nTblCol);
1314 }
1315 p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg,
1316 sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx)
1317 );
1318 while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){
1319 int iCid = sqlite3_column_int(pXInfo, 1);
1320 if( iCid>=0 ) pIter->abIndexed[iCid] = 1;
1321 if( iCid==-2 ){
1322 memset(pIter->abIndexed, 0x01, sizeof(u8)*pIter->nTblCol);
1323 }
1324 }
1325 rbuFinalize(p, pXInfo);
1326 bIndex = 1;
1327 pIter->nIndex++;
1328 }
1329
1330 if( pIter->eType==RBU_PK_WITHOUT_ROWID ){
1331 /* "PRAGMA index_list" includes the main PK b-tree */
1332 pIter->nIndex--;
1333 }
1334
1335 rbuFinalize(p, pList);
1336 if( bIndex==0 ) pIter->abIndexed = 0;
1337 }
1338
1339
1340 /*
1341 ** If they are not already populated, populate the pIter->azTblCol[],
1342 ** pIter->abTblPk[], pIter->nTblCol and pIter->bRowid variables according to
1343 ** the table (not index) that the iterator currently points to.
1344 **
1345 ** Return SQLITE_OK if successful, or an SQLite error code otherwise. If
1346 ** an error does occur, an error code and error message are also left in
1347 ** the RBU handle.
1348 */
1349 static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){
1350 if( pIter->azTblCol==0 ){
1351 sqlite3_stmt *pStmt = 0;
1352 int nCol = 0;
1353 int i; /* for() loop iterator variable */
1354 int bRbuRowid = 0; /* If input table has column "rbu_rowid" */
1355 int iOrder = 0;
1356 int iTnum = 0;
1357
1358 /* Figure out the type of table this step will deal with. */
1359 assert( pIter->eType==0 );
1360 rbuTableType(p, pIter->zTbl, &pIter->eType, &iTnum, &pIter->iPkTnum);
1361 if( p->rc==SQLITE_OK && pIter->eType==RBU_PK_NOTABLE ){
1362 p->rc = SQLITE_ERROR;
1363 p->zErrmsg = sqlite3_mprintf("no such table: %s", pIter->zTbl);
1364 }
1365 if( p->rc ) return p->rc;
1366 if( pIter->zIdx==0 ) pIter->iTnum = iTnum;
1367
1368 assert( pIter->eType==RBU_PK_NONE || pIter->eType==RBU_PK_IPK
1369 || pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_WITHOUT_ROWID
1370 || pIter->eType==RBU_PK_VTAB
1371 );
1372
1373 /* Populate the azTblCol[] and nTblCol variables based on the columns
1374 ** of the input table. Ignore any input table columns that begin with
1375 ** "rbu_". */
1376 p->rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg,
1377 sqlite3_mprintf("SELECT * FROM '%q'", pIter->zDataTbl)
1378 );
1379 if( p->rc==SQLITE_OK ){
1380 nCol = sqlite3_column_count(pStmt);
1381 rbuAllocateIterArrays(p, pIter, nCol);
1382 }
1383 for(i=0; p->rc==SQLITE_OK && i<nCol; i++){
1384 const char *zName = (const char*)sqlite3_column_name(pStmt, i);
1385 if( sqlite3_strnicmp("rbu_", zName, 4) ){
1386 char *zCopy = rbuStrndup(zName, &p->rc);
1387 pIter->aiSrcOrder[pIter->nTblCol] = pIter->nTblCol;
1388 pIter->azTblCol[pIter->nTblCol++] = zCopy;
1389 }
1390 else if( 0==sqlite3_stricmp("rbu_rowid", zName) ){
1391 bRbuRowid = 1;
1392 }
1393 }
1394 sqlite3_finalize(pStmt);
1395 pStmt = 0;
1396
1397 if( p->rc==SQLITE_OK
1398 && rbuIsVacuum(p)==0
1399 && bRbuRowid!=(pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE)
1400 ){
1401 p->rc = SQLITE_ERROR;
1402 p->zErrmsg = sqlite3_mprintf(
1403 "table %q %s rbu_rowid column", pIter->zDataTbl,
1404 (bRbuRowid ? "may not have" : "requires")
1405 );
1406 }
1407
1408 /* Check that all non-HIDDEN columns in the destination table are also
1409 ** present in the input table. Populate the abTblPk[], azTblType[] and
1410 ** aiTblOrder[] arrays at the same time. */
1411 if( p->rc==SQLITE_OK ){
1412 p->rc = prepareFreeAndCollectError(p->dbMain, &pStmt, &p->zErrmsg,
1413 sqlite3_mprintf("PRAGMA table_info(%Q)", pIter->zTbl)
1414 );
1415 }
1416 while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
1417 const char *zName = (const char*)sqlite3_column_text(pStmt, 1);
1418 if( zName==0 ) break; /* An OOM - finalize() below returns S_NOMEM */
1419 for(i=iOrder; i<pIter->nTblCol; i++){
1420 if( 0==strcmp(zName, pIter->azTblCol[i]) ) break;
1421 }
1422 if( i==pIter->nTblCol ){
1423 p->rc = SQLITE_ERROR;
1424 p->zErrmsg = sqlite3_mprintf("column missing from %q: %s",
1425 pIter->zDataTbl, zName
1426 );
1427 }else{
1428 int iPk = sqlite3_column_int(pStmt, 5);
1429 int bNotNull = sqlite3_column_int(pStmt, 3);
1430 const char *zType = (const char*)sqlite3_column_text(pStmt, 2);
1431
1432 if( i!=iOrder ){
1433 SWAP(int, pIter->aiSrcOrder[i], pIter->aiSrcOrder[iOrder]);
1434 SWAP(char*, pIter->azTblCol[i], pIter->azTblCol[iOrder]);
1435 }
1436
1437 pIter->azTblType[iOrder] = rbuStrndup(zType, &p->rc);
1438 assert( iPk>=0 );
1439 pIter->abTblPk[iOrder] = (u8)iPk;
1440 pIter->abNotNull[iOrder] = (u8)bNotNull || (iPk!=0);
1441 iOrder++;
1442 }
1443 }
1444
1445 rbuFinalize(p, pStmt);
1446 rbuObjIterCacheIndexedCols(p, pIter);
1447 assert( pIter->eType!=RBU_PK_VTAB || pIter->abIndexed==0 );
1448 assert( pIter->eType!=RBU_PK_VTAB || pIter->nIndex==0 );
1449 }
1450
1451 return p->rc;
1452 }
1453
1454 /*
1455 ** This function constructs and returns a pointer to a nul-terminated
1456 ** string containing some SQL clause or list based on one or more of the
1457 ** column names currently stored in the pIter->azTblCol[] array.
1458 */
1459 static char *rbuObjIterGetCollist(
1460 sqlite3rbu *p, /* RBU object */
1461 RbuObjIter *pIter /* Object iterator for column names */
1462 ){
1463 char *zList = 0;
1464 const char *zSep = "";
1465 int i;
1466 for(i=0; i<pIter->nTblCol; i++){
1467 const char *z = pIter->azTblCol[i];
1468 zList = rbuMPrintf(p, "%z%s\"%w\"", zList, zSep, z);
1469 zSep = ", ";
1470 }
1471 return zList;
1472 }
1473
1474 /*
1475 ** Return a comma separated list of the quoted PRIMARY KEY column names,
1476 ** in order, for the current table. Before each column name, add the text
1477 ** zPre. After each column name, add the zPost text. Use zSeparator as
1478 ** the separator text (usually ", ").
1479 */
1480 static char *rbuObjIterGetPkList(
1481 sqlite3rbu *p, /* RBU object */
1482 RbuObjIter *pIter, /* Object iterator for column names */
1483 const char *zPre, /* Before each quoted column name */
1484 const char *zSeparator, /* Separator to use between columns */
1485 const char *zPost /* After each quoted column name */
1486 ){
1487 int iPk = 1;
1488 char *zRet = 0;
1489 const char *zSep = "";
1490 while( 1 ){
1491 int i;
1492 for(i=0; i<pIter->nTblCol; i++){
1493 if( (int)pIter->abTblPk[i]==iPk ){
1494 const char *zCol = pIter->azTblCol[i];
1495 zRet = rbuMPrintf(p, "%z%s%s\"%w\"%s", zRet, zSep, zPre, zCol, zPost);
1496 zSep = zSeparator;
1497 break;
1498 }
1499 }
1500 if( i==pIter->nTblCol ) break;
1501 iPk++;
1502 }
1503 return zRet;
1504 }
1505
1506 /*
1507 ** This function is called as part of restarting an RBU vacuum within
1508 ** stage 1 of the process (while the *-oal file is being built) while
1509 ** updating a table (not an index). The table may be a rowid table or
1510 ** a WITHOUT ROWID table. It queries the target database to find the
1511 ** largest key that has already been written to the target table and
1512 ** constructs a WHERE clause that can be used to extract the remaining
1513 ** rows from the source table. For a rowid table, the WHERE clause
1514 ** is of the form:
1515 **
1516 ** "WHERE _rowid_ > ?"
1517 **
1518 ** and for WITHOUT ROWID tables:
1519 **
1520 ** "WHERE (key1, key2) > (?, ?)"
1521 **
1522 ** Instead of "?" placeholders, the actual WHERE clauses created by
1523 ** this function contain literal SQL values.
1524 */
1525 static char *rbuVacuumTableStart(
1526 sqlite3rbu *p, /* RBU handle */
1527 RbuObjIter *pIter, /* RBU iterator object */
1528 int bRowid, /* True for a rowid table */
1529 const char *zWrite /* Target table name prefix */
1530 ){
1531 sqlite3_stmt *pMax = 0;
1532 char *zRet = 0;
1533 if( bRowid ){
1534 p->rc = prepareFreeAndCollectError(p->dbMain, &pMax, &p->zErrmsg,
1535 sqlite3_mprintf(
1536 "SELECT max(_rowid_) FROM \"%s%w\"", zWrite, pIter->zTbl
1537 )
1538 );
1539 if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){
1540 sqlite3_int64 iMax = sqlite3_column_int64(pMax, 0);
1541 zRet = rbuMPrintf(p, " WHERE _rowid_ > %lld ", iMax);
1542 }
1543 rbuFinalize(p, pMax);
1544 }else{
1545 char *zOrder = rbuObjIterGetPkList(p, pIter, "", ", ", " DESC");
1546 char *zSelect = rbuObjIterGetPkList(p, pIter, "quote(", "||','||", ")");
1547 char *zList = rbuObjIterGetPkList(p, pIter, "", ", ", "");
1548
1549 if( p->rc==SQLITE_OK ){
1550 p->rc = prepareFreeAndCollectError(p->dbMain, &pMax, &p->zErrmsg,
1551 sqlite3_mprintf(
1552 "SELECT %s FROM \"%s%w\" ORDER BY %s LIMIT 1",
1553 zSelect, zWrite, pIter->zTbl, zOrder
1554 )
1555 );
1556 if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){
1557 const char *zVal = (const char*)sqlite3_column_text(pMax, 0);
1558 zRet = rbuMPrintf(p, " WHERE (%s) > (%s) ", zList, zVal);
1559 }
1560 rbuFinalize(p, pMax);
1561 }
1562
1563 sqlite3_free(zOrder);
1564 sqlite3_free(zSelect);
1565 sqlite3_free(zList);
1566 }
1567 return zRet;
1568 }
1569
1570 /*
1571 ** This function is called as part of restating an RBU vacuum when the
1572 ** current operation is writing content to an index. If possible, it
1573 ** queries the target index b-tree for the largest key already written to
1574 ** it, then composes and returns an expression that can be used in a WHERE
1575 ** clause to select the remaining required rows from the source table.
1576 ** It is only possible to return such an expression if:
1577 **
1578 ** * The index contains no DESC columns, and
1579 ** * The last key written to the index before the operation was
1580 ** suspended does not contain any NULL values.
1581 **
1582 ** The expression is of the form:
1583 **
1584 ** (index-field1, index-field2, ...) > (?, ?, ...)
1585 **
1586 ** except that the "?" placeholders are replaced with literal values.
1587 **
1588 ** If the expression cannot be created, NULL is returned. In this case,
1589 ** the caller has to use an OFFSET clause to extract only the required
1590 ** rows from the sourct table, just as it does for an RBU update operation.
1591 */
1592 static char *rbuVacuumIndexStart(
1593 sqlite3rbu *p, /* RBU handle */
1594 RbuObjIter *pIter /* RBU iterator object */
1595 ){
1596 char *zOrder = 0;
1597 char *zLhs = 0;
1598 char *zSelect = 0;
1599 char *zVector = 0;
1600 char *zRet = 0;
1601 int bFailed = 0;
1602 const char *zSep = "";
1603 int iCol = 0;
1604 sqlite3_stmt *pXInfo = 0;
1605
1606 p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg,
1607 sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", pIter->zIdx)
1608 );
1609 while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){
1610 int iCid = sqlite3_column_int(pXInfo, 1);
1611 const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4);
1612 const char *zCol;
1613 if( sqlite3_column_int(pXInfo, 3) ){
1614 bFailed = 1;
1615 break;
1616 }
1617
1618 if( iCid<0 ){
1619 if( pIter->eType==RBU_PK_IPK ){
1620 int i;
1621 for(i=0; pIter->abTblPk[i]==0; i++);
1622 assert( i<pIter->nTblCol );
1623 zCol = pIter->azTblCol[i];
1624 }else{
1625 zCol = "_rowid_";
1626 }
1627 }else{
1628 zCol = pIter->azTblCol[iCid];
1629 }
1630
1631 zLhs = rbuMPrintf(p, "%z%s \"%w\" COLLATE %Q",
1632 zLhs, zSep, zCol, zCollate
1633 );
1634 zOrder = rbuMPrintf(p, "%z%s \"rbu_imp_%d%w\" COLLATE %Q DESC",
1635 zOrder, zSep, iCol, zCol, zCollate
1636 );
1637 zSelect = rbuMPrintf(p, "%z%s quote(\"rbu_imp_%d%w\")",
1638 zSelect, zSep, iCol, zCol
1639 );
1640 zSep = ", ";
1641 iCol++;
1642 }
1643 rbuFinalize(p, pXInfo);
1644 if( bFailed ) goto index_start_out;
1645
1646 if( p->rc==SQLITE_OK ){
1647 sqlite3_stmt *pSel = 0;
1648
1649 p->rc = prepareFreeAndCollectError(p->dbMain, &pSel, &p->zErrmsg,
1650 sqlite3_mprintf("SELECT %s FROM \"rbu_imp_%w\" ORDER BY %s LIMIT 1",
1651 zSelect, pIter->zTbl, zOrder
1652 )
1653 );
1654 if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSel) ){
1655 zSep = "";
1656 for(iCol=0; iCol<pIter->nCol; iCol++){
1657 const char *zQuoted = (const char*)sqlite3_column_text(pSel, iCol);
1658 if( zQuoted==0 ){
1659 p->rc = SQLITE_NOMEM;
1660 }else if( zQuoted[0]=='N' ){
1661 bFailed = 1;
1662 break;
1663 }
1664 zVector = rbuMPrintf(p, "%z%s%s", zVector, zSep, zQuoted);
1665 zSep = ", ";
1666 }
1667
1668 if( !bFailed ){
1669 zRet = rbuMPrintf(p, "(%s) > (%s)", zLhs, zVector);
1670 }
1671 }
1672 rbuFinalize(p, pSel);
1673 }
1674
1675 index_start_out:
1676 sqlite3_free(zOrder);
1677 sqlite3_free(zSelect);
1678 sqlite3_free(zVector);
1679 sqlite3_free(zLhs);
1680 return zRet;
1681 }
1682
1683 /*
1684 ** This function is used to create a SELECT list (the list of SQL
1685 ** expressions that follows a SELECT keyword) for a SELECT statement
1686 ** used to read from an data_xxx or rbu_tmp_xxx table while updating the
1687 ** index object currently indicated by the iterator object passed as the
1688 ** second argument. A "PRAGMA index_xinfo = <idxname>" statement is used
1689 ** to obtain the required information.
1690 **
1691 ** If the index is of the following form:
1692 **
1693 ** CREATE INDEX i1 ON t1(c, b COLLATE nocase);
1694 **
1695 ** and "t1" is a table with an explicit INTEGER PRIMARY KEY column
1696 ** "ipk", the returned string is:
1697 **
1698 ** "`c` COLLATE 'BINARY', `b` COLLATE 'NOCASE', `ipk` COLLATE 'BINARY'"
1699 **
1700 ** As well as the returned string, three other malloc'd strings are
1701 ** returned via output parameters. As follows:
1702 **
1703 ** pzImposterCols: ...
1704 ** pzImposterPk: ...
1705 ** pzWhere: ...
1706 */
1707 static char *rbuObjIterGetIndexCols(
1708 sqlite3rbu *p, /* RBU object */
1709 RbuObjIter *pIter, /* Object iterator for column names */
1710 char **pzImposterCols, /* OUT: Columns for imposter table */
1711 char **pzImposterPk, /* OUT: Imposter PK clause */
1712 char **pzWhere, /* OUT: WHERE clause */
1713 int *pnBind /* OUT: Trbul number of columns */
1714 ){
1715 int rc = p->rc; /* Error code */
1716 int rc2; /* sqlite3_finalize() return code */
1717 char *zRet = 0; /* String to return */
1718 char *zImpCols = 0; /* String to return via *pzImposterCols */
1719 char *zImpPK = 0; /* String to return via *pzImposterPK */
1720 char *zWhere = 0; /* String to return via *pzWhere */
1721 int nBind = 0; /* Value to return via *pnBind */
1722 const char *zCom = ""; /* Set to ", " later on */
1723 const char *zAnd = ""; /* Set to " AND " later on */
1724 sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = ? */
1725
1726 if( rc==SQLITE_OK ){
1727 assert( p->zErrmsg==0 );
1728 rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg,
1729 sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", pIter->zIdx)
1730 );
1731 }
1732
1733 while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){
1734 int iCid = sqlite3_column_int(pXInfo, 1);
1735 int bDesc = sqlite3_column_int(pXInfo, 3);
1736 const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4);
1737 const char *zCol = 0;
1738 const char *zType;
1739
1740 if( iCid==-2 ){
1741 int iSeq = sqlite3_column_int(pXInfo, 0);
1742 zRet = sqlite3_mprintf("%z%s(%.*s) COLLATE %Q", zRet, zCom,
1743 pIter->aIdxCol[iSeq].nSpan, pIter->aIdxCol[iSeq].zSpan, zCollate
1744 );
1745 zType = "";
1746 }else {
1747 if( iCid<0 ){
1748 /* An integer primary key. If the table has an explicit IPK, use
1749 ** its name. Otherwise, use "rbu_rowid". */
1750 if( pIter->eType==RBU_PK_IPK ){
1751 int i;
1752 for(i=0; pIter->abTblPk[i]==0; i++);
1753 assert( i<pIter->nTblCol );
1754 zCol = pIter->azTblCol[i];
1755 }else if( rbuIsVacuum(p) ){
1756 zCol = "_rowid_";
1757 }else{
1758 zCol = "rbu_rowid";
1759 }
1760 zType = "INTEGER";
1761 }else{
1762 zCol = pIter->azTblCol[iCid];
1763 zType = pIter->azTblType[iCid];
1764 }
1765 zRet = sqlite3_mprintf("%z%s\"%w\" COLLATE %Q", zRet, zCom,zCol,zCollate);
1766 }
1767
1768 if( pIter->bUnique==0 || sqlite3_column_int(pXInfo, 5) ){
1769 const char *zOrder = (bDesc ? " DESC" : "");
1770 zImpPK = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\"%s",
1771 zImpPK, zCom, nBind, zCol, zOrder
1772 );
1773 }
1774 zImpCols = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\" %s COLLATE %Q",
1775 zImpCols, zCom, nBind, zCol, zType, zCollate
1776 );
1777 zWhere = sqlite3_mprintf(
1778 "%z%s\"rbu_imp_%d%w\" IS ?", zWhere, zAnd, nBind, zCol
1779 );
1780 if( zRet==0 || zImpPK==0 || zImpCols==0 || zWhere==0 ) rc = SQLITE_NOMEM;
1781 zCom = ", ";
1782 zAnd = " AND ";
1783 nBind++;
1784 }
1785
1786 rc2 = sqlite3_finalize(pXInfo);
1787 if( rc==SQLITE_OK ) rc = rc2;
1788
1789 if( rc!=SQLITE_OK ){
1790 sqlite3_free(zRet);
1791 sqlite3_free(zImpCols);
1792 sqlite3_free(zImpPK);
1793 sqlite3_free(zWhere);
1794 zRet = 0;
1795 zImpCols = 0;
1796 zImpPK = 0;
1797 zWhere = 0;
1798 p->rc = rc;
1799 }
1800
1801 *pzImposterCols = zImpCols;
1802 *pzImposterPk = zImpPK;
1803 *pzWhere = zWhere;
1804 *pnBind = nBind;
1805 return zRet;
1806 }
1807
1808 /*
1809 ** Assuming the current table columns are "a", "b" and "c", and the zObj
1810 ** paramter is passed "old", return a string of the form:
1811 **
1812 ** "old.a, old.b, old.b"
1813 **
1814 ** With the column names escaped.
1815 **
1816 ** For tables with implicit rowids - RBU_PK_EXTERNAL and RBU_PK_NONE, append
1817 ** the text ", old._rowid_" to the returned value.
1818 */
1819 static char *rbuObjIterGetOldlist(
1820 sqlite3rbu *p,
1821 RbuObjIter *pIter,
1822 const char *zObj
1823 ){
1824 char *zList = 0;
1825 if( p->rc==SQLITE_OK && pIter->abIndexed ){
1826 const char *zS = "";
1827 int i;
1828 for(i=0; i<pIter->nTblCol; i++){
1829 if( pIter->abIndexed[i] ){
1830 const char *zCol = pIter->azTblCol[i];
1831 zList = sqlite3_mprintf("%z%s%s.\"%w\"", zList, zS, zObj, zCol);
1832 }else{
1833 zList = sqlite3_mprintf("%z%sNULL", zList, zS);
1834 }
1835 zS = ", ";
1836 if( zList==0 ){
1837 p->rc = SQLITE_NOMEM;
1838 break;
1839 }
1840 }
1841
1842 /* For a table with implicit rowids, append "old._rowid_" to the list. */
1843 if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){
1844 zList = rbuMPrintf(p, "%z, %s._rowid_", zList, zObj);
1845 }
1846 }
1847 return zList;
1848 }
1849
1850 /*
1851 ** Return an expression that can be used in a WHERE clause to match the
1852 ** primary key of the current table. For example, if the table is:
1853 **
1854 ** CREATE TABLE t1(a, b, c, PRIMARY KEY(b, c));
1855 **
1856 ** Return the string:
1857 **
1858 ** "b = ?1 AND c = ?2"
1859 */
1860 static char *rbuObjIterGetWhere(
1861 sqlite3rbu *p,
1862 RbuObjIter *pIter
1863 ){
1864 char *zList = 0;
1865 if( pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE ){
1866 zList = rbuMPrintf(p, "_rowid_ = ?%d", pIter->nTblCol+1);
1867 }else if( pIter->eType==RBU_PK_EXTERNAL ){
1868 const char *zSep = "";
1869 int i;
1870 for(i=0; i<pIter->nTblCol; i++){
1871 if( pIter->abTblPk[i] ){
1872 zList = rbuMPrintf(p, "%z%sc%d=?%d", zList, zSep, i, i+1);
1873 zSep = " AND ";
1874 }
1875 }
1876 zList = rbuMPrintf(p,
1877 "_rowid_ = (SELECT id FROM rbu_imposter2 WHERE %z)", zList
1878 );
1879
1880 }else{
1881 const char *zSep = "";
1882 int i;
1883 for(i=0; i<pIter->nTblCol; i++){
1884 if( pIter->abTblPk[i] ){
1885 const char *zCol = pIter->azTblCol[i];
1886 zList = rbuMPrintf(p, "%z%s\"%w\"=?%d", zList, zSep, zCol, i+1);
1887 zSep = " AND ";
1888 }
1889 }
1890 }
1891 return zList;
1892 }
1893
1894 /*
1895 ** The SELECT statement iterating through the keys for the current object
1896 ** (p->objiter.pSelect) currently points to a valid row. However, there
1897 ** is something wrong with the rbu_control value in the rbu_control value
1898 ** stored in the (p->nCol+1)'th column. Set the error code and error message
1899 ** of the RBU handle to something reflecting this.
1900 */
1901 static void rbuBadControlError(sqlite3rbu *p){
1902 p->rc = SQLITE_ERROR;
1903 p->zErrmsg = sqlite3_mprintf("invalid rbu_control value");
1904 }
1905
1906
1907 /*
1908 ** Return a nul-terminated string containing the comma separated list of
1909 ** assignments that should be included following the "SET" keyword of
1910 ** an UPDATE statement used to update the table object that the iterator
1911 ** passed as the second argument currently points to if the rbu_control
1912 ** column of the data_xxx table entry is set to zMask.
1913 **
1914 ** The memory for the returned string is obtained from sqlite3_malloc().
1915 ** It is the responsibility of the caller to eventually free it using
1916 ** sqlite3_free().
1917 **
1918 ** If an OOM error is encountered when allocating space for the new
1919 ** string, an error code is left in the rbu handle passed as the first
1920 ** argument and NULL is returned. Or, if an error has already occurred
1921 ** when this function is called, NULL is returned immediately, without
1922 ** attempting the allocation or modifying the stored error code.
1923 */
1924 static char *rbuObjIterGetSetlist(
1925 sqlite3rbu *p,
1926 RbuObjIter *pIter,
1927 const char *zMask
1928 ){
1929 char *zList = 0;
1930 if( p->rc==SQLITE_OK ){
1931 int i;
1932
1933 if( (int)strlen(zMask)!=pIter->nTblCol ){
1934 rbuBadControlError(p);
1935 }else{
1936 const char *zSep = "";
1937 for(i=0; i<pIter->nTblCol; i++){
1938 char c = zMask[pIter->aiSrcOrder[i]];
1939 if( c=='x' ){
1940 zList = rbuMPrintf(p, "%z%s\"%w\"=?%d",
1941 zList, zSep, pIter->azTblCol[i], i+1
1942 );
1943 zSep = ", ";
1944 }
1945 else if( c=='d' ){
1946 zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_delta(\"%w\", ?%d)",
1947 zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1
1948 );
1949 zSep = ", ";
1950 }
1951 else if( c=='f' ){
1952 zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_fossil_delta(\"%w\", ?%d)",
1953 zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1
1954 );
1955 zSep = ", ";
1956 }
1957 }
1958 }
1959 }
1960 return zList;
1961 }
1962
1963 /*
1964 ** Return a nul-terminated string consisting of nByte comma separated
1965 ** "?" expressions. For example, if nByte is 3, return a pointer to
1966 ** a buffer containing the string "?,?,?".
1967 **
1968 ** The memory for the returned string is obtained from sqlite3_malloc().
1969 ** It is the responsibility of the caller to eventually free it using
1970 ** sqlite3_free().
1971 **
1972 ** If an OOM error is encountered when allocating space for the new
1973 ** string, an error code is left in the rbu handle passed as the first
1974 ** argument and NULL is returned. Or, if an error has already occurred
1975 ** when this function is called, NULL is returned immediately, without
1976 ** attempting the allocation or modifying the stored error code.
1977 */
1978 static char *rbuObjIterGetBindlist(sqlite3rbu *p, int nBind){
1979 char *zRet = 0;
1980 sqlite3_int64 nByte = 2*(sqlite3_int64)nBind + 1;
1981
1982 zRet = (char*)rbuMalloc(p, nByte);
1983 if( zRet ){
1984 int i;
1985 for(i=0; i<nBind; i++){
1986 zRet[i*2] = '?';
1987 zRet[i*2+1] = (i+1==nBind) ? '\0' : ',';
1988 }
1989 }
1990 return zRet;
1991 }
1992
1993 /*
1994 ** The iterator currently points to a table (not index) of type
1995 ** RBU_PK_WITHOUT_ROWID. This function creates the PRIMARY KEY
1996 ** declaration for the corresponding imposter table. For example,
1997 ** if the iterator points to a table created as:
1998 **
1999 ** CREATE TABLE t1(a, b, c, PRIMARY KEY(b, a DESC)) WITHOUT ROWID
2000 **
2001 ** this function returns:
2002 **
2003 ** PRIMARY KEY("b", "a" DESC)
2004 */
2005 static char *rbuWithoutRowidPK(sqlite3rbu *p, RbuObjIter *pIter){
2006 char *z = 0;
2007 assert( pIter->zIdx==0 );
2008 if( p->rc==SQLITE_OK ){
2009 const char *zSep = "PRIMARY KEY(";
2010 sqlite3_stmt *pXList = 0; /* PRAGMA index_list = (pIter->zTbl) */
2011 sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = <pk-index> */
2012
2013 p->rc = prepareFreeAndCollectError(p->dbMain, &pXList, &p->zErrmsg,
2014 sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl)
2015 );
2016 while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXList) ){
2017 const char *zOrig = (const char*)sqlite3_column_text(pXList,3);
2018 if( zOrig && strcmp(zOrig, "pk")==0 ){
2019 const char *zIdx = (const char*)sqlite3_column_text(pXList,1);
2020 if( zIdx ){
2021 p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg,
2022 sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx)
2023 );
2024 }
2025 break;
2026 }
2027 }
2028 rbuFinalize(p, pXList);
2029
2030 while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){
2031 if( sqlite3_column_int(pXInfo, 5) ){
2032 /* int iCid = sqlite3_column_int(pXInfo, 0); */
2033 const char *zCol = (const char*)sqlite3_column_text(pXInfo, 2);
2034 const char *zDesc = sqlite3_column_int(pXInfo, 3) ? " DESC" : "";
2035 z = rbuMPrintf(p, "%z%s\"%w\"%s", z, zSep, zCol, zDesc);
2036 zSep = ", ";
2037 }
2038 }
2039 z = rbuMPrintf(p, "%z)", z);
2040 rbuFinalize(p, pXInfo);
2041 }
2042 return z;
2043 }
2044
2045 /*
2046 ** This function creates the second imposter table used when writing to
2047 ** a table b-tree where the table has an external primary key. If the
2048 ** iterator passed as the second argument does not currently point to
2049 ** a table (not index) with an external primary key, this function is a
2050 ** no-op.
2051 **
2052 ** Assuming the iterator does point to a table with an external PK, this
2053 ** function creates a WITHOUT ROWID imposter table named "rbu_imposter2"
2054 ** used to access that PK index. For example, if the target table is
2055 ** declared as follows:
2056 **
2057 ** CREATE TABLE t1(a, b TEXT, c REAL, PRIMARY KEY(b, c));
2058 **
2059 ** then the imposter table schema is:
2060 **
2061 ** CREATE TABLE rbu_imposter2(c1 TEXT, c2 REAL, id INTEGER) WITHOUT ROWID;
2062 **
2063 */
2064 static void rbuCreateImposterTable2(sqlite3rbu *p, RbuObjIter *pIter){
2065 if( p->rc==SQLITE_OK && pIter->eType==RBU_PK_EXTERNAL ){
2066 int tnum = pIter->iPkTnum; /* Root page of PK index */
2067 sqlite3_stmt *pQuery = 0; /* SELECT name ... WHERE rootpage = $tnum */
2068 const char *zIdx = 0; /* Name of PK index */
2069 sqlite3_stmt *pXInfo = 0; /* PRAGMA main.index_xinfo = $zIdx */
2070 const char *zComma = "";
2071 char *zCols = 0; /* Used to build up list of table cols */
2072 char *zPk = 0; /* Used to build up table PK declaration */
2073
2074 /* Figure out the name of the primary key index for the current table.
2075 ** This is needed for the argument to "PRAGMA index_xinfo". Set
2076 ** zIdx to point to a nul-terminated string containing this name. */
2077 p->rc = prepareAndCollectError(p->dbMain, &pQuery, &p->zErrmsg,
2078 "SELECT name FROM sqlite_schema WHERE rootpage = ?"
2079 );
2080 if( p->rc==SQLITE_OK ){
2081 sqlite3_bind_int(pQuery, 1, tnum);
2082 if( SQLITE_ROW==sqlite3_step(pQuery) ){
2083 zIdx = (const char*)sqlite3_column_text(pQuery, 0);
2084 }
2085 }
2086 if( zIdx ){
2087 p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg,
2088 sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx)
2089 );
2090 }
2091 rbuFinalize(p, pQuery);
2092
2093 while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){
2094 int bKey = sqlite3_column_int(pXInfo, 5);
2095 if( bKey ){
2096 int iCid = sqlite3_column_int(pXInfo, 1);
2097 int bDesc = sqlite3_column_int(pXInfo, 3);
2098 const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4);
2099 zCols = rbuMPrintf(p, "%z%sc%d %s COLLATE %Q", zCols, zComma,
2100 iCid, pIter->azTblType[iCid], zCollate
2101 );
2102 zPk = rbuMPrintf(p, "%z%sc%d%s", zPk, zComma, iCid, bDesc?" DESC":"");
2103 zComma = ", ";
2104 }
2105 }
2106 zCols = rbuMPrintf(p, "%z, id INTEGER", zCols);
2107 rbuFinalize(p, pXInfo);
2108
2109 sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1, tnum);
2110 rbuMPrintfExec(p, p->dbMain,
2111 "CREATE TABLE rbu_imposter2(%z, PRIMARY KEY(%z)) WITHOUT ROWID",
2112 zCols, zPk
2113 );
2114 sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0);
2115 }
2116 }
2117
2118 /*
2119 ** If an error has already occurred when this function is called, it
2120 ** immediately returns zero (without doing any work). Or, if an error
2121 ** occurs during the execution of this function, it sets the error code
2122 ** in the sqlite3rbu object indicated by the first argument and returns
2123 ** zero.
2124 **
2125 ** The iterator passed as the second argument is guaranteed to point to
2126 ** a table (not an index) when this function is called. This function
2127 ** attempts to create any imposter table required to write to the main
2128 ** table b-tree of the table before returning. Non-zero is returned if
2129 ** an imposter table are created, or zero otherwise.
2130 **
2131 ** An imposter table is required in all cases except RBU_PK_VTAB. Only
2132 ** virtual tables are written to directly. The imposter table has the
2133 ** same schema as the actual target table (less any UNIQUE constraints).
2134 ** More precisely, the "same schema" means the same columns, types,
2135 ** collation sequences. For tables that do not have an external PRIMARY
2136 ** KEY, it also means the same PRIMARY KEY declaration.
2137 */
2138 static void rbuCreateImposterTable(sqlite3rbu *p, RbuObjIter *pIter){
2139 if( p->rc==SQLITE_OK && pIter->eType!=RBU_PK_VTAB ){
2140 int tnum = pIter->iTnum;
2141 const char *zComma = "";
2142 char *zSql = 0;
2143 int iCol;
2144 sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 1);
2145
2146 for(iCol=0; p->rc==SQLITE_OK && iCol<pIter->nTblCol; iCol++){
2147 const char *zPk = "";
2148 const char *zCol = pIter->azTblCol[iCol];
2149 const char *zColl = 0;
2150
2151 p->rc = sqlite3_table_column_metadata(
2152 p->dbMain, "main", pIter->zTbl, zCol, 0, &zColl, 0, 0, 0
2153 );
2154
2155 if( pIter->eType==RBU_PK_IPK && pIter->abTblPk[iCol] ){
2156 /* If the target table column is an "INTEGER PRIMARY KEY", add
2157 ** "PRIMARY KEY" to the imposter table column declaration. */
2158 zPk = "PRIMARY KEY ";
2159 }
2160 zSql = rbuMPrintf(p, "%z%s\"%w\" %s %sCOLLATE %Q%s",
2161 zSql, zComma, zCol, pIter->azTblType[iCol], zPk, zColl,
2162 (pIter->abNotNull[iCol] ? " NOT NULL" : "")
2163 );
2164 zComma = ", ";
2165 }
2166
2167 if( pIter->eType==RBU_PK_WITHOUT_ROWID ){
2168 char *zPk = rbuWithoutRowidPK(p, pIter);
2169 if( zPk ){
2170 zSql = rbuMPrintf(p, "%z, %z", zSql, zPk);
2171 }
2172 }
2173
2174 sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1, tnum);
2175 rbuMPrintfExec(p, p->dbMain, "CREATE TABLE \"rbu_imp_%w\"(%z)%s",
2176 pIter->zTbl, zSql,
2177 (pIter->eType==RBU_PK_WITHOUT_ROWID ? " WITHOUT ROWID" : "")
2178 );
2179 sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0);
2180 }
2181 }
2182
2183 /*
2184 ** Prepare a statement used to insert rows into the "rbu_tmp_xxx" table.
2185 ** Specifically a statement of the form:
2186 **
2187 ** INSERT INTO rbu_tmp_xxx VALUES(?, ?, ? ...);
2188 **
2189 ** The number of bound variables is equal to the number of columns in
2190 ** the target table, plus one (for the rbu_control column), plus one more
2191 ** (for the rbu_rowid column) if the target table is an implicit IPK or
2192 ** virtual table.
2193 */
2194 static void rbuObjIterPrepareTmpInsert(
2195 sqlite3rbu *p,
2196 RbuObjIter *pIter,
2197 const char *zCollist,
2198 const char *zRbuRowid
2199 ){
2200 int bRbuRowid = (pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE);
2201 char *zBind = rbuObjIterGetBindlist(p, pIter->nTblCol + 1 + bRbuRowid);
2202 if( zBind ){
2203 assert( pIter->pTmpInsert==0 );
2204 p->rc = prepareFreeAndCollectError(
2205 p->dbRbu, &pIter->pTmpInsert, &p->zErrmsg, sqlite3_mprintf(
2206 "INSERT INTO %s.'rbu_tmp_%q'(rbu_control,%s%s) VALUES(%z)",
2207 p->zStateDb, pIter->zDataTbl, zCollist, zRbuRowid, zBind
2208 ));
2209 }
2210 }
2211
2212 static void rbuTmpInsertFunc(
2213 sqlite3_context *pCtx,
2214 int nVal,
2215 sqlite3_value **apVal
2216 ){
2217 sqlite3rbu *p = sqlite3_user_data(pCtx);
2218 int rc = SQLITE_OK;
2219 int i;
2220
2221 assert( sqlite3_value_int(apVal[0])!=0
2222 || p->objiter.eType==RBU_PK_EXTERNAL
2223 || p->objiter.eType==RBU_PK_NONE
2224 );
2225 if( sqlite3_value_int(apVal[0])!=0 ){
2226 p->nPhaseOneStep += p->objiter.nIndex;
2227 }
2228
2229 for(i=0; rc==SQLITE_OK && i<nVal; i++){
2230 rc = sqlite3_bind_value(p->objiter.pTmpInsert, i+1, apVal[i]);
2231 }
2232 if( rc==SQLITE_OK ){
2233 sqlite3_step(p->objiter.pTmpInsert);
2234 rc = sqlite3_reset(p->objiter.pTmpInsert);
2235 }
2236
2237 if( rc!=SQLITE_OK ){
2238 sqlite3_result_error_code(pCtx, rc);
2239 }
2240 }
2241
2242 static char *rbuObjIterGetIndexWhere(sqlite3rbu *p, RbuObjIter *pIter){
2243 sqlite3_stmt *pStmt = 0;
2244 int rc = p->rc;
2245 char *zRet = 0;
2246
2247 assert( pIter->zIdxSql==0 && pIter->nIdxCol==0 && pIter->aIdxCol==0 );
2248
2249 if( rc==SQLITE_OK ){
2250 rc = prepareAndCollectError(p->dbMain, &pStmt, &p->zErrmsg,
2251 "SELECT trim(sql) FROM sqlite_schema WHERE type='index' AND name=?"
2252 );
2253 }
2254 if( rc==SQLITE_OK ){
2255 int rc2;
2256 rc = sqlite3_bind_text(pStmt, 1, pIter->zIdx, -1, SQLITE_STATIC);
2257 if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
2258 char *zSql = (char*)sqlite3_column_text(pStmt, 0);
2259 if( zSql ){
2260 pIter->zIdxSql = zSql = rbuStrndup(zSql, &rc);
2261 }
2262 if( zSql ){
2263 int nParen = 0; /* Number of open parenthesis */
2264 int i;
2265 int iIdxCol = 0;
2266 int nIdxAlloc = 0;
2267 for(i=0; zSql[i]; i++){
2268 char c = zSql[i];
2269
2270 /* If necessary, grow the pIter->aIdxCol[] array */
2271 if( iIdxCol==nIdxAlloc ){
2272 RbuSpan *aIdxCol = (RbuSpan*)sqlite3_realloc64(
2273 pIter->aIdxCol, nIdxAlloc*sizeof(RbuSpan) + 16*sizeof(RbuSpan)
2274 );
2275 if( aIdxCol==0 ){
2276 rc = SQLITE_NOMEM;
2277 break;
2278 }
2279 pIter->aIdxCol = aIdxCol;
2280 nIdxAlloc += 16;
2281 }
2282
2283 if( c=='(' ){
2284 if( nParen==0 ){
2285 assert( iIdxCol==0 );
2286 pIter->aIdxCol[0].zSpan = &zSql[i+1];
2287 }
2288 nParen++;
2289 }
2290 else if( c==')' ){
2291 nParen--;
2292 if( nParen==0 ){
2293 int nSpan = (int)(&zSql[i] - pIter->aIdxCol[iIdxCol].zSpan);
2294 pIter->aIdxCol[iIdxCol++].nSpan = nSpan;
2295 i++;
2296 break;
2297 }
2298 }else if( c==',' && nParen==1 ){
2299 int nSpan = (int)(&zSql[i] - pIter->aIdxCol[iIdxCol].zSpan);
2300 pIter->aIdxCol[iIdxCol++].nSpan = nSpan;
2301 pIter->aIdxCol[iIdxCol].zSpan = &zSql[i+1];
2302 }else if( c=='"' || c=='\'' || c=='`' ){
2303 for(i++; 1; i++){
2304 if( zSql[i]==c ){
2305 if( zSql[i+1]!=c ) break;
2306 i++;
2307 }
2308 }
2309 }else if( c=='[' ){
2310 for(i++; 1; i++){
2311 if( zSql[i]==']' ) break;
2312 }
2313 }else if( c=='-' && zSql[i+1]=='-' ){
2314 for(i=i+2; zSql[i] && zSql[i]!='\n'; i++);
2315 if( zSql[i]=='\0' ) break;
2316 }else if( c=='/' && zSql[i+1]=='*' ){
2317 for(i=i+2; zSql[i] && (zSql[i]!='*' || zSql[i+1]!='/'); i++);
2318 if( zSql[i]=='\0' ) break;
2319 i++;
2320 }
2321 }
2322 if( zSql[i] ){
2323 zRet = rbuStrndup(&zSql[i], &rc);
2324 }
2325 pIter->nIdxCol = iIdxCol;
2326 }
2327 }
2328
2329 rc2 = sqlite3_finalize(pStmt);
2330 if( rc==SQLITE_OK ) rc = rc2;
2331 }
2332
2333 p->rc = rc;
2334 return zRet;
2335 }
2336
2337 /*
2338 ** Ensure that the SQLite statement handles required to update the
2339 ** target database object currently indicated by the iterator passed
2340 ** as the second argument are available.
2341 */
2342 static int rbuObjIterPrepareAll(
2343 sqlite3rbu *p,
2344 RbuObjIter *pIter,
2345 int nOffset /* Add "LIMIT -1 OFFSET $nOffset" to SELECT */
2346 ){
2347 assert( pIter->bCleanup==0 );
2348 if( pIter->pSelect==0 && rbuObjIterCacheTableInfo(p, pIter)==SQLITE_OK ){
2349 const int tnum = pIter->iTnum;
2350 char *zCollist = 0; /* List of indexed columns */
2351 char **pz = &p->zErrmsg;
2352 const char *zIdx = pIter->zIdx;
2353 char *zLimit = 0;
2354
2355 if( nOffset ){
2356 zLimit = sqlite3_mprintf(" LIMIT -1 OFFSET %d", nOffset);
2357 if( !zLimit ) p->rc = SQLITE_NOMEM;
2358 }
2359
2360 if( zIdx ){
2361 const char *zTbl = pIter->zTbl;
2362 char *zImposterCols = 0; /* Columns for imposter table */
2363 char *zImposterPK = 0; /* Primary key declaration for imposter */
2364 char *zWhere = 0; /* WHERE clause on PK columns */
2365 char *zBind = 0;
2366 char *zPart = 0;
2367 int nBind = 0;
2368
2369 assert( pIter->eType!=RBU_PK_VTAB );
2370 zPart = rbuObjIterGetIndexWhere(p, pIter);
2371 zCollist = rbuObjIterGetIndexCols(
2372 p, pIter, &zImposterCols, &zImposterPK, &zWhere, &nBind
2373 );
2374 zBind = rbuObjIterGetBindlist(p, nBind);
2375
2376 /* Create the imposter table used to write to this index. */
2377 sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 1);
2378 sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1,tnum);
2379 rbuMPrintfExec(p, p->dbMain,
2380 "CREATE TABLE \"rbu_imp_%w\"( %s, PRIMARY KEY( %s ) ) WITHOUT ROWID",
2381 zTbl, zImposterCols, zImposterPK
2382 );
2383 sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0);
2384
2385 /* Create the statement to insert index entries */
2386 pIter->nCol = nBind;
2387 if( p->rc==SQLITE_OK ){
2388 p->rc = prepareFreeAndCollectError(
2389 p->dbMain, &pIter->pInsert, &p->zErrmsg,
2390 sqlite3_mprintf("INSERT INTO \"rbu_imp_%w\" VALUES(%s)", zTbl, zBind)
2391 );
2392 }
2393
2394 /* And to delete index entries */
2395 if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){
2396 p->rc = prepareFreeAndCollectError(
2397 p->dbMain, &pIter->pDelete, &p->zErrmsg,
2398 sqlite3_mprintf("DELETE FROM \"rbu_imp_%w\" WHERE %s", zTbl, zWhere)
2399 );
2400 }
2401
2402 /* Create the SELECT statement to read keys in sorted order */
2403 if( p->rc==SQLITE_OK ){
2404 char *zSql;
2405 if( rbuIsVacuum(p) ){
2406 char *zStart = 0;
2407 if( nOffset ){
2408 zStart = rbuVacuumIndexStart(p, pIter);
2409 if( zStart ){
2410 sqlite3_free(zLimit);
2411 zLimit = 0;
2412 }
2413 }
2414
2415 zSql = sqlite3_mprintf(
2416 "SELECT %s, 0 AS rbu_control FROM '%q' %s %s %s ORDER BY %s%s",
2417 zCollist,
2418 pIter->zDataTbl,
2419 zPart,
2420 (zStart ? (zPart ? "AND" : "WHERE") : ""), zStart,
2421 zCollist, zLimit
2422 );
2423 sqlite3_free(zStart);
2424 }else
2425
2426 if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){
2427 zSql = sqlite3_mprintf(
2428 "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' %s ORDER BY %s%s",
2429 zCollist, p->zStateDb, pIter->zDataTbl,
2430 zPart, zCollist, zLimit
2431 );
2432 }else{
2433 zSql = sqlite3_mprintf(
2434 "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' %s "
2435 "UNION ALL "
2436 "SELECT %s, rbu_control FROM '%q' "
2437 "%s %s typeof(rbu_control)='integer' AND rbu_control!=1 "
2438 "ORDER BY %s%s",
2439 zCollist, p->zStateDb, pIter->zDataTbl, zPart,
2440 zCollist, pIter->zDataTbl,
2441 zPart,
2442 (zPart ? "AND" : "WHERE"),
2443 zCollist, zLimit
2444 );
2445 }
2446 if( p->rc==SQLITE_OK ){
2447 p->rc = prepareFreeAndCollectError(p->dbRbu,&pIter->pSelect,pz,zSql);
2448 }else{
2449 sqlite3_free(zSql);
2450 }
2451 }
2452
2453 sqlite3_free(zImposterCols);
2454 sqlite3_free(zImposterPK);
2455 sqlite3_free(zWhere);
2456 sqlite3_free(zBind);
2457 sqlite3_free(zPart);
2458 }else{
2459 int bRbuRowid = (pIter->eType==RBU_PK_VTAB)
2460 ||(pIter->eType==RBU_PK_NONE)
2461 ||(pIter->eType==RBU_PK_EXTERNAL && rbuIsVacuum(p));
2462 const char *zTbl = pIter->zTbl; /* Table this step applies to */
2463 const char *zWrite; /* Imposter table name */
2464
2465 char *zBindings = rbuObjIterGetBindlist(p, pIter->nTblCol + bRbuRowid);
2466 char *zWhere = rbuObjIterGetWhere(p, pIter);
2467 char *zOldlist = rbuObjIterGetOldlist(p, pIter, "old");
2468 char *zNewlist = rbuObjIterGetOldlist(p, pIter, "new");
2469
2470 zCollist = rbuObjIterGetCollist(p, pIter);
2471 pIter->nCol = pIter->nTblCol;
2472
2473 /* Create the imposter table or tables (if required). */
2474 rbuCreateImposterTable(p, pIter);
2475 rbuCreateImposterTable2(p, pIter);
2476 zWrite = (pIter->eType==RBU_PK_VTAB ? "" : "rbu_imp_");
2477
2478 /* Create the INSERT statement to write to the target PK b-tree */
2479 if( p->rc==SQLITE_OK ){
2480 p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pInsert, pz,
2481 sqlite3_mprintf(
2482 "INSERT INTO \"%s%w\"(%s%s) VALUES(%s)",
2483 zWrite, zTbl, zCollist, (bRbuRowid ? ", _rowid_" : ""), zBindings
2484 )
2485 );
2486 }
2487
2488 /* Create the DELETE statement to write to the target PK b-tree.
2489 ** Because it only performs INSERT operations, this is not required for
2490 ** an rbu vacuum handle. */
2491 if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){
2492 p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pDelete, pz,
2493 sqlite3_mprintf(
2494 "DELETE FROM \"%s%w\" WHERE %s", zWrite, zTbl, zWhere
2495 )
2496 );
2497 }
2498
2499 if( rbuIsVacuum(p)==0 && pIter->abIndexed ){
2500 const char *zRbuRowid = "";
2501 if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){
2502 zRbuRowid = ", rbu_rowid";
2503 }
2504
2505 /* Create the rbu_tmp_xxx table and the triggers to populate it. */
2506 rbuMPrintfExec(p, p->dbRbu,
2507 "CREATE TABLE IF NOT EXISTS %s.'rbu_tmp_%q' AS "
2508 "SELECT *%s FROM '%q' WHERE 0;"
2509 , p->zStateDb, pIter->zDataTbl
2510 , (pIter->eType==RBU_PK_EXTERNAL ? ", 0 AS rbu_rowid" : "")
2511 , pIter->zDataTbl
2512 );
2513
2514 rbuMPrintfExec(p, p->dbMain,
2515 "CREATE TEMP TRIGGER rbu_delete_tr BEFORE DELETE ON \"%s%w\" "
2516 "BEGIN "
2517 " SELECT rbu_tmp_insert(3, %s);"
2518 "END;"
2519
2520 "CREATE TEMP TRIGGER rbu_update1_tr BEFORE UPDATE ON \"%s%w\" "
2521 "BEGIN "
2522 " SELECT rbu_tmp_insert(3, %s);"
2523 "END;"
2524
2525 "CREATE TEMP TRIGGER rbu_update2_tr AFTER UPDATE ON \"%s%w\" "
2526 "BEGIN "
2527 " SELECT rbu_tmp_insert(4, %s);"
2528 "END;",
2529 zWrite, zTbl, zOldlist,
2530 zWrite, zTbl, zOldlist,
2531 zWrite, zTbl, zNewlist
2532 );
2533
2534 if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){
2535 rbuMPrintfExec(p, p->dbMain,
2536 "CREATE TEMP TRIGGER rbu_insert_tr AFTER INSERT ON \"%s%w\" "
2537 "BEGIN "
2538 " SELECT rbu_tmp_insert(0, %s);"
2539 "END;",
2540 zWrite, zTbl, zNewlist
2541 );
2542 }
2543
2544 rbuObjIterPrepareTmpInsert(p, pIter, zCollist, zRbuRowid);
2545 }
2546
2547 /* Create the SELECT statement to read keys from data_xxx */
2548 if( p->rc==SQLITE_OK ){
2549 const char *zRbuRowid = "";
2550 char *zStart = 0;
2551 char *zOrder = 0;
2552 if( bRbuRowid ){
2553 zRbuRowid = rbuIsVacuum(p) ? ",_rowid_ " : ",rbu_rowid";
2554 }
2555
2556 if( rbuIsVacuum(p) ){
2557 if( nOffset ){
2558 zStart = rbuVacuumTableStart(p, pIter, bRbuRowid, zWrite);
2559 if( zStart ){
2560 sqlite3_free(zLimit);
2561 zLimit = 0;
2562 }
2563 }
2564 if( bRbuRowid ){
2565 zOrder = rbuMPrintf(p, "_rowid_");
2566 }else{
2567 zOrder = rbuObjIterGetPkList(p, pIter, "", ", ", "");
2568 }
2569 }
2570
2571 if( p->rc==SQLITE_OK ){
2572 p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz,
2573 sqlite3_mprintf(
2574 "SELECT %s,%s rbu_control%s FROM '%q'%s %s %s %s",
2575 zCollist,
2576 (rbuIsVacuum(p) ? "0 AS " : ""),
2577 zRbuRowid,
2578 pIter->zDataTbl, (zStart ? zStart : ""),
2579 (zOrder ? "ORDER BY" : ""), zOrder,
2580 zLimit
2581 )
2582 );
2583 }
2584 sqlite3_free(zStart);
2585 sqlite3_free(zOrder);
2586 }
2587
2588 sqlite3_free(zWhere);
2589 sqlite3_free(zOldlist);
2590 sqlite3_free(zNewlist);
2591 sqlite3_free(zBindings);
2592 }
2593 sqlite3_free(zCollist);
2594 sqlite3_free(zLimit);
2595 }
2596
2597 return p->rc;
2598 }
2599
2600 /*
2601 ** Set output variable *ppStmt to point to an UPDATE statement that may
2602 ** be used to update the imposter table for the main table b-tree of the
2603 ** table object that pIter currently points to, assuming that the
2604 ** rbu_control column of the data_xyz table contains zMask.
2605 **
2606 ** If the zMask string does not specify any columns to update, then this
2607 ** is not an error. Output variable *ppStmt is set to NULL in this case.
2608 */
2609 static int rbuGetUpdateStmt(
2610 sqlite3rbu *p, /* RBU handle */
2611 RbuObjIter *pIter, /* Object iterator */
2612 const char *zMask, /* rbu_control value ('x.x.') */
2613 sqlite3_stmt **ppStmt /* OUT: UPDATE statement handle */
2614 ){
2615 RbuUpdateStmt **pp;
2616 RbuUpdateStmt *pUp = 0;
2617 int nUp = 0;
2618
2619 /* In case an error occurs */
2620 *ppStmt = 0;
2621
2622 /* Search for an existing statement. If one is found, shift it to the front
2623 ** of the LRU queue and return immediately. Otherwise, leave nUp pointing
2624 ** to the number of statements currently in the cache and pUp to the
2625 ** last object in the list. */
2626 for(pp=&pIter->pRbuUpdate; *pp; pp=&((*pp)->pNext)){
2627 pUp = *pp;
2628 if( strcmp(pUp->zMask, zMask)==0 ){
2629 *pp = pUp->pNext;
2630 pUp->pNext = pIter->pRbuUpdate;
2631 pIter->pRbuUpdate = pUp;
2632 *ppStmt = pUp->pUpdate;
2633 return SQLITE_OK;
2634 }
2635 nUp++;
2636 }
2637 assert( pUp==0 || pUp->pNext==0 );
2638
2639 if( nUp>=SQLITE_RBU_UPDATE_CACHESIZE ){
2640 for(pp=&pIter->pRbuUpdate; *pp!=pUp; pp=&((*pp)->pNext));
2641 *pp = 0;
2642 sqlite3_finalize(pUp->pUpdate);
2643 pUp->pUpdate = 0;
2644 }else{
2645 pUp = (RbuUpdateStmt*)rbuMalloc(p, sizeof(RbuUpdateStmt)+pIter->nTblCol+1);
2646 }
2647
2648 if( pUp ){
2649 char *zWhere = rbuObjIterGetWhere(p, pIter);
2650 char *zSet = rbuObjIterGetSetlist(p, pIter, zMask);
2651 char *zUpdate = 0;
2652
2653 pUp->zMask = (char*)&pUp[1];
2654 memcpy(pUp->zMask, zMask, pIter->nTblCol);
2655 pUp->pNext = pIter->pRbuUpdate;
2656 pIter->pRbuUpdate = pUp;
2657
2658 if( zSet ){
2659 const char *zPrefix = "";
2660
2661 if( pIter->eType!=RBU_PK_VTAB ) zPrefix = "rbu_imp_";
2662 zUpdate = sqlite3_mprintf("UPDATE \"%s%w\" SET %s WHERE %s",
2663 zPrefix, pIter->zTbl, zSet, zWhere
2664 );
2665 p->rc = prepareFreeAndCollectError(
2666 p->dbMain, &pUp->pUpdate, &p->zErrmsg, zUpdate
2667 );
2668 *ppStmt = pUp->pUpdate;
2669 }
2670 sqlite3_free(zWhere);
2671 sqlite3_free(zSet);
2672 }
2673
2674 return p->rc;
2675 }
2676
2677 static sqlite3 *rbuOpenDbhandle(
2678 sqlite3rbu *p,
2679 const char *zName,
2680 int bUseVfs
2681 ){
2682 sqlite3 *db = 0;
2683 if( p->rc==SQLITE_OK ){
2684 const int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_URI;
2685 p->rc = sqlite3_open_v2(zName, &db, flags, bUseVfs ? p->zVfsName : 0);
2686 if( p->rc ){
2687 p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
2688 sqlite3_close(db);
2689 db = 0;
2690 }
2691 }
2692 return db;
2693 }
2694
2695 /*
2696 ** Free an RbuState object allocated by rbuLoadState().
2697 */
2698 static void rbuFreeState(RbuState *p){
2699 if( p ){
2700 sqlite3_free(p->zTbl);
2701 sqlite3_free(p->zDataTbl);
2702 sqlite3_free(p->zIdx);
2703 sqlite3_free(p);
2704 }
2705 }
2706
2707 /*
2708 ** Allocate an RbuState object and load the contents of the rbu_state
2709 ** table into it. Return a pointer to the new object. It is the
2710 ** responsibility of the caller to eventually free the object using
2711 ** sqlite3_free().
2712 **
2713 ** If an error occurs, leave an error code and message in the rbu handle
2714 ** and return NULL.
2715 */
2716 static RbuState *rbuLoadState(sqlite3rbu *p){
2717 RbuState *pRet = 0;
2718 sqlite3_stmt *pStmt = 0;
2719 int rc;
2720 int rc2;
2721
2722 pRet = (RbuState*)rbuMalloc(p, sizeof(RbuState));
2723 if( pRet==0 ) return 0;
2724
2725 rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg,
2726 sqlite3_mprintf("SELECT k, v FROM %s.rbu_state", p->zStateDb)
2727 );
2728 while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
2729 switch( sqlite3_column_int(pStmt, 0) ){
2730 case RBU_STATE_STAGE:
2731 pRet->eStage = sqlite3_column_int(pStmt, 1);
2732 if( pRet->eStage!=RBU_STAGE_OAL
2733 && pRet->eStage!=RBU_STAGE_MOVE
2734 && pRet->eStage!=RBU_STAGE_CKPT
2735 ){
2736 p->rc = SQLITE_CORRUPT;
2737 }
2738 break;
2739
2740 case RBU_STATE_TBL:
2741 pRet->zTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc);
2742 break;
2743
2744 case RBU_STATE_IDX:
2745 pRet->zIdx = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc);
2746 break;
2747
2748 case RBU_STATE_ROW:
2749 pRet->nRow = sqlite3_column_int(pStmt, 1);
2750 break;
2751
2752 case RBU_STATE_PROGRESS:
2753 pRet->nProgress = sqlite3_column_int64(pStmt, 1);
2754 break;
2755
2756 case RBU_STATE_CKPT:
2757 pRet->iWalCksum = sqlite3_column_int64(pStmt, 1);
2758 break;
2759
2760 case RBU_STATE_COOKIE:
2761 pRet->iCookie = (u32)sqlite3_column_int64(pStmt, 1);
2762 break;
2763
2764 case RBU_STATE_OALSZ:
2765 pRet->iOalSz = sqlite3_column_int64(pStmt, 1);
2766 break;
2767
2768 case RBU_STATE_PHASEONESTEP:
2769 pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1);
2770 break;
2771
2772 case RBU_STATE_DATATBL:
2773 pRet->zDataTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc);
2774 break;
2775
2776 default:
2777 rc = SQLITE_CORRUPT;
2778 break;
2779 }
2780 }
2781 rc2 = sqlite3_finalize(pStmt);
2782 if( rc==SQLITE_OK ) rc = rc2;
2783
2784 p->rc = rc;
2785 return pRet;
2786 }
2787
2788
2789 /*
2790 ** Open the database handle and attach the RBU database as "rbu". If an
2791 ** error occurs, leave an error code and message in the RBU handle.
2792 **
2793 ** If argument dbMain is not NULL, then it is a database handle already
2794 ** open on the target database. Use this handle instead of opening a new
2795 ** one.
2796 */
2797 static void rbuOpenDatabase(sqlite3rbu *p, sqlite3 *dbMain, int *pbRetry){
2798 assert( p->rc || (p->dbMain==0 && p->dbRbu==0) );
2799 assert( p->rc || rbuIsVacuum(p) || p->zTarget!=0 );
2800 assert( dbMain==0 || rbuIsVacuum(p)==0 );
2801
2802 /* Open the RBU database */
2803 p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1);
2804 p->dbMain = dbMain;
2805
2806 if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){
2807 sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p);
2808 if( p->zState==0 ){
2809 const char *zFile = sqlite3_db_filename(p->dbRbu, "main");
2810 p->zState = rbuMPrintf(p, "file:///%s-vacuum?modeof=%s", zFile, zFile);
2811 }
2812 }
2813
2814 /* If using separate RBU and state databases, attach the state database to
2815 ** the RBU db handle now. */
2816 if( p->zState ){
2817 rbuMPrintfExec(p, p->dbRbu, "ATTACH %Q AS stat", p->zState);
2818 memcpy(p->zStateDb, "stat", 4);
2819 }else{
2820 memcpy(p->zStateDb, "main", 4);
2821 }
2822
2823 #if 0
2824 if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){
2825 p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, 0);
2826 }
2827 #endif
2828
2829 /* If it has not already been created, create the rbu_state table */
2830 rbuMPrintfExec(p, p->dbRbu, RBU_CREATE_STATE, p->zStateDb);
2831
2832 #if 0
2833 if( rbuIsVacuum(p) ){
2834 if( p->rc==SQLITE_OK ){
2835 int rc2;
2836 int bOk = 0;
2837 sqlite3_stmt *pCnt = 0;
2838 p->rc = prepareAndCollectError(p->dbRbu, &pCnt, &p->zErrmsg,
2839 "SELECT count(*) FROM stat.sqlite_schema"
2840 );
2841 if( p->rc==SQLITE_OK
2842 && sqlite3_step(pCnt)==SQLITE_ROW
2843 && 1==sqlite3_column_int(pCnt, 0)
2844 ){
2845 bOk = 1;
2846 }
2847 rc2 = sqlite3_finalize(pCnt);
2848 if( p->rc==SQLITE_OK ) p->rc = rc2;
2849
2850 if( p->rc==SQLITE_OK && bOk==0 ){
2851 p->rc = SQLITE_ERROR;
2852 p->zErrmsg = sqlite3_mprintf("invalid state database");
2853 }
2854
2855 if( p->rc==SQLITE_OK ){
2856 p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, 0);
2857 }
2858 }
2859 }
2860 #endif
2861
2862 if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){
2863 int bOpen = 0;
2864 int rc;
2865 p->nRbu = 0;
2866 p->pRbuFd = 0;
2867 rc = sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p);
2868 if( rc!=SQLITE_NOTFOUND ) p->rc = rc;
2869 if( p->eStage>=RBU_STAGE_MOVE ){
2870 bOpen = 1;
2871 }else{
2872 RbuState *pState = rbuLoadState(p);
2873 if( pState ){
2874 bOpen = (pState->eStage>=RBU_STAGE_MOVE);
2875 rbuFreeState(pState);
2876 }
2877 }
2878 if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, p->nRbu<=1);
2879 }
2880
2881 p->eStage = 0;
2882 if( p->rc==SQLITE_OK && p->dbMain==0 ){
2883 if( !rbuIsVacuum(p) ){
2884 p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1);
2885 }else if( p->pRbuFd->pWalFd ){
2886 if( pbRetry ){
2887 p->pRbuFd->bNolock = 0;
2888 sqlite3_close(p->dbRbu);
2889 sqlite3_close(p->dbMain);
2890 p->dbMain = 0;
2891 p->dbRbu = 0;
2892 *pbRetry = 1;
2893 return;
2894 }
2895 p->rc = SQLITE_ERROR;
2896 p->zErrmsg = sqlite3_mprintf("cannot vacuum wal mode database");
2897 }else{
2898 char *zTarget;
2899 char *zExtra = 0;
2900 if( strlen(p->zRbu)>=5 && 0==memcmp("file:", p->zRbu, 5) ){
2901 zExtra = &p->zRbu[5];
2902 while( *zExtra ){
2903 if( *zExtra++=='?' ) break;
2904 }
2905 if( *zExtra=='\0' ) zExtra = 0;
2906 }
2907
2908 zTarget = sqlite3_mprintf("file:%s-vactmp?rbu_memory=1%s%s",
2909 sqlite3_db_filename(p->dbRbu, "main"),
2910 (zExtra==0 ? "" : "&"), (zExtra==0 ? "" : zExtra)
2911 );
2912
2913 if( zTarget==0 ){
2914 p->rc = SQLITE_NOMEM;
2915 return;
2916 }
2917 p->dbMain = rbuOpenDbhandle(p, zTarget, p->nRbu<=1);
2918 sqlite3_free(zTarget);
2919 }
2920 }
2921
2922 if( p->rc==SQLITE_OK ){
2923 p->rc = sqlite3_create_function(p->dbMain,
2924 "rbu_tmp_insert", -1, SQLITE_UTF8, (void*)p, rbuTmpInsertFunc, 0, 0
2925 );
2926 }
2927
2928 if( p->rc==SQLITE_OK ){
2929 p->rc = sqlite3_create_function(p->dbMain,
2930 "rbu_fossil_delta", 2, SQLITE_UTF8, 0, rbuFossilDeltaFunc, 0, 0
2931 );
2932 }
2933
2934 if( p->rc==SQLITE_OK ){
2935 p->rc = sqlite3_create_function(p->dbRbu,
2936 "rbu_target_name", -1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0
2937 );
2938 }
2939
2940 if( p->rc==SQLITE_OK ){
2941 p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p);
2942 }
2943 rbuMPrintfExec(p, p->dbMain, "SELECT * FROM sqlite_schema");
2944
2945 /* Mark the database file just opened as an RBU target database. If
2946 ** this call returns SQLITE_NOTFOUND, then the RBU vfs is not in use.
2947 ** This is an error. */
2948 if( p->rc==SQLITE_OK ){
2949 p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p);
2950 }
2951
2952 if( p->rc==SQLITE_NOTFOUND ){
2953 p->rc = SQLITE_ERROR;
2954 p->zErrmsg = sqlite3_mprintf("rbu vfs not found");
2955 }
2956 }
2957
2958 /*
2959 ** This routine is a copy of the sqlite3FileSuffix3() routine from the core.
2960 ** It is a no-op unless SQLITE_ENABLE_8_3_NAMES is defined.
2961 **
2962 ** If SQLITE_ENABLE_8_3_NAMES is set at compile-time and if the database
2963 ** filename in zBaseFilename is a URI with the "8_3_names=1" parameter and
2964 ** if filename in z[] has a suffix (a.k.a. "extension") that is longer than
2965 ** three characters, then shorten the suffix on z[] to be the last three
2966 ** characters of the original suffix.
2967 **
2968 ** If SQLITE_ENABLE_8_3_NAMES is set to 2 at compile-time, then always
2969 ** do the suffix shortening regardless of URI parameter.
2970 **
2971 ** Examples:
2972 **
2973 ** test.db-journal => test.nal
2974 ** test.db-wal => test.wal
2975 ** test.db-shm => test.shm
2976 ** test.db-mj7f3319fa => test.9fa
2977 */
2978 static void rbuFileSuffix3(const char *zBase, char *z){
2979 #ifdef SQLITE_ENABLE_8_3_NAMES
2980 #if SQLITE_ENABLE_8_3_NAMES<2
2981 if( sqlite3_uri_boolean(zBase, "8_3_names", 0) )
2982 #endif
2983 {
2984 int i, sz;
2985 sz = (int)strlen(z)&0xffffff;
2986 for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){}
2987 if( z[i]=='.' && sz>i+4 ) memmove(&z[i+1], &z[sz-3], 4);
2988 }
2989 #else
2990 UNUSED_PARAMETER2(zBase,z);
2991 #endif
2992 }
2993
2994 /*
2995 ** Return the current wal-index header checksum for the target database
2996 ** as a 64-bit integer.
2997 **
2998 ** The checksum is store in the first page of xShmMap memory as an 8-byte
2999 ** blob starting at byte offset 40.
3000 */
3001 static i64 rbuShmChecksum(sqlite3rbu *p){
3002 i64 iRet = 0;
3003 if( p->rc==SQLITE_OK ){
3004 sqlite3_file *pDb = p->pTargetFd->pReal;
3005 u32 volatile *ptr;
3006 p->rc = pDb->pMethods->xShmMap(pDb, 0, 32*1024, 0, (void volatile**)&ptr);
3007 if( p->rc==SQLITE_OK ){
3008 iRet = (i64)(((u64)ptr[10] << 32) + ptr[11]);
3009 }
3010 }
3011 return iRet;
3012 }
3013
3014 /*
3015 ** This function is called as part of initializing or reinitializing an
3016 ** incremental checkpoint.
3017 **
3018 ** It populates the sqlite3rbu.aFrame[] array with the set of
3019 ** (wal frame -> db page) copy operations required to checkpoint the
3020 ** current wal file, and obtains the set of shm locks required to safely
3021 ** perform the copy operations directly on the file-system.
3022 **
3023 ** If argument pState is not NULL, then the incremental checkpoint is
3024 ** being resumed. In this case, if the checksum of the wal-index-header
3025 ** following recovery is not the same as the checksum saved in the RbuState
3026 ** object, then the rbu handle is set to DONE state. This occurs if some
3027 ** other client appends a transaction to the wal file in the middle of
3028 ** an incremental checkpoint.
3029 */
3030 static void rbuSetupCheckpoint(sqlite3rbu *p, RbuState *pState){
3031
3032 /* If pState is NULL, then the wal file may not have been opened and
3033 ** recovered. Running a read-statement here to ensure that doing so
3034 ** does not interfere with the "capture" process below. */
3035 if( pState==0 ){
3036 p->eStage = 0;
3037 if( p->rc==SQLITE_OK ){
3038 p->rc = sqlite3_exec(p->dbMain, "SELECT * FROM sqlite_schema", 0, 0, 0);
3039 }
3040 }
3041
3042 /* Assuming no error has occurred, run a "restart" checkpoint with the
3043 ** sqlite3rbu.eStage variable set to CAPTURE. This turns on the following
3044 ** special behaviour in the rbu VFS:
3045 **
3046 ** * If the exclusive shm WRITER or READ0 lock cannot be obtained,
3047 ** the checkpoint fails with SQLITE_BUSY (normally SQLite would
3048 ** proceed with running a passive checkpoint instead of failing).
3049 **
3050 ** * Attempts to read from the *-wal file or write to the database file
3051 ** do not perform any IO. Instead, the frame/page combinations that
3052 ** would be read/written are recorded in the sqlite3rbu.aFrame[]
3053 ** array.
3054 **
3055 ** * Calls to xShmLock(UNLOCK) to release the exclusive shm WRITER,
3056 ** READ0 and CHECKPOINT locks taken as part of the checkpoint are
3057 ** no-ops. These locks will not be released until the connection
3058 ** is closed.
3059 **
3060 ** * Attempting to xSync() the database file causes an SQLITE_NOTICE
3061 ** error.
3062 **
3063 ** As a result, unless an error (i.e. OOM or SQLITE_BUSY) occurs, the
3064 ** checkpoint below fails with SQLITE_NOTICE, and leaves the aFrame[]
3065 ** array populated with a set of (frame -> page) mappings. Because the
3066 ** WRITER, CHECKPOINT and READ0 locks are still held, it is safe to copy
3067 ** data from the wal file into the database file according to the
3068 ** contents of aFrame[].
3069 */
3070 if( p->rc==SQLITE_OK ){
3071 int rc2;
3072 p->eStage = RBU_STAGE_CAPTURE;
3073 rc2 = sqlite3_exec(p->dbMain, "PRAGMA main.wal_checkpoint=restart", 0, 0,0);
3074 if( rc2!=SQLITE_NOTICE ) p->rc = rc2;
3075 }
3076
3077 if( p->rc==SQLITE_OK && p->nFrame>0 ){
3078 p->eStage = RBU_STAGE_CKPT;
3079 p->nStep = (pState ? pState->nRow : 0);
3080 p->aBuf = rbuMalloc(p, p->pgsz);
3081 p->iWalCksum = rbuShmChecksum(p);
3082 }
3083
3084 if( p->rc==SQLITE_OK ){
3085 if( p->nFrame==0 || (pState && pState->iWalCksum!=p->iWalCksum) ){
3086 p->rc = SQLITE_DONE;
3087 p->eStage = RBU_STAGE_DONE;
3088 }else{
3089 int nSectorSize;
3090 sqlite3_file *pDb = p->pTargetFd->pReal;
3091 sqlite3_file *pWal = p->pTargetFd->pWalFd->pReal;
3092 assert( p->nPagePerSector==0 );
3093 nSectorSize = pDb->pMethods->xSectorSize(pDb);
3094 if( nSectorSize>p->pgsz ){
3095 p->nPagePerSector = nSectorSize / p->pgsz;
3096 }else{
3097 p->nPagePerSector = 1;
3098 }
3099
3100 /* Call xSync() on the wal file. This causes SQLite to sync the
3101 ** directory in which the target database and the wal file reside, in
3102 ** case it has not been synced since the rename() call in
3103 ** rbuMoveOalFile(). */
3104 p->rc = pWal->pMethods->xSync(pWal, SQLITE_SYNC_NORMAL);
3105 }
3106 }
3107 }
3108
3109 /*
3110 ** Called when iAmt bytes are read from offset iOff of the wal file while
3111 ** the rbu object is in capture mode. Record the frame number of the frame
3112 ** being read in the aFrame[] array.
3113 */
3114 static int rbuCaptureWalRead(sqlite3rbu *pRbu, i64 iOff, int iAmt){
3115 const u32 mReq = (1<<WAL_LOCK_WRITE)|(1<<WAL_LOCK_CKPT)|(1<<WAL_LOCK_READ0);
3116 u32 iFrame;
3117
3118 if( pRbu->mLock!=mReq ){
3119 pRbu->rc = SQLITE_BUSY;
3120 return SQLITE_NOTICE_RBU;
3121 }
3122
3123 pRbu->pgsz = iAmt;
3124 if( pRbu->nFrame==pRbu->nFrameAlloc ){
3125 int nNew = (pRbu->nFrameAlloc ? pRbu->nFrameAlloc : 64) * 2;
3126 RbuFrame *aNew;
3127 aNew = (RbuFrame*)sqlite3_realloc64(pRbu->aFrame, nNew * sizeof(RbuFrame));
3128 if( aNew==0 ) return SQLITE_NOMEM;
3129 pRbu->aFrame = aNew;
3130 pRbu->nFrameAlloc = nNew;
3131 }
3132
3133 iFrame = (u32)((iOff-32) / (i64)(iAmt+24)) + 1;
3134 if( pRbu->iMaxFrame<iFrame ) pRbu->iMaxFrame = iFrame;
3135 pRbu->aFrame[pRbu->nFrame].iWalFrame = iFrame;
3136 pRbu->aFrame[pRbu->nFrame].iDbPage = 0;
3137 pRbu->nFrame++;
3138 return SQLITE_OK;
3139 }
3140
3141 /*
3142 ** Called when a page of data is written to offset iOff of the database
3143 ** file while the rbu handle is in capture mode. Record the page number
3144 ** of the page being written in the aFrame[] array.
3145 */
3146 static int rbuCaptureDbWrite(sqlite3rbu *pRbu, i64 iOff){
3147 pRbu->aFrame[pRbu->nFrame-1].iDbPage = (u32)(iOff / pRbu->pgsz) + 1;
3148 return SQLITE_OK;
3149 }
3150
3151 /*
3152 ** This is called as part of an incremental checkpoint operation. Copy
3153 ** a single frame of data from the wal file into the database file, as
3154 ** indicated by the RbuFrame object.
3155 */
3156 static void rbuCheckpointFrame(sqlite3rbu *p, RbuFrame *pFrame){
3157 sqlite3_file *pWal = p->pTargetFd->pWalFd->pReal;
3158 sqlite3_file *pDb = p->pTargetFd->pReal;
3159 i64 iOff;
3160
3161 assert( p->rc==SQLITE_OK );
3162 iOff = (i64)(pFrame->iWalFrame-1) * (p->pgsz + 24) + 32 + 24;
3163 p->rc = pWal->pMethods->xRead(pWal, p->aBuf, p->pgsz, iOff);
3164 if( p->rc ) return;
3165
3166 iOff = (i64)(pFrame->iDbPage-1) * p->pgsz;
3167 p->rc = pDb->pMethods->xWrite(pDb, p->aBuf, p->pgsz, iOff);
3168 }
3169
3170 /*
3171 ** This value is copied from the definition of ZIPVFS_CTRL_FILE_POINTER
3172 ** in zipvfs.h.
3173 */
3174 #define RBU_ZIPVFS_CTRL_FILE_POINTER 230439
3175
3176 /*
3177 ** Take an EXCLUSIVE lock on the database file. Return SQLITE_OK if
3178 ** successful, or an SQLite error code otherwise.
3179 */
3180 static int rbuLockDatabase(sqlite3 *db){
3181 int rc = SQLITE_OK;
3182 sqlite3_file *fd = 0;
3183
3184 sqlite3_file_control(db, "main", RBU_ZIPVFS_CTRL_FILE_POINTER, &fd);
3185 if( fd ){
3186 sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, &fd);
3187 rc = fd->pMethods->xLock(fd, SQLITE_LOCK_SHARED);
3188 if( rc==SQLITE_OK ){
3189 rc = fd->pMethods->xUnlock(fd, SQLITE_LOCK_NONE);
3190 }
3191 sqlite3_file_control(db, "main", RBU_ZIPVFS_CTRL_FILE_POINTER, &fd);
3192 }else{
3193 sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, &fd);
3194 }
3195
3196 if( rc==SQLITE_OK && fd->pMethods ){
3197 rc = fd->pMethods->xLock(fd, SQLITE_LOCK_SHARED);
3198 if( rc==SQLITE_OK ){
3199 rc = fd->pMethods->xLock(fd, SQLITE_LOCK_EXCLUSIVE);
3200 }
3201 }
3202 return rc;
3203 }
3204
3205 /*
3206 ** Return true if the database handle passed as the only argument
3207 ** was opened with the rbu_exclusive_checkpoint=1 URI parameter
3208 ** specified. Or false otherwise.
3209 */
3210 static int rbuExclusiveCheckpoint(sqlite3 *db){
3211 const char *zUri = sqlite3_db_filename(db, 0);
3212 return sqlite3_uri_boolean(zUri, RBU_EXCLUSIVE_CHECKPOINT, 0);
3213 }
3214
3215 #if defined(_WIN32_WCE)
3216 static LPWSTR rbuWinUtf8ToUnicode(const char *zFilename){
3217 int nChar;
3218 LPWSTR zWideFilename;
3219
3220 nChar = MultiByteToWideChar(CP_UTF8, 0, zFilename, -1, NULL, 0);
3221 if( nChar==0 ){
3222 return 0;
3223 }
3224 zWideFilename = sqlite3_malloc64( nChar*sizeof(zWideFilename[0]) );
3225 if( zWideFilename==0 ){
3226 return 0;
3227 }
3228 memset(zWideFilename, 0, nChar*sizeof(zWideFilename[0]));
3229 nChar = MultiByteToWideChar(CP_UTF8, 0, zFilename, -1, zWideFilename,
3230 nChar);
3231 if( nChar==0 ){
3232 sqlite3_free(zWideFilename);
3233 zWideFilename = 0;
3234 }
3235 return zWideFilename;
3236 }
3237 #endif
3238
3239 /*
3240 ** The RBU handle is currently in RBU_STAGE_OAL state, with a SHARED lock
3241 ** on the database file. This proc moves the *-oal file to the *-wal path,
3242 ** then reopens the database file (this time in vanilla, non-oal, WAL mode).
3243 ** If an error occurs, leave an error code and error message in the rbu
3244 ** handle.
3245 */
3246 static void rbuMoveOalFile(sqlite3rbu *p){
3247 const char *zBase = sqlite3_db_filename(p->dbMain, "main");
3248 const char *zMove = zBase;
3249 char *zOal;
3250 char *zWal;
3251
3252 if( rbuIsVacuum(p) ){
3253 zMove = sqlite3_db_filename(p->dbRbu, "main");
3254 }
3255 zOal = sqlite3_mprintf("%s-oal", zMove);
3256 zWal = sqlite3_mprintf("%s-wal", zMove);
3257
3258 assert( p->eStage==RBU_STAGE_MOVE );
3259 assert( p->rc==SQLITE_OK && p->zErrmsg==0 );
3260 if( zWal==0 || zOal==0 ){
3261 p->rc = SQLITE_NOMEM;
3262 }else{
3263 /* Move the *-oal file to *-wal. At this point connection p->db is
3264 ** holding a SHARED lock on the target database file (because it is
3265 ** in WAL mode). So no other connection may be writing the db.
3266 **
3267 ** In order to ensure that there are no database readers, an EXCLUSIVE
3268 ** lock is obtained here before the *-oal is moved to *-wal.
3269 */
3270 sqlite3 *dbMain = 0;
3271 rbuFileSuffix3(zBase, zWal);
3272 rbuFileSuffix3(zBase, zOal);
3273
3274 /* Re-open the databases. */
3275 rbuObjIterFinalize(&p->objiter);
3276 sqlite3_close(p->dbRbu);
3277 sqlite3_close(p->dbMain);
3278 p->dbMain = 0;
3279 p->dbRbu = 0;
3280
3281 dbMain = rbuOpenDbhandle(p, p->zTarget, 1);
3282 if( dbMain ){
3283 assert( p->rc==SQLITE_OK );
3284 p->rc = rbuLockDatabase(dbMain);
3285 }
3286
3287 if( p->rc==SQLITE_OK ){
3288 p->rc = p->xRename(p->pRenameArg, zOal, zWal);
3289 }
3290
3291 if( p->rc!=SQLITE_OK
3292 || rbuIsVacuum(p)
3293 || rbuExclusiveCheckpoint(dbMain)==0
3294 ){
3295 sqlite3_close(dbMain);
3296 dbMain = 0;
3297 }
3298
3299 if( p->rc==SQLITE_OK ){
3300 rbuOpenDatabase(p, dbMain, 0);
3301 rbuSetupCheckpoint(p, 0);
3302 }
3303 }
3304
3305 sqlite3_free(zWal);
3306 sqlite3_free(zOal);
3307 }
3308
3309 /*
3310 ** The SELECT statement iterating through the keys for the current object
3311 ** (p->objiter.pSelect) currently points to a valid row. This function
3312 ** determines the type of operation requested by this row and returns
3313 ** one of the following values to indicate the result:
3314 **
3315 ** * RBU_INSERT
3316 ** * RBU_DELETE
3317 ** * RBU_IDX_DELETE
3318 ** * RBU_UPDATE
3319 **
3320 ** If RBU_UPDATE is returned, then output variable *pzMask is set to
3321 ** point to the text value indicating the columns to update.
3322 **
3323 ** If the rbu_control field contains an invalid value, an error code and
3324 ** message are left in the RBU handle and zero returned.
3325 */
3326 static int rbuStepType(sqlite3rbu *p, const char **pzMask){
3327 int iCol = p->objiter.nCol; /* Index of rbu_control column */
3328 int res = 0; /* Return value */
3329
3330 switch( sqlite3_column_type(p->objiter.pSelect, iCol) ){
3331 case SQLITE_INTEGER: {
3332 int iVal = sqlite3_column_int(p->objiter.pSelect, iCol);
3333 switch( iVal ){
3334 case 0: res = RBU_INSERT; break;
3335 case 1: res = RBU_DELETE; break;
3336 case 2: res = RBU_REPLACE; break;
3337 case 3: res = RBU_IDX_DELETE; break;
3338 case 4: res = RBU_IDX_INSERT; break;
3339 }
3340 break;
3341 }
3342
3343 case SQLITE_TEXT: {
3344 const unsigned char *z = sqlite3_column_text(p->objiter.pSelect, iCol);
3345 if( z==0 ){
3346 p->rc = SQLITE_NOMEM;
3347 }else{
3348 *pzMask = (const char*)z;
3349 }
3350 res = RBU_UPDATE;
3351
3352 break;
3353 }
3354
3355 default:
3356 break;
3357 }
3358
3359 if( res==0 ){
3360 rbuBadControlError(p);
3361 }
3362 return res;
3363 }
3364
3365 #ifdef SQLITE_DEBUG
3366 /*
3367 ** Assert that column iCol of statement pStmt is named zName.
3368 */
3369 static void assertColumnName(sqlite3_stmt *pStmt, int iCol, const char *zName){
3370 const char *zCol = sqlite3_column_name(pStmt, iCol);
3371 assert( 0==sqlite3_stricmp(zName, zCol) );
3372 }
3373 #else
3374 # define assertColumnName(x,y,z)
3375 #endif
3376
3377 /*
3378 ** Argument eType must be one of RBU_INSERT, RBU_DELETE, RBU_IDX_INSERT or
3379 ** RBU_IDX_DELETE. This function performs the work of a single
3380 ** sqlite3rbu_step() call for the type of operation specified by eType.
3381 */
3382 static void rbuStepOneOp(sqlite3rbu *p, int eType){
3383 RbuObjIter *pIter = &p->objiter;
3384 sqlite3_value *pVal;
3385 sqlite3_stmt *pWriter;
3386 int i;
3387
3388 assert( p->rc==SQLITE_OK );
3389 assert( eType!=RBU_DELETE || pIter->zIdx==0 );
3390 assert( eType==RBU_DELETE || eType==RBU_IDX_DELETE
3391 || eType==RBU_INSERT || eType==RBU_IDX_INSERT
3392 );
3393
3394 /* If this is a delete, decrement nPhaseOneStep by nIndex. If the DELETE
3395 ** statement below does actually delete a row, nPhaseOneStep will be
3396 ** incremented by the same amount when SQL function rbu_tmp_insert()
3397 ** is invoked by the trigger. */
3398 if( eType==RBU_DELETE ){
3399 p->nPhaseOneStep -= p->objiter.nIndex;
3400 }
3401
3402 if( eType==RBU_IDX_DELETE || eType==RBU_DELETE ){
3403 pWriter = pIter->pDelete;
3404 }else{
3405 pWriter = pIter->pInsert;
3406 }
3407
3408 for(i=0; i<pIter->nCol; i++){
3409 /* If this is an INSERT into a table b-tree and the table has an
3410 ** explicit INTEGER PRIMARY KEY, check that this is not an attempt
3411 ** to write a NULL into the IPK column. That is not permitted. */
3412 if( eType==RBU_INSERT
3413 && pIter->zIdx==0 && pIter->eType==RBU_PK_IPK && pIter->abTblPk[i]
3414 && sqlite3_column_type(pIter->pSelect, i)==SQLITE_NULL
3415 ){
3416 p->rc = SQLITE_MISMATCH;
3417 p->zErrmsg = sqlite3_mprintf("datatype mismatch");
3418 return;
3419 }
3420
3421 if( eType==RBU_DELETE && pIter->abTblPk[i]==0 ){
3422 continue;
3423 }
3424
3425 pVal = sqlite3_column_value(pIter->pSelect, i);
3426 p->rc = sqlite3_bind_value(pWriter, i+1, pVal);
3427 if( p->rc ) return;
3428 }
3429 if( pIter->zIdx==0 ){
3430 if( pIter->eType==RBU_PK_VTAB
3431 || pIter->eType==RBU_PK_NONE
3432 || (pIter->eType==RBU_PK_EXTERNAL && rbuIsVacuum(p))
3433 ){
3434 /* For a virtual table, or a table with no primary key, the
3435 ** SELECT statement is:
3436 **
3437 ** SELECT <cols>, rbu_control, rbu_rowid FROM ....
3438 **
3439 ** Hence column_value(pIter->nCol+1).
3440 */
3441 assertColumnName(pIter->pSelect, pIter->nCol+1,
3442 rbuIsVacuum(p) ? "rowid" : "rbu_rowid"
3443 );
3444 pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1);
3445 p->rc = sqlite3_bind_value(pWriter, pIter->nCol+1, pVal);
3446 }
3447 }
3448 if( p->rc==SQLITE_OK ){
3449 sqlite3_step(pWriter);
3450 p->rc = resetAndCollectError(pWriter, &p->zErrmsg);
3451 }
3452 }
3453
3454 /*
3455 ** This function does the work for an sqlite3rbu_step() call.
3456 **
3457 ** The object-iterator (p->objiter) currently points to a valid object,
3458 ** and the input cursor (p->objiter.pSelect) currently points to a valid
3459 ** input row. Perform whatever processing is required and return.
3460 **
3461 ** If no error occurs, SQLITE_OK is returned. Otherwise, an error code
3462 ** and message is left in the RBU handle and a copy of the error code
3463 ** returned.
3464 */
3465 static int rbuStep(sqlite3rbu *p){
3466 RbuObjIter *pIter = &p->objiter;
3467 const char *zMask = 0;
3468 int eType = rbuStepType(p, &zMask);
3469
3470 if( eType ){
3471 assert( eType==RBU_INSERT || eType==RBU_DELETE
3472 || eType==RBU_REPLACE || eType==RBU_IDX_DELETE
3473 || eType==RBU_IDX_INSERT || eType==RBU_UPDATE
3474 );
3475 assert( eType!=RBU_UPDATE || pIter->zIdx==0 );
3476
3477 if( pIter->zIdx==0 && (eType==RBU_IDX_DELETE || eType==RBU_IDX_INSERT) ){
3478 rbuBadControlError(p);
3479 }
3480 else if( eType==RBU_REPLACE ){
3481 if( pIter->zIdx==0 ){
3482 p->nPhaseOneStep += p->objiter.nIndex;
3483 rbuStepOneOp(p, RBU_DELETE);
3484 }
3485 if( p->rc==SQLITE_OK ) rbuStepOneOp(p, RBU_INSERT);
3486 }
3487 else if( eType!=RBU_UPDATE ){
3488 rbuStepOneOp(p, eType);
3489 }
3490 else{
3491 sqlite3_value *pVal;
3492 sqlite3_stmt *pUpdate = 0;
3493 assert( eType==RBU_UPDATE );
3494 p->nPhaseOneStep -= p->objiter.nIndex;
3495 rbuGetUpdateStmt(p, pIter, zMask, &pUpdate);
3496 if( pUpdate ){
3497 int i;
3498 for(i=0; p->rc==SQLITE_OK && i<pIter->nCol; i++){
3499 char c = zMask[pIter->aiSrcOrder[i]];
3500 pVal = sqlite3_column_value(pIter->pSelect, i);
3501 if( pIter->abTblPk[i] || c!='.' ){
3502 p->rc = sqlite3_bind_value(pUpdate, i+1, pVal);
3503 }
3504 }
3505 if( p->rc==SQLITE_OK
3506 && (pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE)
3507 ){
3508 /* Bind the rbu_rowid value to column _rowid_ */
3509 assertColumnName(pIter->pSelect, pIter->nCol+1, "rbu_rowid");
3510 pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1);
3511 p->rc = sqlite3_bind_value(pUpdate, pIter->nCol+1, pVal);
3512 }
3513 if( p->rc==SQLITE_OK ){
3514 sqlite3_step(pUpdate);
3515 p->rc = resetAndCollectError(pUpdate, &p->zErrmsg);
3516 }
3517 }
3518 }
3519 }
3520 return p->rc;
3521 }
3522
3523 /*
3524 ** Increment the schema cookie of the main database opened by p->dbMain.
3525 **
3526 ** Or, if this is an RBU vacuum, set the schema cookie of the main db
3527 ** opened by p->dbMain to one more than the schema cookie of the main
3528 ** db opened by p->dbRbu.
3529 */
3530 static void rbuIncrSchemaCookie(sqlite3rbu *p){
3531 if( p->rc==SQLITE_OK ){
3532 sqlite3 *dbread = (rbuIsVacuum(p) ? p->dbRbu : p->dbMain);
3533 int iCookie = 1000000;
3534 sqlite3_stmt *pStmt;
3535
3536 p->rc = prepareAndCollectError(dbread, &pStmt, &p->zErrmsg,
3537 "PRAGMA schema_version"
3538 );
3539 if( p->rc==SQLITE_OK ){
3540 /* Coverage: it may be that this sqlite3_step() cannot fail. There
3541 ** is already a transaction open, so the prepared statement cannot
3542 ** throw an SQLITE_SCHEMA exception. The only database page the
3543 ** statement reads is page 1, which is guaranteed to be in the cache.
3544 ** And no memory allocations are required. */
3545 if( SQLITE_ROW==sqlite3_step(pStmt) ){
3546 iCookie = sqlite3_column_int(pStmt, 0);
3547 }
3548 rbuFinalize(p, pStmt);
3549 }
3550 if( p->rc==SQLITE_OK ){
3551 rbuMPrintfExec(p, p->dbMain, "PRAGMA schema_version = %d", iCookie+1);
3552 }
3553 }
3554 }
3555
3556 /*
3557 ** Update the contents of the rbu_state table within the rbu database. The
3558 ** value stored in the RBU_STATE_STAGE column is eStage. All other values
3559 ** are determined by inspecting the rbu handle passed as the first argument.
3560 */
3561 static void rbuSaveState(sqlite3rbu *p, int eStage){
3562 if( p->rc==SQLITE_OK || p->rc==SQLITE_DONE ){
3563 sqlite3_stmt *pInsert = 0;
3564 rbu_file *pFd = (rbuIsVacuum(p) ? p->pRbuFd : p->pTargetFd);
3565 int rc;
3566
3567 assert( p->zErrmsg==0 );
3568 rc = prepareFreeAndCollectError(p->dbRbu, &pInsert, &p->zErrmsg,
3569 sqlite3_mprintf(
3570 "INSERT OR REPLACE INTO %s.rbu_state(k, v) VALUES "
3571 "(%d, %d), "
3572 "(%d, %Q), "
3573 "(%d, %Q), "
3574 "(%d, %d), "
3575 "(%d, %lld), "
3576 "(%d, %lld), "
3577 "(%d, %lld), "
3578 "(%d, %lld), "
3579 "(%d, %lld), "
3580 "(%d, %Q) ",
3581 p->zStateDb,
3582 RBU_STATE_STAGE, eStage,
3583 RBU_STATE_TBL, p->objiter.zTbl,
3584 RBU_STATE_IDX, p->objiter.zIdx,
3585 RBU_STATE_ROW, p->nStep,
3586 RBU_STATE_PROGRESS, p->nProgress,
3587 RBU_STATE_CKPT, p->iWalCksum,
3588 RBU_STATE_COOKIE, (i64)pFd->iCookie,
3589 RBU_STATE_OALSZ, p->iOalSz,
3590 RBU_STATE_PHASEONESTEP, p->nPhaseOneStep,
3591 RBU_STATE_DATATBL, p->objiter.zDataTbl
3592 )
3593 );
3594 assert( pInsert==0 || rc==SQLITE_OK );
3595
3596 if( rc==SQLITE_OK ){
3597 sqlite3_step(pInsert);
3598 rc = sqlite3_finalize(pInsert);
3599 }
3600 if( rc!=SQLITE_OK ) p->rc = rc;
3601 }
3602 }
3603
3604
3605 /*
3606 ** The second argument passed to this function is the name of a PRAGMA
3607 ** setting - "page_size", "auto_vacuum", "user_version" or "application_id".
3608 ** This function executes the following on sqlite3rbu.dbRbu:
3609 **
3610 ** "PRAGMA main.$zPragma"
3611 **
3612 ** where $zPragma is the string passed as the second argument, then
3613 ** on sqlite3rbu.dbMain:
3614 **
3615 ** "PRAGMA main.$zPragma = $val"
3616 **
3617 ** where $val is the value returned by the first PRAGMA invocation.
3618 **
3619 ** In short, it copies the value of the specified PRAGMA setting from
3620 ** dbRbu to dbMain.
3621 */
3622 static void rbuCopyPragma(sqlite3rbu *p, const char *zPragma){
3623 if( p->rc==SQLITE_OK ){
3624 sqlite3_stmt *pPragma = 0;
3625 p->rc = prepareFreeAndCollectError(p->dbRbu, &pPragma, &p->zErrmsg,
3626 sqlite3_mprintf("PRAGMA main.%s", zPragma)
3627 );
3628 if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPragma) ){
3629 p->rc = rbuMPrintfExec(p, p->dbMain, "PRAGMA main.%s = %d",
3630 zPragma, sqlite3_column_int(pPragma, 0)
3631 );
3632 }
3633 rbuFinalize(p, pPragma);
3634 }
3635 }
3636
3637 /*
3638 ** The RBU handle passed as the only argument has just been opened and
3639 ** the state database is empty. If this RBU handle was opened for an
3640 ** RBU vacuum operation, create the schema in the target db.
3641 */
3642 static void rbuCreateTargetSchema(sqlite3rbu *p){
3643 sqlite3_stmt *pSql = 0;
3644 sqlite3_stmt *pInsert = 0;
3645
3646 assert( rbuIsVacuum(p) );
3647 p->rc = sqlite3_exec(p->dbMain, "PRAGMA writable_schema=1", 0,0, &p->zErrmsg);
3648 if( p->rc==SQLITE_OK ){
3649 p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg,
3650 "SELECT sql FROM sqlite_schema WHERE sql!='' AND rootpage!=0"
3651 " AND name!='sqlite_sequence' "
3652 " ORDER BY type DESC"
3653 );
3654 }
3655
3656 while( p->rc==SQLITE_OK && sqlite3_step(pSql)==SQLITE_ROW ){
3657 const char *zSql = (const char*)sqlite3_column_text(pSql, 0);
3658 p->rc = sqlite3_exec(p->dbMain, zSql, 0, 0, &p->zErrmsg);
3659 }
3660 rbuFinalize(p, pSql);
3661 if( p->rc!=SQLITE_OK ) return;
3662
3663 if( p->rc==SQLITE_OK ){
3664 p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg,
3665 "SELECT * FROM sqlite_schema WHERE rootpage=0 OR rootpage IS NULL"
3666 );
3667 }
3668
3669 if( p->rc==SQLITE_OK ){
3670 p->rc = prepareAndCollectError(p->dbMain, &pInsert, &p->zErrmsg,
3671 "INSERT INTO sqlite_schema VALUES(?,?,?,?,?)"
3672 );
3673 }
3674
3675 while( p->rc==SQLITE_OK && sqlite3_step(pSql)==SQLITE_ROW ){
3676 int i;
3677 for(i=0; i<5; i++){
3678 sqlite3_bind_value(pInsert, i+1, sqlite3_column_value(pSql, i));
3679 }
3680 sqlite3_step(pInsert);
3681 p->rc = sqlite3_reset(pInsert);
3682 }
3683 if( p->rc==SQLITE_OK ){
3684 p->rc = sqlite3_exec(p->dbMain, "PRAGMA writable_schema=0",0,0,&p->zErrmsg);
3685 }
3686
3687 rbuFinalize(p, pSql);
3688 rbuFinalize(p, pInsert);
3689 }
3690
3691 /*
3692 ** Step the RBU object.
3693 */
3694 int sqlite3rbu_step(sqlite3rbu *p){
3695 if( p ){
3696 switch( p->eStage ){
3697 case RBU_STAGE_OAL: {
3698 RbuObjIter *pIter = &p->objiter;
3699
3700 /* If this is an RBU vacuum operation and the state table was empty
3701 ** when this handle was opened, create the target database schema. */
3702 if( rbuIsVacuum(p) && p->nProgress==0 && p->rc==SQLITE_OK ){
3703 rbuCreateTargetSchema(p);
3704 rbuCopyPragma(p, "user_version");
3705 rbuCopyPragma(p, "application_id");
3706 }
3707
3708 while( p->rc==SQLITE_OK && pIter->zTbl ){
3709
3710 if( pIter->bCleanup ){
3711 /* Clean up the rbu_tmp_xxx table for the previous table. It
3712 ** cannot be dropped as there are currently active SQL statements.
3713 ** But the contents can be deleted. */
3714 if( rbuIsVacuum(p)==0 && pIter->abIndexed ){
3715 rbuMPrintfExec(p, p->dbRbu,
3716 "DELETE FROM %s.'rbu_tmp_%q'", p->zStateDb, pIter->zDataTbl
3717 );
3718 }
3719 }else{
3720 rbuObjIterPrepareAll(p, pIter, 0);
3721
3722 /* Advance to the next row to process. */
3723 if( p->rc==SQLITE_OK ){
3724 int rc = sqlite3_step(pIter->pSelect);
3725 if( rc==SQLITE_ROW ){
3726 p->nProgress++;
3727 p->nStep++;
3728 return rbuStep(p);
3729 }
3730 p->rc = sqlite3_reset(pIter->pSelect);
3731 p->nStep = 0;
3732 }
3733 }
3734
3735 rbuObjIterNext(p, pIter);
3736 }
3737
3738 if( p->rc==SQLITE_OK ){
3739 assert( pIter->zTbl==0 );
3740 rbuSaveState(p, RBU_STAGE_MOVE);
3741 rbuIncrSchemaCookie(p);
3742 if( p->rc==SQLITE_OK ){
3743 p->rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, &p->zErrmsg);
3744 }
3745 if( p->rc==SQLITE_OK ){
3746 p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, &p->zErrmsg);
3747 }
3748 p->eStage = RBU_STAGE_MOVE;
3749 }
3750 break;
3751 }
3752
3753 case RBU_STAGE_MOVE: {
3754 if( p->rc==SQLITE_OK ){
3755 rbuMoveOalFile(p);
3756 p->nProgress++;
3757 }
3758 break;
3759 }
3760
3761 case RBU_STAGE_CKPT: {
3762 if( p->rc==SQLITE_OK ){
3763 if( p->nStep>=p->nFrame ){
3764 sqlite3_file *pDb = p->pTargetFd->pReal;
3765
3766 /* Sync the db file */
3767 p->rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL);
3768
3769 /* Update nBackfill */
3770 if( p->rc==SQLITE_OK ){
3771 void volatile *ptr;
3772 p->rc = pDb->pMethods->xShmMap(pDb, 0, 32*1024, 0, &ptr);
3773 if( p->rc==SQLITE_OK ){
3774 ((u32 volatile*)ptr)[24] = p->iMaxFrame;
3775 }
3776 }
3777
3778 if( p->rc==SQLITE_OK ){
3779 p->eStage = RBU_STAGE_DONE;
3780 p->rc = SQLITE_DONE;
3781 }
3782 }else{
3783 /* At one point the following block copied a single frame from the
3784 ** wal file to the database file. So that one call to sqlite3rbu_step()
3785 ** checkpointed a single frame.
3786 **
3787 ** However, if the sector-size is larger than the page-size, and the
3788 ** application calls sqlite3rbu_savestate() or close() immediately
3789 ** after this step, then rbu_step() again, then a power failure occurs,
3790 ** then the database page written here may be damaged. Work around
3791 ** this by checkpointing frames until the next page in the aFrame[]
3792 ** lies on a different disk sector to the current one. */
3793 u32 iSector;
3794 do{
3795 RbuFrame *pFrame = &p->aFrame[p->nStep];
3796 iSector = (pFrame->iDbPage-1) / p->nPagePerSector;
3797 rbuCheckpointFrame(p, pFrame);
3798 p->nStep++;
3799 }while( p->nStep<p->nFrame
3800 && iSector==((p->aFrame[p->nStep].iDbPage-1) / p->nPagePerSector)
3801 && p->rc==SQLITE_OK
3802 );
3803 }
3804 p->nProgress++;
3805 }
3806 break;
3807 }
3808
3809 default:
3810 break;
3811 }
3812 return p->rc;
3813 }else{
3814 return SQLITE_NOMEM;
3815 }
3816 }
3817
3818 /*
3819 ** Compare strings z1 and z2, returning 0 if they are identical, or non-zero
3820 ** otherwise. Either or both argument may be NULL. Two NULL values are
3821 ** considered equal, and NULL is considered distinct from all other values.
3822 */
3823 static int rbuStrCompare(const char *z1, const char *z2){
3824 if( z1==0 && z2==0 ) return 0;
3825 if( z1==0 || z2==0 ) return 1;
3826 return (sqlite3_stricmp(z1, z2)!=0);
3827 }
3828
3829 /*
3830 ** This function is called as part of sqlite3rbu_open() when initializing
3831 ** an rbu handle in OAL stage. If the rbu update has not started (i.e.
3832 ** the rbu_state table was empty) it is a no-op. Otherwise, it arranges
3833 ** things so that the next call to sqlite3rbu_step() continues on from
3834 ** where the previous rbu handle left off.
3835 **
3836 ** If an error occurs, an error code and error message are left in the
3837 ** rbu handle passed as the first argument.
3838 */
3839 static void rbuSetupOal(sqlite3rbu *p, RbuState *pState){
3840 assert( p->rc==SQLITE_OK );
3841 if( pState->zTbl ){
3842 RbuObjIter *pIter = &p->objiter;
3843 int rc = SQLITE_OK;
3844
3845 while( rc==SQLITE_OK && pIter->zTbl && (pIter->bCleanup
3846 || rbuStrCompare(pIter->zIdx, pState->zIdx)
3847 || (pState->zDataTbl==0 && rbuStrCompare(pIter->zTbl, pState->zTbl))
3848 || (pState->zDataTbl && rbuStrCompare(pIter->zDataTbl, pState->zDataTbl))
3849 )){
3850 rc = rbuObjIterNext(p, pIter);
3851 }
3852
3853 if( rc==SQLITE_OK && !pIter->zTbl ){
3854 rc = SQLITE_ERROR;
3855 p->zErrmsg = sqlite3_mprintf("rbu_state mismatch error");
3856 }
3857
3858 if( rc==SQLITE_OK ){
3859 p->nStep = pState->nRow;
3860 rc = rbuObjIterPrepareAll(p, &p->objiter, p->nStep);
3861 }
3862
3863 p->rc = rc;
3864 }
3865 }
3866
3867 /*
3868 ** If there is a "*-oal" file in the file-system corresponding to the
3869 ** target database in the file-system, delete it. If an error occurs,
3870 ** leave an error code and error message in the rbu handle.
3871 */
3872 static void rbuDeleteOalFile(sqlite3rbu *p){
3873 char *zOal = rbuMPrintf(p, "%s-oal", p->zTarget);
3874 if( zOal ){
3875 sqlite3_vfs *pVfs = 0;
3876 sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_VFS_POINTER, &pVfs);
3877 assert( pVfs && p->rc==SQLITE_OK && p->zErrmsg==0 );
3878 pVfs->xDelete(pVfs, zOal, 0);
3879 sqlite3_free(zOal);
3880 }
3881 }
3882
3883 /*
3884 ** Allocate a private rbu VFS for the rbu handle passed as the only
3885 ** argument. This VFS will be used unless the call to sqlite3rbu_open()
3886 ** specified a URI with a vfs=? option in place of a target database
3887 ** file name.
3888 */
3889 static void rbuCreateVfs(sqlite3rbu *p){
3890 int rnd;
3891 char zRnd[64];
3892
3893 assert( p->rc==SQLITE_OK );
3894 sqlite3_randomness(sizeof(int), (void*)&rnd);
3895 sqlite3_snprintf(sizeof(zRnd), zRnd, "rbu_vfs_%d", rnd);
3896 p->rc = sqlite3rbu_create_vfs(zRnd, 0);
3897 if( p->rc==SQLITE_OK ){
3898 sqlite3_vfs *pVfs = sqlite3_vfs_find(zRnd);
3899 assert( pVfs );
3900 p->zVfsName = pVfs->zName;
3901 ((rbu_vfs*)pVfs)->pRbu = p;
3902 }
3903 }
3904
3905 /*
3906 ** Destroy the private VFS created for the rbu handle passed as the only
3907 ** argument by an earlier call to rbuCreateVfs().
3908 */
3909 static void rbuDeleteVfs(sqlite3rbu *p){
3910 if( p->zVfsName ){
3911 sqlite3rbu_destroy_vfs(p->zVfsName);
3912 p->zVfsName = 0;
3913 }
3914 }
3915
3916 /*
3917 ** This user-defined SQL function is invoked with a single argument - the
3918 ** name of a table expected to appear in the target database. It returns
3919 ** the number of auxilliary indexes on the table.
3920 */
3921 static void rbuIndexCntFunc(
3922 sqlite3_context *pCtx,
3923 int nVal,
3924 sqlite3_value **apVal
3925 ){
3926 sqlite3rbu *p = (sqlite3rbu*)sqlite3_user_data(pCtx);
3927 sqlite3_stmt *pStmt = 0;
3928 char *zErrmsg = 0;
3929 int rc;
3930 sqlite3 *db = (rbuIsVacuum(p) ? p->dbRbu : p->dbMain);
3931
3932 assert( nVal==1 );
3933 UNUSED_PARAMETER(nVal);
3934
3935 rc = prepareFreeAndCollectError(db, &pStmt, &zErrmsg,
3936 sqlite3_mprintf("SELECT count(*) FROM sqlite_schema "
3937 "WHERE type='index' AND tbl_name = %Q", sqlite3_value_text(apVal[0]))
3938 );
3939 if( rc!=SQLITE_OK ){
3940 sqlite3_result_error(pCtx, zErrmsg, -1);
3941 }else{
3942 int nIndex = 0;
3943 if( SQLITE_ROW==sqlite3_step(pStmt) ){
3944 nIndex = sqlite3_column_int(pStmt, 0);
3945 }
3946 rc = sqlite3_finalize(pStmt);
3947 if( rc==SQLITE_OK ){
3948 sqlite3_result_int(pCtx, nIndex);
3949 }else{
3950 sqlite3_result_error(pCtx, sqlite3_errmsg(db), -1);
3951 }
3952 }
3953
3954 sqlite3_free(zErrmsg);
3955 }
3956
3957 /*
3958 ** If the RBU database contains the rbu_count table, use it to initialize
3959 ** the sqlite3rbu.nPhaseOneStep variable. The schema of the rbu_count table
3960 ** is assumed to contain the same columns as:
3961 **
3962 ** CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID;
3963 **
3964 ** There should be one row in the table for each data_xxx table in the
3965 ** database. The 'tbl' column should contain the name of a data_xxx table,
3966 ** and the cnt column the number of rows it contains.
3967 **
3968 ** sqlite3rbu.nPhaseOneStep is initialized to the sum of (1 + nIndex) * cnt
3969 ** for all rows in the rbu_count table, where nIndex is the number of
3970 ** indexes on the corresponding target database table.
3971 */
3972 static void rbuInitPhaseOneSteps(sqlite3rbu *p){
3973 if( p->rc==SQLITE_OK ){
3974 sqlite3_stmt *pStmt = 0;
3975 int bExists = 0; /* True if rbu_count exists */
3976
3977 p->nPhaseOneStep = -1;
3978
3979 p->rc = sqlite3_create_function(p->dbRbu,
3980 "rbu_index_cnt", 1, SQLITE_UTF8, (void*)p, rbuIndexCntFunc, 0, 0
3981 );
3982
3983 /* Check for the rbu_count table. If it does not exist, or if an error
3984 ** occurs, nPhaseOneStep will be left set to -1. */
3985 if( p->rc==SQLITE_OK ){
3986 p->rc = prepareAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg,
3987 "SELECT 1 FROM sqlite_schema WHERE tbl_name = 'rbu_count'"
3988 );
3989 }
3990 if( p->rc==SQLITE_OK ){
3991 if( SQLITE_ROW==sqlite3_step(pStmt) ){
3992 bExists = 1;
3993 }
3994 p->rc = sqlite3_finalize(pStmt);
3995 }
3996
3997 if( p->rc==SQLITE_OK && bExists ){
3998 p->rc = prepareAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg,
3999 "SELECT sum(cnt * (1 + rbu_index_cnt(rbu_target_name(tbl))))"
4000 "FROM rbu_count"
4001 );
4002 if( p->rc==SQLITE_OK ){
4003 if( SQLITE_ROW==sqlite3_step(pStmt) ){
4004 p->nPhaseOneStep = sqlite3_column_int64(pStmt, 0);
4005 }
4006 p->rc = sqlite3_finalize(pStmt);
4007 }
4008 }
4009 }
4010 }
4011
4012
4013 static sqlite3rbu *openRbuHandle(
4014 const char *zTarget,
4015 const char *zRbu,
4016 const char *zState
4017 ){
4018 sqlite3rbu *p;
4019 size_t nTarget = zTarget ? strlen(zTarget) : 0;
4020 size_t nRbu = strlen(zRbu);
4021 size_t nByte = sizeof(sqlite3rbu) + nTarget+1 + nRbu+1;
4022
4023 p = (sqlite3rbu*)sqlite3_malloc64(nByte);
4024 if( p ){
4025 RbuState *pState = 0;
4026
4027 /* Create the custom VFS. */
4028 memset(p, 0, sizeof(sqlite3rbu));
4029 sqlite3rbu_rename_handler(p, 0, 0);
4030 rbuCreateVfs(p);
4031
4032 /* Open the target, RBU and state databases */
4033 if( p->rc==SQLITE_OK ){
4034 char *pCsr = (char*)&p[1];
4035 int bRetry = 0;
4036 if( zTarget ){
4037 p->zTarget = pCsr;
4038 memcpy(p->zTarget, zTarget, nTarget+1);
4039 pCsr += nTarget+1;
4040 }
4041 p->zRbu = pCsr;
4042 memcpy(p->zRbu, zRbu, nRbu+1);
4043 pCsr += nRbu+1;
4044 if( zState ){
4045 p->zState = rbuMPrintf(p, "%s", zState);
4046 }
4047
4048 /* If the first attempt to open the database file fails and the bRetry
4049 ** flag it set, this means that the db was not opened because it seemed
4050 ** to be a wal-mode db. But, this may have happened due to an earlier
4051 ** RBU vacuum operation leaving an old wal file in the directory.
4052 ** If this is the case, it will have been checkpointed and deleted
4053 ** when the handle was closed and a second attempt to open the
4054 ** database may succeed. */
4055 rbuOpenDatabase(p, 0, &bRetry);
4056 if( bRetry ){
4057 rbuOpenDatabase(p, 0, 0);
4058 }
4059 }
4060
4061 if( p->rc==SQLITE_OK ){
4062 pState = rbuLoadState(p);
4063 assert( pState || p->rc!=SQLITE_OK );
4064 if( p->rc==SQLITE_OK ){
4065
4066 if( pState->eStage==0 ){
4067 rbuDeleteOalFile(p);
4068 rbuInitPhaseOneSteps(p);
4069 p->eStage = RBU_STAGE_OAL;
4070 }else{
4071 p->eStage = pState->eStage;
4072 p->nPhaseOneStep = pState->nPhaseOneStep;
4073 }
4074 p->nProgress = pState->nProgress;
4075 p->iOalSz = pState->iOalSz;
4076 }
4077 }
4078 assert( p->rc!=SQLITE_OK || p->eStage!=0 );
4079
4080 if( p->rc==SQLITE_OK && p->pTargetFd->pWalFd ){
4081 if( p->eStage==RBU_STAGE_OAL ){
4082 p->rc = SQLITE_ERROR;
4083 p->zErrmsg = sqlite3_mprintf("cannot update wal mode database");
4084 }else if( p->eStage==RBU_STAGE_MOVE ){
4085 p->eStage = RBU_STAGE_CKPT;
4086 p->nStep = 0;
4087 }
4088 }
4089
4090 if( p->rc==SQLITE_OK
4091 && (p->eStage==RBU_STAGE_OAL || p->eStage==RBU_STAGE_MOVE)
4092 && pState->eStage!=0
4093 ){
4094 rbu_file *pFd = (rbuIsVacuum(p) ? p->pRbuFd : p->pTargetFd);
4095 if( pFd->iCookie!=pState->iCookie ){
4096 /* At this point (pTargetFd->iCookie) contains the value of the
4097 ** change-counter cookie (the thing that gets incremented when a
4098 ** transaction is committed in rollback mode) currently stored on
4099 ** page 1 of the database file. */
4100 p->rc = SQLITE_BUSY;
4101 p->zErrmsg = sqlite3_mprintf("database modified during rbu %s",
4102 (rbuIsVacuum(p) ? "vacuum" : "update")
4103 );
4104 }
4105 }
4106
4107 if( p->rc==SQLITE_OK ){
4108 if( p->eStage==RBU_STAGE_OAL ){
4109 sqlite3 *db = p->dbMain;
4110 p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, &p->zErrmsg);
4111
4112 /* Point the object iterator at the first object */
4113 if( p->rc==SQLITE_OK ){
4114 p->rc = rbuObjIterFirst(p, &p->objiter);
4115 }
4116
4117 /* If the RBU database contains no data_xxx tables, declare the RBU
4118 ** update finished. */
4119 if( p->rc==SQLITE_OK && p->objiter.zTbl==0 ){
4120 p->rc = SQLITE_DONE;
4121 p->eStage = RBU_STAGE_DONE;
4122 }else{
4123 if( p->rc==SQLITE_OK && pState->eStage==0 && rbuIsVacuum(p) ){
4124 rbuCopyPragma(p, "page_size");
4125 rbuCopyPragma(p, "auto_vacuum");
4126 }
4127
4128 /* Open transactions both databases. The *-oal file is opened or
4129 ** created at this point. */
4130 if( p->rc==SQLITE_OK ){
4131 p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg);
4132 }
4133
4134 /* Check if the main database is a zipvfs db. If it is, set the upper
4135 ** level pager to use "journal_mode=off". This prevents it from
4136 ** generating a large journal using a temp file. */
4137 if( p->rc==SQLITE_OK ){
4138 int frc = sqlite3_file_control(db, "main", SQLITE_FCNTL_ZIPVFS, 0);
4139 if( frc==SQLITE_OK ){
4140 p->rc = sqlite3_exec(
4141 db, "PRAGMA journal_mode=off",0,0,&p->zErrmsg);
4142 }
4143 }
4144
4145 if( p->rc==SQLITE_OK ){
4146 rbuSetupOal(p, pState);
4147 }
4148 }
4149 }else if( p->eStage==RBU_STAGE_MOVE ){
4150 /* no-op */
4151 }else if( p->eStage==RBU_STAGE_CKPT ){
4152 if( !rbuIsVacuum(p) && rbuExclusiveCheckpoint(p->dbMain) ){
4153 /* If the rbu_exclusive_checkpoint=1 URI parameter was specified
4154 ** and an incremental checkpoint is being resumed, attempt an
4155 ** exclusive lock on the db file. If this fails, so be it. */
4156 p->eStage = RBU_STAGE_DONE;
4157 rbuLockDatabase(p->dbMain);
4158 p->eStage = RBU_STAGE_CKPT;
4159 }
4160 rbuSetupCheckpoint(p, pState);
4161 }else if( p->eStage==RBU_STAGE_DONE ){
4162 p->rc = SQLITE_DONE;
4163 }else{
4164 p->rc = SQLITE_CORRUPT;
4165 }
4166 }
4167
4168 rbuFreeState(pState);
4169 }
4170
4171 return p;
4172 }
4173
4174 /*
4175 ** Allocate and return an RBU handle with all fields zeroed except for the
4176 ** error code, which is set to SQLITE_MISUSE.
4177 */
4178 static sqlite3rbu *rbuMisuseError(void){
4179 sqlite3rbu *pRet;
4180 pRet = sqlite3_malloc64(sizeof(sqlite3rbu));
4181 if( pRet ){
4182 memset(pRet, 0, sizeof(sqlite3rbu));
4183 pRet->rc = SQLITE_MISUSE;
4184 }
4185 return pRet;
4186 }
4187
4188 /*
4189 ** Open and return a new RBU handle.
4190 */
4191 sqlite3rbu *sqlite3rbu_open(
4192 const char *zTarget,
4193 const char *zRbu,
4194 const char *zState
4195 ){
4196 if( zTarget==0 || zRbu==0 ){ return rbuMisuseError(); }
4197 return openRbuHandle(zTarget, zRbu, zState);
4198 }
4199
4200 /*
4201 ** Open a handle to begin or resume an RBU VACUUM operation.
4202 */
4203 sqlite3rbu *sqlite3rbu_vacuum(
4204 const char *zTarget,
4205 const char *zState
4206 ){
4207 if( zTarget==0 ){ return rbuMisuseError(); }
4208 if( zState ){
4209 size_t n = strlen(zState);
4210 if( n>=7 && 0==memcmp("-vactmp", &zState[n-7], 7) ){
4211 return rbuMisuseError();
4212 }
4213 }
4214 /* TODO: Check that both arguments are non-NULL */
4215 return openRbuHandle(0, zTarget, zState);
4216 }
4217
4218 /*
4219 ** Return the database handle used by pRbu.
4220 */
4221 sqlite3 *sqlite3rbu_db(sqlite3rbu *pRbu, int bRbu){
4222 sqlite3 *db = 0;
4223 if( pRbu ){
4224 db = (bRbu ? pRbu->dbRbu : pRbu->dbMain);
4225 }
4226 return db;
4227 }
4228
4229
4230 /*
4231 ** If the error code currently stored in the RBU handle is SQLITE_CONSTRAINT,
4232 ** then edit any error message string so as to remove all occurrences of
4233 ** the pattern "rbu_imp_[0-9]*".
4234 */
4235 static void rbuEditErrmsg(sqlite3rbu *p){
4236 if( p->rc==SQLITE_CONSTRAINT && p->zErrmsg ){
4237 unsigned int i;
4238 size_t nErrmsg = strlen(p->zErrmsg);
4239 for(i=0; i<(nErrmsg-8); i++){
4240 if( memcmp(&p->zErrmsg[i], "rbu_imp_", 8)==0 ){
4241 int nDel = 8;
4242 while( p->zErrmsg[i+nDel]>='0' && p->zErrmsg[i+nDel]<='9' ) nDel++;
4243 memmove(&p->zErrmsg[i], &p->zErrmsg[i+nDel], nErrmsg + 1 - i - nDel);
4244 nErrmsg -= nDel;
4245 }
4246 }
4247 }
4248 }
4249
4250 /*
4251 ** Close the RBU handle.
4252 */
4253 int sqlite3rbu_close(sqlite3rbu *p, char **pzErrmsg){
4254 int rc;
4255 if( p ){
4256
4257 /* Commit the transaction to the *-oal file. */
4258 if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_OAL ){
4259 p->rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, &p->zErrmsg);
4260 }
4261
4262 /* Sync the db file if currently doing an incremental checkpoint */
4263 if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_CKPT ){
4264 sqlite3_file *pDb = p->pTargetFd->pReal;
4265 p->rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL);
4266 }
4267
4268 rbuSaveState(p, p->eStage);
4269
4270 if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_OAL ){
4271 p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, &p->zErrmsg);
4272 }
4273
4274 /* Close any open statement handles. */
4275 rbuObjIterFinalize(&p->objiter);
4276
4277 /* If this is an RBU vacuum handle and the vacuum has either finished
4278 ** successfully or encountered an error, delete the contents of the
4279 ** state table. This causes the next call to sqlite3rbu_vacuum()
4280 ** specifying the current target and state databases to start a new
4281 ** vacuum from scratch. */
4282 if( rbuIsVacuum(p) && p->rc!=SQLITE_OK && p->dbRbu ){
4283 int rc2 = sqlite3_exec(p->dbRbu, "DELETE FROM stat.rbu_state", 0, 0, 0);
4284 if( p->rc==SQLITE_DONE && rc2!=SQLITE_OK ) p->rc = rc2;
4285 }
4286
4287 /* Close the open database handle and VFS object. */
4288 sqlite3_close(p->dbRbu);
4289 sqlite3_close(p->dbMain);
4290 assert( p->szTemp==0 );
4291 rbuDeleteVfs(p);
4292 sqlite3_free(p->aBuf);
4293 sqlite3_free(p->aFrame);
4294
4295 rbuEditErrmsg(p);
4296 rc = p->rc;
4297 if( pzErrmsg ){
4298 *pzErrmsg = p->zErrmsg;
4299 }else{
4300 sqlite3_free(p->zErrmsg);
4301 }
4302 sqlite3_free(p->zState);
4303 sqlite3_free(p);
4304 }else{
4305 rc = SQLITE_NOMEM;
4306 *pzErrmsg = 0;
4307 }
4308 return rc;
4309 }
4310
4311 /*
4312 ** Return the total number of key-value operations (inserts, deletes or
4313 ** updates) that have been performed on the target database since the
4314 ** current RBU update was started.
4315 */
4316 sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu){
4317 return pRbu->nProgress;
4318 }
4319
4320 /*
4321 ** Return permyriadage progress indications for the two main stages of
4322 ** an RBU update.
4323 */
4324 void sqlite3rbu_bp_progress(sqlite3rbu *p, int *pnOne, int *pnTwo){
4325 const int MAX_PROGRESS = 10000;
4326 switch( p->eStage ){
4327 case RBU_STAGE_OAL:
4328 if( p->nPhaseOneStep>0 ){
4329 *pnOne = (int)(MAX_PROGRESS * (i64)p->nProgress/(i64)p->nPhaseOneStep);
4330 }else{
4331 *pnOne = -1;
4332 }
4333 *pnTwo = 0;
4334 break;
4335
4336 case RBU_STAGE_MOVE:
4337 *pnOne = MAX_PROGRESS;
4338 *pnTwo = 0;
4339 break;
4340
4341 case RBU_STAGE_CKPT:
4342 *pnOne = MAX_PROGRESS;
4343 *pnTwo = (int)(MAX_PROGRESS * (i64)p->nStep / (i64)p->nFrame);
4344 break;
4345
4346 case RBU_STAGE_DONE:
4347 *pnOne = MAX_PROGRESS;
4348 *pnTwo = MAX_PROGRESS;
4349 break;
4350
4351 default:
4352 assert( 0 );
4353 }
4354 }
4355
4356 /*
4357 ** Return the current state of the RBU vacuum or update operation.
4358 */
4359 int sqlite3rbu_state(sqlite3rbu *p){
4360 int aRes[] = {
4361 0, SQLITE_RBU_STATE_OAL, SQLITE_RBU_STATE_MOVE,
4362 0, SQLITE_RBU_STATE_CHECKPOINT, SQLITE_RBU_STATE_DONE
4363 };
4364
4365 assert( RBU_STAGE_OAL==1 );
4366 assert( RBU_STAGE_MOVE==2 );
4367 assert( RBU_STAGE_CKPT==4 );
4368 assert( RBU_STAGE_DONE==5 );
4369 assert( aRes[RBU_STAGE_OAL]==SQLITE_RBU_STATE_OAL );
4370 assert( aRes[RBU_STAGE_MOVE]==SQLITE_RBU_STATE_MOVE );
4371 assert( aRes[RBU_STAGE_CKPT]==SQLITE_RBU_STATE_CHECKPOINT );
4372 assert( aRes[RBU_STAGE_DONE]==SQLITE_RBU_STATE_DONE );
4373
4374 if( p->rc!=SQLITE_OK && p->rc!=SQLITE_DONE ){
4375 return SQLITE_RBU_STATE_ERROR;
4376 }else{
4377 assert( p->rc!=SQLITE_DONE || p->eStage==RBU_STAGE_DONE );
4378 assert( p->eStage==RBU_STAGE_OAL
4379 || p->eStage==RBU_STAGE_MOVE
4380 || p->eStage==RBU_STAGE_CKPT
4381 || p->eStage==RBU_STAGE_DONE
4382 );
4383 return aRes[p->eStage];
4384 }
4385 }
4386
4387 int sqlite3rbu_savestate(sqlite3rbu *p){
4388 int rc = p->rc;
4389 if( rc==SQLITE_DONE ) return SQLITE_OK;
4390
4391 assert( p->eStage>=RBU_STAGE_OAL && p->eStage<=RBU_STAGE_DONE );
4392 if( p->eStage==RBU_STAGE_OAL ){
4393 assert( rc!=SQLITE_DONE );
4394 if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, 0);
4395 }
4396
4397 /* Sync the db file */
4398 if( rc==SQLITE_OK && p->eStage==RBU_STAGE_CKPT ){
4399 sqlite3_file *pDb = p->pTargetFd->pReal;
4400 rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL);
4401 }
4402
4403 p->rc = rc;
4404 rbuSaveState(p, p->eStage);
4405 rc = p->rc;
4406
4407 if( p->eStage==RBU_STAGE_OAL ){
4408 assert( rc!=SQLITE_DONE );
4409 if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, 0);
4410 if( rc==SQLITE_OK ){
4411 const char *zBegin = rbuIsVacuum(p) ? "BEGIN" : "BEGIN IMMEDIATE";
4412 rc = sqlite3_exec(p->dbRbu, zBegin, 0, 0, 0);
4413 }
4414 if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbMain, "BEGIN IMMEDIATE", 0, 0,0);
4415 }
4416
4417 p->rc = rc;
4418 return rc;
4419 }
4420
4421 /*
4422 ** Default xRename callback for RBU.
4423 */
4424 static int xDefaultRename(void *pArg, const char *zOld, const char *zNew){
4425 int rc = SQLITE_OK;
4426 UNUSED_PARAMETER(pArg);
4427 #if defined(_WIN32_WCE)
4428 {
4429 LPWSTR zWideOld;
4430 LPWSTR zWideNew;
4431
4432 zWideOld = rbuWinUtf8ToUnicode(zOld);
4433 if( zWideOld ){
4434 zWideNew = rbuWinUtf8ToUnicode(zNew);
4435 if( zWideNew ){
4436 if( MoveFileW(zWideOld, zWideNew) ){
4437 rc = SQLITE_OK;
4438 }else{
4439 rc = SQLITE_IOERR;
4440 }
4441 sqlite3_free(zWideNew);
4442 }else{
4443 rc = SQLITE_IOERR_NOMEM;
4444 }
4445 sqlite3_free(zWideOld);
4446 }else{
4447 rc = SQLITE_IOERR_NOMEM;
4448 }
4449 }
4450 #else
4451 rc = rename(zOld, zNew) ? SQLITE_IOERR : SQLITE_OK;
4452 #endif
4453 return rc;
4454 }
4455
4456 void sqlite3rbu_rename_handler(
4457 sqlite3rbu *pRbu,
4458 void *pArg,
4459 int (*xRename)(void *pArg, const char *zOld, const char *zNew)
4460 ){
4461 if( xRename ){
4462 pRbu->xRename = xRename;
4463 pRbu->pRenameArg = pArg;
4464 }else{
4465 pRbu->xRename = xDefaultRename;
4466 pRbu->pRenameArg = 0;
4467 }
4468 }
4469
4470 /**************************************************************************
4471 ** Beginning of RBU VFS shim methods. The VFS shim modifies the behaviour
4472 ** of a standard VFS in the following ways:
4473 **
4474 ** 1. Whenever the first page of a main database file is read or
4475 ** written, the value of the change-counter cookie is stored in
4476 ** rbu_file.iCookie. Similarly, the value of the "write-version"
4477 ** database header field is stored in rbu_file.iWriteVer. This ensures
4478 ** that the values are always trustworthy within an open transaction.
4479 **
4480 ** 2. Whenever an SQLITE_OPEN_WAL file is opened, the (rbu_file.pWalFd)
4481 ** member variable of the associated database file descriptor is set
4482 ** to point to the new file. A mutex protected linked list of all main
4483 ** db fds opened using a particular RBU VFS is maintained at
4484 ** rbu_vfs.pMain to facilitate this.
4485 **
4486 ** 3. Using a new file-control "SQLITE_FCNTL_RBU", a main db rbu_file
4487 ** object can be marked as the target database of an RBU update. This
4488 ** turns on the following extra special behaviour:
4489 **
4490 ** 3a. If xAccess() is called to check if there exists a *-wal file
4491 ** associated with an RBU target database currently in RBU_STAGE_OAL
4492 ** stage (preparing the *-oal file), the following special handling
4493 ** applies:
4494 **
4495 ** * if the *-wal file does exist, return SQLITE_CANTOPEN. An RBU
4496 ** target database may not be in wal mode already.
4497 **
4498 ** * if the *-wal file does not exist, set the output parameter to
4499 ** non-zero (to tell SQLite that it does exist) anyway.
4500 **
4501 ** Then, when xOpen() is called to open the *-wal file associated with
4502 ** the RBU target in RBU_STAGE_OAL stage, instead of opening the *-wal
4503 ** file, the rbu vfs opens the corresponding *-oal file instead.
4504 **
4505 ** 3b. The *-shm pages returned by xShmMap() for a target db file in
4506 ** RBU_STAGE_OAL mode are actually stored in heap memory. This is to
4507 ** avoid creating a *-shm file on disk. Additionally, xShmLock() calls
4508 ** are no-ops on target database files in RBU_STAGE_OAL mode. This is
4509 ** because assert() statements in some VFS implementations fail if
4510 ** xShmLock() is called before xShmMap().
4511 **
4512 ** 3c. If an EXCLUSIVE lock is attempted on a target database file in any
4513 ** mode except RBU_STAGE_DONE (all work completed and checkpointed), it
4514 ** fails with an SQLITE_BUSY error. This is to stop RBU connections
4515 ** from automatically checkpointing a *-wal (or *-oal) file from within
4516 ** sqlite3_close().
4517 **
4518 ** 3d. In RBU_STAGE_CAPTURE mode, all xRead() calls on the wal file, and
4519 ** all xWrite() calls on the target database file perform no IO.
4520 ** Instead the frame and page numbers that would be read and written
4521 ** are recorded. Additionally, successful attempts to obtain exclusive
4522 ** xShmLock() WRITER, CHECKPOINTER and READ0 locks on the target
4523 ** database file are recorded. xShmLock() calls to unlock the same
4524 ** locks are no-ops (so that once obtained, these locks are never
4525 ** relinquished). Finally, calls to xSync() on the target database
4526 ** file fail with SQLITE_NOTICE errors.
4527 */
4528
4529 static void rbuUnlockShm(rbu_file *p){
4530 assert( p->openFlags & SQLITE_OPEN_MAIN_DB );
4531 if( p->pRbu ){
4532 int (*xShmLock)(sqlite3_file*,int,int,int) = p->pReal->pMethods->xShmLock;
4533 int i;
4534 for(i=0; i<SQLITE_SHM_NLOCK;i++){
4535 if( (1<<i) & p->pRbu->mLock ){
4536 xShmLock(p->pReal, i, 1, SQLITE_SHM_UNLOCK|SQLITE_SHM_EXCLUSIVE);
4537 }
4538 }
4539 p->pRbu->mLock = 0;
4540 }
4541 }
4542
4543 /*
4544 */
4545 static int rbuUpdateTempSize(rbu_file *pFd, sqlite3_int64 nNew){
4546 sqlite3rbu *pRbu = pFd->pRbu;
4547 i64 nDiff = nNew - pFd->sz;
4548 pRbu->szTemp += nDiff;
4549 pFd->sz = nNew;
4550 assert( pRbu->szTemp>=0 );
4551 if( pRbu->szTempLimit && pRbu->szTemp>pRbu->szTempLimit ) return SQLITE_FULL;
4552 return SQLITE_OK;
4553 }
4554
4555 /*
4556 ** Add an item to the main-db lists, if it is not already present.
4557 **
4558 ** There are two main-db lists. One for all file descriptors, and one
4559 ** for all file descriptors with rbu_file.pDb!=0. If the argument has
4560 ** rbu_file.pDb!=0, then it is assumed to already be present on the
4561 ** main list and is only added to the pDb!=0 list.
4562 */
4563 static void rbuMainlistAdd(rbu_file *p){
4564 rbu_vfs *pRbuVfs = p->pRbuVfs;
4565 rbu_file *pIter;
4566 assert( (p->openFlags & SQLITE_OPEN_MAIN_DB) );
4567 sqlite3_mutex_enter(pRbuVfs->mutex);
4568 if( p->pRbu==0 ){
4569 for(pIter=pRbuVfs->pMain; pIter; pIter=pIter->pMainNext);
4570 p->pMainNext = pRbuVfs->pMain;
4571 pRbuVfs->pMain = p;
4572 }else{
4573 for(pIter=pRbuVfs->pMainRbu; pIter && pIter!=p; pIter=pIter->pMainRbuNext){}
4574 if( pIter==0 ){
4575 p->pMainRbuNext = pRbuVfs->pMainRbu;
4576 pRbuVfs->pMainRbu = p;
4577 }
4578 }
4579 sqlite3_mutex_leave(pRbuVfs->mutex);
4580 }
4581
4582 /*
4583 ** Remove an item from the main-db lists.
4584 */
4585 static void rbuMainlistRemove(rbu_file *p){
4586 rbu_file **pp;
4587 sqlite3_mutex_enter(p->pRbuVfs->mutex);
4588 for(pp=&p->pRbuVfs->pMain; *pp && *pp!=p; pp=&((*pp)->pMainNext)){}
4589 if( *pp ) *pp = p->pMainNext;
4590 p->pMainNext = 0;
4591 for(pp=&p->pRbuVfs->pMainRbu; *pp && *pp!=p; pp=&((*pp)->pMainRbuNext)){}
4592 if( *pp ) *pp = p->pMainRbuNext;
4593 p->pMainRbuNext = 0;
4594 sqlite3_mutex_leave(p->pRbuVfs->mutex);
4595 }
4596
4597 /*
4598 ** Given that zWal points to a buffer containing a wal file name passed to
4599 ** either the xOpen() or xAccess() VFS method, search the main-db list for
4600 ** a file-handle opened by the same database connection on the corresponding
4601 ** database file.
4602 **
4603 ** If parameter bRbu is true, only search for file-descriptors with
4604 ** rbu_file.pDb!=0.
4605 */
4606 static rbu_file *rbuFindMaindb(rbu_vfs *pRbuVfs, const char *zWal, int bRbu){
4607 rbu_file *pDb;
4608 sqlite3_mutex_enter(pRbuVfs->mutex);
4609 if( bRbu ){
4610 for(pDb=pRbuVfs->pMainRbu; pDb && pDb->zWal!=zWal; pDb=pDb->pMainRbuNext){}
4611 }else{
4612 for(pDb=pRbuVfs->pMain; pDb && pDb->zWal!=zWal; pDb=pDb->pMainNext){}
4613 }
4614 sqlite3_mutex_leave(pRbuVfs->mutex);
4615 return pDb;
4616 }
4617
4618 /*
4619 ** Close an rbu file.
4620 */
4621 static int rbuVfsClose(sqlite3_file *pFile){
4622 rbu_file *p = (rbu_file*)pFile;
4623 int rc;
4624 int i;
4625
4626 /* Free the contents of the apShm[] array. And the array itself. */
4627 for(i=0; i<p->nShm; i++){
4628 sqlite3_free(p->apShm[i]);
4629 }
4630 sqlite3_free(p->apShm);
4631 p->apShm = 0;
4632 sqlite3_free(p->zDel);
4633
4634 if( p->openFlags & SQLITE_OPEN_MAIN_DB ){
4635 const sqlite3_io_methods *pMeth = p->pReal->pMethods;
4636 rbuMainlistRemove(p);
4637 rbuUnlockShm(p);
4638 if( pMeth->iVersion>1 && pMeth->xShmUnmap ){
4639 pMeth->xShmUnmap(p->pReal, 0);
4640 }
4641 }
4642 else if( (p->openFlags & SQLITE_OPEN_DELETEONCLOSE) && p->pRbu ){
4643 rbuUpdateTempSize(p, 0);
4644 }
4645 assert( p->pMainNext==0 && p->pRbuVfs->pMain!=p );
4646
4647 /* Close the underlying file handle */
4648 rc = p->pReal->pMethods->xClose(p->pReal);
4649 return rc;
4650 }
4651
4652
4653 /*
4654 ** Read and return an unsigned 32-bit big-endian integer from the buffer
4655 ** passed as the only argument.
4656 */
4657 static u32 rbuGetU32(u8 *aBuf){
4658 return ((u32)aBuf[0] << 24)
4659 + ((u32)aBuf[1] << 16)
4660 + ((u32)aBuf[2] << 8)
4661 + ((u32)aBuf[3]);
4662 }
4663
4664 /*
4665 ** Write an unsigned 32-bit value in big-endian format to the supplied
4666 ** buffer.
4667 */
4668 static void rbuPutU32(u8 *aBuf, u32 iVal){
4669 aBuf[0] = (iVal >> 24) & 0xFF;
4670 aBuf[1] = (iVal >> 16) & 0xFF;
4671 aBuf[2] = (iVal >> 8) & 0xFF;
4672 aBuf[3] = (iVal >> 0) & 0xFF;
4673 }
4674
4675 static void rbuPutU16(u8 *aBuf, u16 iVal){
4676 aBuf[0] = (iVal >> 8) & 0xFF;
4677 aBuf[1] = (iVal >> 0) & 0xFF;
4678 }
4679
4680 /*
4681 ** Read data from an rbuVfs-file.
4682 */
4683 static int rbuVfsRead(
4684 sqlite3_file *pFile,
4685 void *zBuf,
4686 int iAmt,
4687 sqlite_int64 iOfst
4688 ){
4689 rbu_file *p = (rbu_file*)pFile;
4690 sqlite3rbu *pRbu = p->pRbu;
4691 int rc;
4692
4693 if( pRbu && pRbu->eStage==RBU_STAGE_CAPTURE ){
4694 assert( p->openFlags & SQLITE_OPEN_WAL );
4695 rc = rbuCaptureWalRead(p->pRbu, iOfst, iAmt);
4696 }else{
4697 if( pRbu && pRbu->eStage==RBU_STAGE_OAL
4698 && (p->openFlags & SQLITE_OPEN_WAL)
4699 && iOfst>=pRbu->iOalSz
4700 ){
4701 rc = SQLITE_OK;
4702 memset(zBuf, 0, iAmt);
4703 }else{
4704 rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst);
4705 #if 1
4706 /* If this is being called to read the first page of the target
4707 ** database as part of an rbu vacuum operation, synthesize the
4708 ** contents of the first page if it does not yet exist. Otherwise,
4709 ** SQLite will not check for a *-wal file. */
4710 if( pRbu && rbuIsVacuum(pRbu)
4711 && rc==SQLITE_IOERR_SHORT_READ && iOfst==0
4712 && (p->openFlags & SQLITE_OPEN_MAIN_DB)
4713 && pRbu->rc==SQLITE_OK
4714 ){
4715 sqlite3_file *pFd = (sqlite3_file*)pRbu->pRbuFd;
4716 rc = pFd->pMethods->xRead(pFd, zBuf, iAmt, iOfst);
4717 if( rc==SQLITE_OK ){
4718 u8 *aBuf = (u8*)zBuf;
4719 u32 iRoot = rbuGetU32(&aBuf[52]) ? 1 : 0;
4720 rbuPutU32(&aBuf[52], iRoot); /* largest root page number */
4721 rbuPutU32(&aBuf[36], 0); /* number of free pages */
4722 rbuPutU32(&aBuf[32], 0); /* first page on free list trunk */
4723 rbuPutU32(&aBuf[28], 1); /* size of db file in pages */
4724 rbuPutU32(&aBuf[24], pRbu->pRbuFd->iCookie+1); /* Change counter */
4725
4726 if( iAmt>100 ){
4727 memset(&aBuf[100], 0, iAmt-100);
4728 rbuPutU16(&aBuf[105], iAmt & 0xFFFF);
4729 aBuf[100] = 0x0D;
4730 }
4731 }
4732 }
4733 #endif
4734 }
4735 if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){
4736 /* These look like magic numbers. But they are stable, as they are part
4737 ** of the definition of the SQLite file format, which may not change. */
4738 u8 *pBuf = (u8*)zBuf;
4739 p->iCookie = rbuGetU32(&pBuf[24]);
4740 p->iWriteVer = pBuf[19];
4741 }
4742 }
4743 return rc;
4744 }
4745
4746 /*
4747 ** Write data to an rbuVfs-file.
4748 */
4749 static int rbuVfsWrite(
4750 sqlite3_file *pFile,
4751 const void *zBuf,
4752 int iAmt,
4753 sqlite_int64 iOfst
4754 ){
4755 rbu_file *p = (rbu_file*)pFile;
4756 sqlite3rbu *pRbu = p->pRbu;
4757 int rc;
4758
4759 if( pRbu && pRbu->eStage==RBU_STAGE_CAPTURE ){
4760 assert( p->openFlags & SQLITE_OPEN_MAIN_DB );
4761 rc = rbuCaptureDbWrite(p->pRbu, iOfst);
4762 }else{
4763 if( pRbu ){
4764 if( pRbu->eStage==RBU_STAGE_OAL
4765 && (p->openFlags & SQLITE_OPEN_WAL)
4766 && iOfst>=pRbu->iOalSz
4767 ){
4768 pRbu->iOalSz = iAmt + iOfst;
4769 }else if( p->openFlags & SQLITE_OPEN_DELETEONCLOSE ){
4770 i64 szNew = iAmt+iOfst;
4771 if( szNew>p->sz ){
4772 rc = rbuUpdateTempSize(p, szNew);
4773 if( rc!=SQLITE_OK ) return rc;
4774 }
4775 }
4776 }
4777 rc = p->pReal->pMethods->xWrite(p->pReal, zBuf, iAmt, iOfst);
4778 if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){
4779 /* These look like magic numbers. But they are stable, as they are part
4780 ** of the definition of the SQLite file format, which may not change. */
4781 u8 *pBuf = (u8*)zBuf;
4782 p->iCookie = rbuGetU32(&pBuf[24]);
4783 p->iWriteVer = pBuf[19];
4784 }
4785 }
4786 return rc;
4787 }
4788
4789 /*
4790 ** Truncate an rbuVfs-file.
4791 */
4792 static int rbuVfsTruncate(sqlite3_file *pFile, sqlite_int64 size){
4793 rbu_file *p = (rbu_file*)pFile;
4794 if( (p->openFlags & SQLITE_OPEN_DELETEONCLOSE) && p->pRbu ){
4795 int rc = rbuUpdateTempSize(p, size);
4796 if( rc!=SQLITE_OK ) return rc;
4797 }
4798 return p->pReal->pMethods->xTruncate(p->pReal, size);
4799 }
4800
4801 /*
4802 ** Sync an rbuVfs-file.
4803 */
4804 static int rbuVfsSync(sqlite3_file *pFile, int flags){
4805 rbu_file *p = (rbu_file *)pFile;
4806 if( p->pRbu && p->pRbu->eStage==RBU_STAGE_CAPTURE ){
4807 if( p->openFlags & SQLITE_OPEN_MAIN_DB ){
4808 return SQLITE_NOTICE_RBU;
4809 }
4810 return SQLITE_OK;
4811 }
4812 return p->pReal->pMethods->xSync(p->pReal, flags);
4813 }
4814
4815 /*
4816 ** Return the current file-size of an rbuVfs-file.
4817 */
4818 static int rbuVfsFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
4819 rbu_file *p = (rbu_file *)pFile;
4820 int rc;
4821 rc = p->pReal->pMethods->xFileSize(p->pReal, pSize);
4822
4823 /* If this is an RBU vacuum operation and this is the target database,
4824 ** pretend that it has at least one page. Otherwise, SQLite will not
4825 ** check for the existence of a *-wal file. rbuVfsRead() contains
4826 ** similar logic. */
4827 if( rc==SQLITE_OK && *pSize==0
4828 && p->pRbu && rbuIsVacuum(p->pRbu)
4829 && (p->openFlags & SQLITE_OPEN_MAIN_DB)
4830 ){
4831 *pSize = 1024;
4832 }
4833 return rc;
4834 }
4835
4836 /*
4837 ** Lock an rbuVfs-file.
4838 */
4839 static int rbuVfsLock(sqlite3_file *pFile, int eLock){
4840 rbu_file *p = (rbu_file*)pFile;
4841 sqlite3rbu *pRbu = p->pRbu;
4842 int rc = SQLITE_OK;
4843
4844 assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
4845 if( eLock==SQLITE_LOCK_EXCLUSIVE
4846 && (p->bNolock || (pRbu && pRbu->eStage!=RBU_STAGE_DONE))
4847 ){
4848 /* Do not allow EXCLUSIVE locks. Preventing SQLite from taking this
4849 ** prevents it from checkpointing the database from sqlite3_close(). */
4850 rc = SQLITE_BUSY;
4851 }else{
4852 rc = p->pReal->pMethods->xLock(p->pReal, eLock);
4853 }
4854
4855 return rc;
4856 }
4857
4858 /*
4859 ** Unlock an rbuVfs-file.
4860 */
4861 static int rbuVfsUnlock(sqlite3_file *pFile, int eLock){
4862 rbu_file *p = (rbu_file *)pFile;
4863 return p->pReal->pMethods->xUnlock(p->pReal, eLock);
4864 }
4865
4866 /*
4867 ** Check if another file-handle holds a RESERVED lock on an rbuVfs-file.
4868 */
4869 static int rbuVfsCheckReservedLock(sqlite3_file *pFile, int *pResOut){
4870 rbu_file *p = (rbu_file *)pFile;
4871 return p->pReal->pMethods->xCheckReservedLock(p->pReal, pResOut);
4872 }
4873
4874 /*
4875 ** File control method. For custom operations on an rbuVfs-file.
4876 */
4877 static int rbuVfsFileControl(sqlite3_file *pFile, int op, void *pArg){
4878 rbu_file *p = (rbu_file *)pFile;
4879 int (*xControl)(sqlite3_file*,int,void*) = p->pReal->pMethods->xFileControl;
4880 int rc;
4881
4882 assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB)
4883 || p->openFlags & (SQLITE_OPEN_TRANSIENT_DB|SQLITE_OPEN_TEMP_JOURNAL)
4884 );
4885 if( op==SQLITE_FCNTL_RBU ){
4886 sqlite3rbu *pRbu = (sqlite3rbu*)pArg;
4887
4888 /* First try to find another RBU vfs lower down in the vfs stack. If
4889 ** one is found, this vfs will operate in pass-through mode. The lower
4890 ** level vfs will do the special RBU handling. */
4891 rc = xControl(p->pReal, op, pArg);
4892
4893 if( rc==SQLITE_NOTFOUND ){
4894 /* Now search for a zipvfs instance lower down in the VFS stack. If
4895 ** one is found, this is an error. */
4896 void *dummy = 0;
4897 rc = xControl(p->pReal, SQLITE_FCNTL_ZIPVFS, &dummy);
4898 if( rc==SQLITE_OK ){
4899 rc = SQLITE_ERROR;
4900 pRbu->zErrmsg = sqlite3_mprintf("rbu/zipvfs setup error");
4901 }else if( rc==SQLITE_NOTFOUND ){
4902 pRbu->pTargetFd = p;
4903 p->pRbu = pRbu;
4904 rbuMainlistAdd(p);
4905 if( p->pWalFd ) p->pWalFd->pRbu = pRbu;
4906 rc = SQLITE_OK;
4907 }
4908 }
4909 return rc;
4910 }
4911 else if( op==SQLITE_FCNTL_RBUCNT ){
4912 sqlite3rbu *pRbu = (sqlite3rbu*)pArg;
4913 pRbu->nRbu++;
4914 pRbu->pRbuFd = p;
4915 p->bNolock = 1;
4916 }
4917
4918 rc = xControl(p->pReal, op, pArg);
4919 if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){
4920 rbu_vfs *pRbuVfs = p->pRbuVfs;
4921 char *zIn = *(char**)pArg;
4922 char *zOut = sqlite3_mprintf("rbu(%s)/%z", pRbuVfs->base.zName, zIn);
4923 *(char**)pArg = zOut;
4924 if( zOut==0 ) rc = SQLITE_NOMEM;
4925 }
4926
4927 return rc;
4928 }
4929
4930 /*
4931 ** Return the sector-size in bytes for an rbuVfs-file.
4932 */
4933 static int rbuVfsSectorSize(sqlite3_file *pFile){
4934 rbu_file *p = (rbu_file *)pFile;
4935 return p->pReal->pMethods->xSectorSize(p->pReal);
4936 }
4937
4938 /*
4939 ** Return the device characteristic flags supported by an rbuVfs-file.
4940 */
4941 static int rbuVfsDeviceCharacteristics(sqlite3_file *pFile){
4942 rbu_file *p = (rbu_file *)pFile;
4943 return p->pReal->pMethods->xDeviceCharacteristics(p->pReal);
4944 }
4945
4946 /*
4947 ** Take or release a shared-memory lock.
4948 */
4949 static int rbuVfsShmLock(sqlite3_file *pFile, int ofst, int n, int flags){
4950 rbu_file *p = (rbu_file*)pFile;
4951 sqlite3rbu *pRbu = p->pRbu;
4952 int rc = SQLITE_OK;
4953
4954 #ifdef SQLITE_AMALGAMATION
4955 assert( WAL_CKPT_LOCK==1 );
4956 #endif
4957
4958 assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
4959 if( pRbu && (
4960 pRbu->eStage==RBU_STAGE_OAL
4961 || pRbu->eStage==RBU_STAGE_MOVE
4962 || pRbu->eStage==RBU_STAGE_DONE
4963 )){
4964 /* Prevent SQLite from taking a shm-lock on the target file when it
4965 ** is supplying heap memory to the upper layer in place of *-shm
4966 ** segments. */
4967 if( ofst==WAL_LOCK_CKPT && n==1 ) rc = SQLITE_BUSY;
4968 }else{
4969 int bCapture = 0;
4970 if( pRbu && pRbu->eStage==RBU_STAGE_CAPTURE ){
4971 bCapture = 1;
4972 }
4973 if( bCapture==0 || 0==(flags & SQLITE_SHM_UNLOCK) ){
4974 rc = p->pReal->pMethods->xShmLock(p->pReal, ofst, n, flags);
4975 if( bCapture && rc==SQLITE_OK ){
4976 pRbu->mLock |= ((1<<n) - 1) << ofst;
4977 }
4978 }
4979 }
4980
4981 return rc;
4982 }
4983
4984 /*
4985 ** Obtain a pointer to a mapping of a single 32KiB page of the *-shm file.
4986 */
4987 static int rbuVfsShmMap(
4988 sqlite3_file *pFile,
4989 int iRegion,
4990 int szRegion,
4991 int isWrite,
4992 void volatile **pp
4993 ){
4994 rbu_file *p = (rbu_file*)pFile;
4995 int rc = SQLITE_OK;
4996 int eStage = (p->pRbu ? p->pRbu->eStage : 0);
4997
4998 /* If not in RBU_STAGE_OAL, allow this call to pass through. Or, if this
4999 ** rbu is in the RBU_STAGE_OAL state, use heap memory for *-shm space
5000 ** instead of a file on disk. */
5001 assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
5002 if( eStage==RBU_STAGE_OAL ){
5003 sqlite3_int64 nByte = (iRegion+1) * sizeof(char*);
5004 char **apNew = (char**)sqlite3_realloc64(p->apShm, nByte);
5005
5006 /* This is an RBU connection that uses its own heap memory for the
5007 ** pages of the *-shm file. Since no other process can have run
5008 ** recovery, the connection must request *-shm pages in order
5009 ** from start to finish. */
5010 assert( iRegion==p->nShm );
5011 if( apNew==0 ){
5012 rc = SQLITE_NOMEM;
5013 }else{
5014 memset(&apNew[p->nShm], 0, sizeof(char*) * (1 + iRegion - p->nShm));
5015 p->apShm = apNew;
5016 p->nShm = iRegion+1;
5017 }
5018
5019 if( rc==SQLITE_OK ){
5020 char *pNew = (char*)sqlite3_malloc64(szRegion);
5021 if( pNew==0 ){
5022 rc = SQLITE_NOMEM;
5023 }else{
5024 memset(pNew, 0, szRegion);
5025 p->apShm[iRegion] = pNew;
5026 }
5027 }
5028
5029 if( rc==SQLITE_OK ){
5030 *pp = p->apShm[iRegion];
5031 }else{
5032 *pp = 0;
5033 }
5034 }else{
5035 assert( p->apShm==0 );
5036 rc = p->pReal->pMethods->xShmMap(p->pReal, iRegion, szRegion, isWrite, pp);
5037 }
5038
5039 return rc;
5040 }
5041
5042 /*
5043 ** Memory barrier.
5044 */
5045 static void rbuVfsShmBarrier(sqlite3_file *pFile){
5046 rbu_file *p = (rbu_file *)pFile;
5047 p->pReal->pMethods->xShmBarrier(p->pReal);
5048 }
5049
5050 /*
5051 ** The xShmUnmap method.
5052 */
5053 static int rbuVfsShmUnmap(sqlite3_file *pFile, int delFlag){
5054 rbu_file *p = (rbu_file*)pFile;
5055 int rc = SQLITE_OK;
5056 int eStage = (p->pRbu ? p->pRbu->eStage : 0);
5057
5058 assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
5059 if( eStage==RBU_STAGE_OAL || eStage==RBU_STAGE_MOVE ){
5060 /* no-op */
5061 }else{
5062 /* Release the checkpointer and writer locks */
5063 rbuUnlockShm(p);
5064 rc = p->pReal->pMethods->xShmUnmap(p->pReal, delFlag);
5065 }
5066 return rc;
5067 }
5068
5069 /*
5070 ** Open an rbu file handle.
5071 */
5072 static int rbuVfsOpen(
5073 sqlite3_vfs *pVfs,
5074 const char *zName,
5075 sqlite3_file *pFile,
5076 int flags,
5077 int *pOutFlags
5078 ){
5079 static sqlite3_io_methods rbuvfs_io_methods = {
5080 2, /* iVersion */
5081 rbuVfsClose, /* xClose */
5082 rbuVfsRead, /* xRead */
5083 rbuVfsWrite, /* xWrite */
5084 rbuVfsTruncate, /* xTruncate */
5085 rbuVfsSync, /* xSync */
5086 rbuVfsFileSize, /* xFileSize */
5087 rbuVfsLock, /* xLock */
5088 rbuVfsUnlock, /* xUnlock */
5089 rbuVfsCheckReservedLock, /* xCheckReservedLock */
5090 rbuVfsFileControl, /* xFileControl */
5091 rbuVfsSectorSize, /* xSectorSize */
5092 rbuVfsDeviceCharacteristics, /* xDeviceCharacteristics */
5093 rbuVfsShmMap, /* xShmMap */
5094 rbuVfsShmLock, /* xShmLock */
5095 rbuVfsShmBarrier, /* xShmBarrier */
5096 rbuVfsShmUnmap, /* xShmUnmap */
5097 0, 0 /* xFetch, xUnfetch */
5098 };
5099 static sqlite3_io_methods rbuvfs_io_methods1 = {
5100 1, /* iVersion */
5101 rbuVfsClose, /* xClose */
5102 rbuVfsRead, /* xRead */
5103 rbuVfsWrite, /* xWrite */
5104 rbuVfsTruncate, /* xTruncate */
5105 rbuVfsSync, /* xSync */
5106 rbuVfsFileSize, /* xFileSize */
5107 rbuVfsLock, /* xLock */
5108 rbuVfsUnlock, /* xUnlock */
5109 rbuVfsCheckReservedLock, /* xCheckReservedLock */
5110 rbuVfsFileControl, /* xFileControl */
5111 rbuVfsSectorSize, /* xSectorSize */
5112 rbuVfsDeviceCharacteristics, /* xDeviceCharacteristics */
5113 0, 0, 0, 0, 0, 0
5114 };
5115
5116
5117
5118 rbu_vfs *pRbuVfs = (rbu_vfs*)pVfs;
5119 sqlite3_vfs *pRealVfs = pRbuVfs->pRealVfs;
5120 rbu_file *pFd = (rbu_file *)pFile;
5121 int rc = SQLITE_OK;
5122 const char *zOpen = zName;
5123 int oflags = flags;
5124
5125 memset(pFd, 0, sizeof(rbu_file));
5126 pFd->pReal = (sqlite3_file*)&pFd[1];
5127 pFd->pRbuVfs = pRbuVfs;
5128 pFd->openFlags = flags;
5129 if( zName ){
5130 if( flags & SQLITE_OPEN_MAIN_DB ){
5131 /* A main database has just been opened. The following block sets
5132 ** (pFd->zWal) to point to a buffer owned by SQLite that contains
5133 ** the name of the *-wal file this db connection will use. SQLite
5134 ** happens to pass a pointer to this buffer when using xAccess()
5135 ** or xOpen() to operate on the *-wal file. */
5136 pFd->zWal = sqlite3_filename_wal(zName);
5137 }
5138 else if( flags & SQLITE_OPEN_WAL ){
5139 rbu_file *pDb = rbuFindMaindb(pRbuVfs, zName, 0);
5140 if( pDb ){
5141 if( pDb->pRbu && pDb->pRbu->eStage==RBU_STAGE_OAL ){
5142 /* This call is to open a *-wal file. Intead, open the *-oal. */
5143 size_t nOpen;
5144 if( rbuIsVacuum(pDb->pRbu) ){
5145 zOpen = sqlite3_db_filename(pDb->pRbu->dbRbu, "main");
5146 zOpen = sqlite3_filename_wal(zOpen);
5147 }
5148 nOpen = strlen(zOpen);
5149 ((char*)zOpen)[nOpen-3] = 'o';
5150 pFd->pRbu = pDb->pRbu;
5151 }
5152 pDb->pWalFd = pFd;
5153 }
5154 }
5155 }else{
5156 pFd->pRbu = pRbuVfs->pRbu;
5157 }
5158
5159 if( oflags & SQLITE_OPEN_MAIN_DB
5160 && sqlite3_uri_boolean(zName, "rbu_memory", 0)
5161 ){
5162 assert( oflags & SQLITE_OPEN_MAIN_DB );
5163 oflags = SQLITE_OPEN_TEMP_DB | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE |
5164 SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE;
5165 zOpen = 0;
5166 }
5167
5168 if( rc==SQLITE_OK ){
5169 rc = pRealVfs->xOpen(pRealVfs, zOpen, pFd->pReal, oflags, pOutFlags);
5170 }
5171 if( pFd->pReal->pMethods ){
5172 const sqlite3_io_methods *pMeth = pFd->pReal->pMethods;
5173 /* The xOpen() operation has succeeded. Set the sqlite3_file.pMethods
5174 ** pointer and, if the file is a main database file, link it into the
5175 ** mutex protected linked list of all such files. */
5176 if( pMeth->iVersion<2 || pMeth->xShmLock==0 ){
5177 pFile->pMethods = &rbuvfs_io_methods1;
5178 }else{
5179 pFile->pMethods = &rbuvfs_io_methods;
5180 }
5181 if( flags & SQLITE_OPEN_MAIN_DB ){
5182 rbuMainlistAdd(pFd);
5183 }
5184 }else{
5185 sqlite3_free(pFd->zDel);
5186 }
5187
5188 return rc;
5189 }
5190
5191 /*
5192 ** Delete the file located at zPath.
5193 */
5194 static int rbuVfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
5195 sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
5196 return pRealVfs->xDelete(pRealVfs, zPath, dirSync);
5197 }
5198
5199 /*
5200 ** Test for access permissions. Return true if the requested permission
5201 ** is available, or false otherwise.
5202 */
5203 static int rbuVfsAccess(
5204 sqlite3_vfs *pVfs,
5205 const char *zPath,
5206 int flags,
5207 int *pResOut
5208 ){
5209 rbu_vfs *pRbuVfs = (rbu_vfs*)pVfs;
5210 sqlite3_vfs *pRealVfs = pRbuVfs->pRealVfs;
5211 int rc;
5212
5213 rc = pRealVfs->xAccess(pRealVfs, zPath, flags, pResOut);
5214
5215 /* If this call is to check if a *-wal file associated with an RBU target
5216 ** database connection exists, and the RBU update is in RBU_STAGE_OAL,
5217 ** the following special handling is activated:
5218 **
5219 ** a) if the *-wal file does exist, return SQLITE_CANTOPEN. This
5220 ** ensures that the RBU extension never tries to update a database
5221 ** in wal mode, even if the first page of the database file has
5222 ** been damaged.
5223 **
5224 ** b) if the *-wal file does not exist, claim that it does anyway,
5225 ** causing SQLite to call xOpen() to open it. This call will also
5226 ** be intercepted (see the rbuVfsOpen() function) and the *-oal
5227 ** file opened instead.
5228 */
5229 if( rc==SQLITE_OK && flags==SQLITE_ACCESS_EXISTS ){
5230 rbu_file *pDb = rbuFindMaindb(pRbuVfs, zPath, 1);
5231 if( pDb && pDb->pRbu->eStage==RBU_STAGE_OAL ){
5232 assert( pDb->pRbu );
5233 if( *pResOut ){
5234 rc = SQLITE_CANTOPEN;
5235 }else{
5236 sqlite3_int64 sz = 0;
5237 rc = rbuVfsFileSize(&pDb->base, &sz);
5238 *pResOut = (sz>0);
5239 }
5240 }
5241 }
5242
5243 return rc;
5244 }
5245
5246 /*
5247 ** Populate buffer zOut with the full canonical pathname corresponding
5248 ** to the pathname in zPath. zOut is guaranteed to point to a buffer
5249 ** of at least (DEVSYM_MAX_PATHNAME+1) bytes.
5250 */
5251 static int rbuVfsFullPathname(
5252 sqlite3_vfs *pVfs,
5253 const char *zPath,
5254 int nOut,
5255 char *zOut
5256 ){
5257 sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
5258 return pRealVfs->xFullPathname(pRealVfs, zPath, nOut, zOut);
5259 }
5260
5261 #ifndef SQLITE_OMIT_LOAD_EXTENSION
5262 /*
5263 ** Open the dynamic library located at zPath and return a handle.
5264 */
5265 static void *rbuVfsDlOpen(sqlite3_vfs *pVfs, const char *zPath){
5266 sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
5267 return pRealVfs->xDlOpen(pRealVfs, zPath);
5268 }
5269
5270 /*
5271 ** Populate the buffer zErrMsg (size nByte bytes) with a human readable
5272 ** utf-8 string describing the most recent error encountered associated
5273 ** with dynamic libraries.
5274 */
5275 static void rbuVfsDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){
5276 sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
5277 pRealVfs->xDlError(pRealVfs, nByte, zErrMsg);
5278 }
5279
5280 /*
5281 ** Return a pointer to the symbol zSymbol in the dynamic library pHandle.
5282 */
5283 static void (*rbuVfsDlSym(
5284 sqlite3_vfs *pVfs,
5285 void *pArg,
5286 const char *zSym
5287 ))(void){
5288 sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
5289 return pRealVfs->xDlSym(pRealVfs, pArg, zSym);
5290 }
5291
5292 /*
5293 ** Close the dynamic library handle pHandle.
5294 */
5295 static void rbuVfsDlClose(sqlite3_vfs *pVfs, void *pHandle){
5296 sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
5297 pRealVfs->xDlClose(pRealVfs, pHandle);
5298 }
5299 #endif /* SQLITE_OMIT_LOAD_EXTENSION */
5300
5301 /*
5302 ** Populate the buffer pointed to by zBufOut with nByte bytes of
5303 ** random data.
5304 */
5305 static int rbuVfsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){
5306 sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
5307 return pRealVfs->xRandomness(pRealVfs, nByte, zBufOut);
5308 }
5309
5310 /*
5311 ** Sleep for nMicro microseconds. Return the number of microseconds
5312 ** actually slept.
5313 */
5314 static int rbuVfsSleep(sqlite3_vfs *pVfs, int nMicro){
5315 sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
5316 return pRealVfs->xSleep(pRealVfs, nMicro);
5317 }
5318
5319 /*
5320 ** Return the current time as a Julian Day number in *pTimeOut.
5321 */
5322 static int rbuVfsCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){
5323 sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
5324 return pRealVfs->xCurrentTime(pRealVfs, pTimeOut);
5325 }
5326
5327 /*
5328 ** No-op.
5329 */
5330 static int rbuVfsGetLastError(sqlite3_vfs *pVfs, int a, char *b){
5331 UNUSED_PARAMETER(pVfs);
5332 UNUSED_PARAMETER(a);
5333 UNUSED_PARAMETER(b);
5334 return 0;
5335 }
5336
5337 /*
5338 ** Deregister and destroy an RBU vfs created by an earlier call to
5339 ** sqlite3rbu_create_vfs().
5340 */
5341 void sqlite3rbu_destroy_vfs(const char *zName){
5342 sqlite3_vfs *pVfs = sqlite3_vfs_find(zName);
5343 if( pVfs && pVfs->xOpen==rbuVfsOpen ){
5344 sqlite3_mutex_free(((rbu_vfs*)pVfs)->mutex);
5345 sqlite3_vfs_unregister(pVfs);
5346 sqlite3_free(pVfs);
5347 }
5348 }
5349
5350 /*
5351 ** Create an RBU VFS named zName that accesses the underlying file-system
5352 ** via existing VFS zParent. The new object is registered as a non-default
5353 ** VFS with SQLite before returning.
5354 */
5355 int sqlite3rbu_create_vfs(const char *zName, const char *zParent){
5356
5357 /* Template for VFS */
5358 static sqlite3_vfs vfs_template = {
5359 1, /* iVersion */
5360 0, /* szOsFile */
5361 0, /* mxPathname */
5362 0, /* pNext */
5363 0, /* zName */
5364 0, /* pAppData */
5365 rbuVfsOpen, /* xOpen */
5366 rbuVfsDelete, /* xDelete */
5367 rbuVfsAccess, /* xAccess */
5368 rbuVfsFullPathname, /* xFullPathname */
5369
5370 #ifndef SQLITE_OMIT_LOAD_EXTENSION
5371 rbuVfsDlOpen, /* xDlOpen */
5372 rbuVfsDlError, /* xDlError */
5373 rbuVfsDlSym, /* xDlSym */
5374 rbuVfsDlClose, /* xDlClose */
5375 #else
5376 0, 0, 0, 0,
5377 #endif
5378
5379 rbuVfsRandomness, /* xRandomness */
5380 rbuVfsSleep, /* xSleep */
5381 rbuVfsCurrentTime, /* xCurrentTime */
5382 rbuVfsGetLastError, /* xGetLastError */
5383 0, /* xCurrentTimeInt64 (version 2) */
5384 0, 0, 0 /* Unimplemented version 3 methods */
5385 };
5386
5387 rbu_vfs *pNew = 0; /* Newly allocated VFS */
5388 int rc = SQLITE_OK;
5389 size_t nName;
5390 size_t nByte;
5391
5392 nName = strlen(zName);
5393 nByte = sizeof(rbu_vfs) + nName + 1;
5394 pNew = (rbu_vfs*)sqlite3_malloc64(nByte);
5395 if( pNew==0 ){
5396 rc = SQLITE_NOMEM;
5397 }else{
5398 sqlite3_vfs *pParent; /* Parent VFS */
5399 memset(pNew, 0, nByte);
5400 pParent = sqlite3_vfs_find(zParent);
5401 if( pParent==0 ){
5402 rc = SQLITE_NOTFOUND;
5403 }else{
5404 char *zSpace;
5405 memcpy(&pNew->base, &vfs_template, sizeof(sqlite3_vfs));
5406 pNew->base.mxPathname = pParent->mxPathname;
5407 pNew->base.szOsFile = sizeof(rbu_file) + pParent->szOsFile;
5408 pNew->pRealVfs = pParent;
5409 pNew->base.zName = (const char*)(zSpace = (char*)&pNew[1]);
5410 memcpy(zSpace, zName, nName);
5411
5412 /* Allocate the mutex and register the new VFS (not as the default) */
5413 pNew->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_RECURSIVE);
5414 if( pNew->mutex==0 ){
5415 rc = SQLITE_NOMEM;
5416 }else{
5417 rc = sqlite3_vfs_register(&pNew->base, 0);
5418 }
5419 }
5420
5421 if( rc!=SQLITE_OK ){
5422 sqlite3_mutex_free(pNew->mutex);
5423 sqlite3_free(pNew);
5424 }
5425 }
5426
5427 return rc;
5428 }
5429
5430 /*
5431 ** Configure the aggregate temp file size limit for this RBU handle.
5432 */
5433 sqlite3_int64 sqlite3rbu_temp_size_limit(sqlite3rbu *pRbu, sqlite3_int64 n){
5434 if( n>=0 ){
5435 pRbu->szTempLimit = n;
5436 }
5437 return pRbu->szTempLimit;
5438 }
5439
5440 sqlite3_int64 sqlite3rbu_temp_size(sqlite3rbu *pRbu){
5441 return pRbu->szTemp;
5442 }
5443
5444
5445 /**************************************************************************/
5446
5447 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU) */