| 1 |
# 2005 November 30 |
| 2 |
# |
| 3 |
# The author disclaims copyright to this source code. In place of |
| 4 |
# a legal notice, here is a blessing: |
| 5 |
# |
| 6 |
# May you do good and not evil. |
| 7 |
# May you find forgiveness for yourself and forgive others. |
| 8 |
# May you share freely, never taking more than you give. |
| 9 |
# |
| 10 |
#*********************************************************************** |
| 11 |
# |
| 12 |
# This file contains test cases focused on the two memory-management APIs, |
| 13 |
# sqlite3_soft_heap_limit() and sqlite3_release_memory(). |
| 14 |
# |
| 15 |
# Prior to version 3.6.2, calling sqlite3_release_memory() or exceeding |
| 16 |
# the configured soft heap limit could cause sqlite to upgrade database |
| 17 |
# locks and flush dirty pages to the file system. As of 3.6.2, this is |
| 18 |
# no longer the case. In version 3.6.2, sqlite3_release_memory() only |
| 19 |
# reclaims clean pages. This test file has been updated accordingly. |
| 20 |
# |
| 21 |
# $Id: malloc5.test,v 1.22 2009/04/11 19:09:54 drh Exp $ |
| 22 |
|
| 23 |
set testdir [file dirname $argv0] |
| 24 |
source $testdir/tester.tcl |
| 25 |
source $testdir/malloc_common.tcl |
| 26 |
db close |
| 27 |
|
| 28 |
# Only run these tests if memory debugging is turned on. |
| 29 |
# |
| 30 |
if {!$MEMDEBUG} { |
| 31 |
puts "Skipping malloc5 tests: not compiled with -DSQLITE_MEMDEBUG..." |
| 32 |
finish_test |
| 33 |
return |
| 34 |
} |
| 35 |
|
| 36 |
# Skip these tests if OMIT_MEMORY_MANAGEMENT was defined at compile time. |
| 37 |
ifcapable !memorymanage { |
| 38 |
finish_test |
| 39 |
return |
| 40 |
} |
| 41 |
|
| 42 |
# The sizes of memory allocations from system malloc() might vary, |
| 43 |
# depending on the memory allocator algorithms used. The following |
| 44 |
# routine is designed to support answers that fall within a range |
| 45 |
# of values while also supplying easy-to-understand "expected" values |
| 46 |
# when errors occur. |
| 47 |
# |
| 48 |
proc value_in_range {target x args} { |
| 49 |
set v [lindex $args 0] |
| 50 |
if {$v!=""} { |
| 51 |
if {$v<$target*$x} {return $v} |
| 52 |
if {$v>$target/$x} {return $v} |
| 53 |
} |
| 54 |
return "number between [expr {int($target*$x)}] and [expr {int($target/$x)}]" |
| 55 |
} |
| 56 |
set mrange 0.98 ;# plus or minus 2% |
| 57 |
|
| 58 |
test_set_config_pagecache 0 100 |
| 59 |
|
| 60 |
sqlite3_soft_heap_limit 0 |
| 61 |
sqlite3 db test.db |
| 62 |
# db eval {PRAGMA cache_size=1} |
| 63 |
|
| 64 |
do_test malloc5-1.1 { |
| 65 |
# Simplest possible test. Call sqlite3_release_memory when there is exactly |
| 66 |
# one unused page in a single pager cache. The page cannot be freed, as |
| 67 |
# it is dirty. So sqlite3_release_memory() returns 0. |
| 68 |
# |
| 69 |
execsql { |
| 70 |
PRAGMA auto_vacuum=OFF; |
| 71 |
BEGIN; |
| 72 |
CREATE TABLE abc(a, b, c); |
| 73 |
} |
| 74 |
sqlite3_release_memory |
| 75 |
} {0} |
| 76 |
|
| 77 |
do_test malloc5-1.2 { |
| 78 |
# Test that the transaction started in the above test is still active. |
| 79 |
# The lock on the database file should not have been upgraded (this was |
| 80 |
# not the case before version 3.6.2). |
| 81 |
# |
| 82 |
sqlite3 db2 test.db |
| 83 |
execsql {PRAGMA cache_size=2; SELECT * FROM sqlite_master } db2 |
| 84 |
} {} |
| 85 |
do_test malloc5-1.3 { |
| 86 |
# Call [sqlite3_release_memory] when there is exactly one unused page |
| 87 |
# in the cache belonging to db2. |
| 88 |
# |
| 89 |
set ::pgalloc [sqlite3_release_memory] |
| 90 |
value_in_range 1288 0.75 |
| 91 |
} [value_in_range 1288 0.75] |
| 92 |
|
| 93 |
do_test malloc5-1.4 { |
| 94 |
# Commit the transaction and open a new one. Read 1 page into the cache. |
| 95 |
# Because the page is not dirty, it is eligible for collection even |
| 96 |
# before the transaction is concluded. |
| 97 |
# |
| 98 |
execsql { |
| 99 |
COMMIT; |
| 100 |
BEGIN; |
| 101 |
SELECT * FROM abc; |
| 102 |
} |
| 103 |
value_in_range $::pgalloc $::mrange [sqlite3_release_memory] |
| 104 |
} [value_in_range $::pgalloc $::mrange] |
| 105 |
|
| 106 |
do_test malloc5-1.5 { |
| 107 |
# Conclude the transaction opened in the previous [do_test] block. This |
| 108 |
# causes another page (page 1) to become eligible for recycling. |
| 109 |
# |
| 110 |
execsql { COMMIT } |
| 111 |
value_in_range $::pgalloc $::mrange [sqlite3_release_memory] |
| 112 |
} [value_in_range $::pgalloc $::mrange] |
| 113 |
|
| 114 |
do_test malloc5-1.6 { |
| 115 |
# Manipulate the cache so that it contains two unused pages. One requires |
| 116 |
# a journal-sync to free, the other does not. |
| 117 |
db2 close |
| 118 |
execsql { |
| 119 |
BEGIN; |
| 120 |
CREATE TABLE def(d, e, f); |
| 121 |
SELECT * FROM abc; |
| 122 |
} |
| 123 |
value_in_range $::pgalloc $::mrange [sqlite3_release_memory 500] |
| 124 |
} [value_in_range $::pgalloc $::mrange] |
| 125 |
do_test malloc5-1.7 { |
| 126 |
# Database should not be locked this time. |
| 127 |
sqlite3 db2 test.db |
| 128 |
catchsql { SELECT * FROM abc } db2 |
| 129 |
} {0 {}} |
| 130 |
do_test malloc5-1.8 { |
| 131 |
# Try to release another block of memory. This will fail as the only |
| 132 |
# pages currently in the cache are dirty (page 3) or pinned (page 1). |
| 133 |
db2 close |
| 134 |
sqlite3_release_memory 500 |
| 135 |
} 0 |
| 136 |
do_test malloc5-1.8 { |
| 137 |
# Database is still not locked. |
| 138 |
# |
| 139 |
sqlite3 db2 test.db |
| 140 |
catchsql { SELECT * FROM abc } db2 |
| 141 |
} {0 {}} |
| 142 |
do_test malloc5-1.9 { |
| 143 |
execsql { |
| 144 |
COMMIT; |
| 145 |
} |
| 146 |
} {} |
| 147 |
|
| 148 |
do_test malloc5-2.1 { |
| 149 |
# Put some data in tables abc and def. Both tables are still wholly |
| 150 |
# contained within their root pages. |
| 151 |
execsql { |
| 152 |
INSERT INTO abc VALUES(1, 2, 3); |
| 153 |
INSERT INTO abc VALUES(4, 5, 6); |
| 154 |
INSERT INTO def VALUES(7, 8, 9); |
| 155 |
INSERT INTO def VALUES(10,11,12); |
| 156 |
} |
| 157 |
} {} |
| 158 |
do_test malloc5-2.2 { |
| 159 |
# Load the root-page for table def into the cache. Then query table abc. |
| 160 |
# Halfway through the query call sqlite3_release_memory(). The goal of this |
| 161 |
# test is to make sure we don't free pages that are in use (specifically, |
| 162 |
# the root of table abc). |
| 163 |
sqlite3_release_memory |
| 164 |
set nRelease 0 |
| 165 |
execsql { |
| 166 |
BEGIN; |
| 167 |
SELECT * FROM def; |
| 168 |
} |
| 169 |
set data [list] |
| 170 |
db eval {SELECT * FROM abc} { |
| 171 |
incr nRelease [sqlite3_release_memory] |
| 172 |
lappend data $a $b $c |
| 173 |
} |
| 174 |
execsql { |
| 175 |
COMMIT; |
| 176 |
} |
| 177 |
value_in_range $::pgalloc $::mrange $nRelease |
| 178 |
} [value_in_range $::pgalloc $::mrange] |
| 179 |
do_test malloc5-2.2.1 { |
| 180 |
set data |
| 181 |
} {1 2 3 4 5 6} |
| 182 |
|
| 183 |
do_test malloc5-3.1 { |
| 184 |
# Simple test to show that if two pagers are opened from within this |
| 185 |
# thread, memory is freed from both when sqlite3_release_memory() is |
| 186 |
# called. |
| 187 |
execsql { |
| 188 |
BEGIN; |
| 189 |
SELECT * FROM abc; |
| 190 |
} |
| 191 |
execsql { |
| 192 |
SELECT * FROM sqlite_master; |
| 193 |
BEGIN; |
| 194 |
SELECT * FROM def; |
| 195 |
} db2 |
| 196 |
value_in_range [expr $::pgalloc*2] 0.99 [sqlite3_release_memory] |
| 197 |
} [value_in_range [expr $::pgalloc * 2] 0.99] |
| 198 |
do_test malloc5-3.2 { |
| 199 |
concat \ |
| 200 |
[execsql {SELECT * FROM abc; COMMIT}] \ |
| 201 |
[execsql {SELECT * FROM def; COMMIT} db2] |
| 202 |
} {1 2 3 4 5 6 7 8 9 10 11 12} |
| 203 |
|
| 204 |
db2 close |
| 205 |
puts "Highwater mark: [sqlite3_memory_highwater]" |
| 206 |
|
| 207 |
# The following two test cases each execute a transaction in which |
| 208 |
# 10000 rows are inserted into table abc. The first test case is used |
| 209 |
# to ensure that more than 1MB of dynamic memory is used to perform |
| 210 |
# the transaction. |
| 211 |
# |
| 212 |
# The second test case sets the "soft-heap-limit" to 100,000 bytes (0.1 MB) |
| 213 |
# and tests to see that this limit is not exceeded at any point during |
| 214 |
# transaction execution. |
| 215 |
# |
| 216 |
# Before executing malloc5-4.* we save the value of the current soft heap |
| 217 |
# limit in variable ::soft_limit. The original value is restored after |
| 218 |
# running the tests. |
| 219 |
# |
| 220 |
set ::soft_limit [sqlite3_soft_heap_limit -1] |
| 221 |
execsql {PRAGMA cache_size=2000} |
| 222 |
do_test malloc5-4.1 { |
| 223 |
execsql {BEGIN;} |
| 224 |
execsql {DELETE FROM abc;} |
| 225 |
for {set i 0} {$i < 10000} {incr i} { |
| 226 |
execsql "INSERT INTO abc VALUES($i, $i, '[string repeat X 100]');" |
| 227 |
} |
| 228 |
execsql {COMMIT;} |
| 229 |
db cache flush |
| 230 |
sqlite3_release_memory |
| 231 |
sqlite3_memory_highwater 1 |
| 232 |
execsql {SELECT * FROM abc} |
| 233 |
set nMaxBytes [sqlite3_memory_highwater 1] |
| 234 |
puts -nonewline " (Highwater mark: $nMaxBytes) " |
| 235 |
expr $nMaxBytes > 1000000 |
| 236 |
} {1} |
| 237 |
do_test malloc5-4.2 { |
| 238 |
db eval {PRAGMA cache_size=1} |
| 239 |
db cache flush |
| 240 |
sqlite3_release_memory |
| 241 |
sqlite3_soft_heap_limit 200000 |
| 242 |
sqlite3_memory_highwater 1 |
| 243 |
execsql {SELECT * FROM abc} |
| 244 |
set nMaxBytes [sqlite3_memory_highwater 1] |
| 245 |
puts -nonewline " (Highwater mark: $nMaxBytes) " |
| 246 |
expr $nMaxBytes <= 210000 |
| 247 |
} {1} |
| 248 |
do_test malloc5-4.3 { |
| 249 |
# Check that the content of table abc is at least roughly as expected. |
| 250 |
execsql { |
| 251 |
SELECT count(*), sum(a), sum(b) FROM abc; |
| 252 |
} |
| 253 |
} [list 10000 [expr int(10000.0 * 4999.5)] [expr int(10000.0 * 4999.5)]] |
| 254 |
|
| 255 |
# Restore the soft heap limit. |
| 256 |
sqlite3_soft_heap_limit $::soft_limit |
| 257 |
|
| 258 |
# Test that there are no problems calling sqlite3_release_memory when |
| 259 |
# there are open in-memory databases. |
| 260 |
# |
| 261 |
# At one point these tests would cause a seg-fault. |
| 262 |
# |
| 263 |
do_test malloc5-5.1 { |
| 264 |
db close |
| 265 |
sqlite3 db :memory: |
| 266 |
execsql { |
| 267 |
BEGIN; |
| 268 |
CREATE TABLE abc(a, b, c); |
| 269 |
INSERT INTO abc VALUES('abcdefghi', 1234567890, NULL); |
| 270 |
INSERT INTO abc SELECT * FROM abc; |
| 271 |
INSERT INTO abc SELECT * FROM abc; |
| 272 |
INSERT INTO abc SELECT * FROM abc; |
| 273 |
INSERT INTO abc SELECT * FROM abc; |
| 274 |
INSERT INTO abc SELECT * FROM abc; |
| 275 |
INSERT INTO abc SELECT * FROM abc; |
| 276 |
INSERT INTO abc SELECT * FROM abc; |
| 277 |
} |
| 278 |
sqlite3_release_memory |
| 279 |
} 0 |
| 280 |
do_test malloc5-5.2 { |
| 281 |
sqlite3_soft_heap_limit 5000 |
| 282 |
execsql { |
| 283 |
COMMIT; |
| 284 |
PRAGMA temp_store = memory; |
| 285 |
SELECT * FROM abc ORDER BY a; |
| 286 |
} |
| 287 |
expr 1 |
| 288 |
} {1} |
| 289 |
sqlite3_soft_heap_limit $::soft_limit |
| 290 |
|
| 291 |
#------------------------------------------------------------------------- |
| 292 |
# The following test cases (malloc5-6.*) test the new global LRU list |
| 293 |
# used to determine the pages to recycle when sqlite3_release_memory is |
| 294 |
# called and there is more than one pager open. |
| 295 |
# |
| 296 |
proc nPage {db} { |
| 297 |
set bt [btree_from_db $db] |
| 298 |
array set stats [btree_pager_stats $bt] |
| 299 |
set stats(page) |
| 300 |
} |
| 301 |
db close |
| 302 |
forcedelete test.db test.db-journal test2.db test2.db-journal |
| 303 |
|
| 304 |
# This block of test-cases (malloc5-6.1.*) prepares two database files |
| 305 |
# for the subsequent tests. |
| 306 |
do_test malloc5-6.1.1 { |
| 307 |
sqlite3 db test.db |
| 308 |
execsql { |
| 309 |
PRAGMA page_size=1024; |
| 310 |
PRAGMA default_cache_size=2; |
| 311 |
} |
| 312 |
execsql { |
| 313 |
PRAGMA temp_store = memory; |
| 314 |
BEGIN; |
| 315 |
CREATE TABLE abc(a PRIMARY KEY, b, c); |
| 316 |
INSERT INTO abc VALUES(randstr(50,50), randstr(75,75), randstr(100,100)); |
| 317 |
INSERT INTO abc |
| 318 |
SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc; |
| 319 |
INSERT INTO abc |
| 320 |
SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc; |
| 321 |
INSERT INTO abc |
| 322 |
SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc; |
| 323 |
INSERT INTO abc |
| 324 |
SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc; |
| 325 |
INSERT INTO abc |
| 326 |
SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc; |
| 327 |
INSERT INTO abc |
| 328 |
SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc; |
| 329 |
COMMIT; |
| 330 |
} |
| 331 |
forcecopy test.db test2.db |
| 332 |
sqlite3 db2 test2.db |
| 333 |
db2 eval {PRAGMA cache_size=2} |
| 334 |
list \ |
| 335 |
[expr ([file size test.db]/1024)>20] [expr ([file size test2.db]/1024)>20] |
| 336 |
} {1 1} |
| 337 |
do_test malloc5-6.1.2 { |
| 338 |
list [execsql {PRAGMA cache_size}] [execsql {PRAGMA cache_size} db2] |
| 339 |
} {2 2} |
| 340 |
|
| 341 |
do_test malloc5-6.2.1 { |
| 342 |
execsql {SELECT * FROM abc} db2 |
| 343 |
execsql {SELECT * FROM abc} db |
| 344 |
expr [nPage db] + [nPage db2] |
| 345 |
} {4} |
| 346 |
|
| 347 |
do_test malloc5-6.2.2 { |
| 348 |
# If we now try to reclaim some memory, it should come from the db2 cache. |
| 349 |
sqlite3_release_memory 3000 |
| 350 |
expr [nPage db] + [nPage db2] |
| 351 |
} {1} |
| 352 |
do_test malloc5-6.2.3 { |
| 353 |
# Access the db2 cache again, so that all the db2 pages have been used |
| 354 |
# more recently than all the db pages. Then try to reclaim 3000 bytes. |
| 355 |
# This time, 3 pages should be pulled from the db cache. |
| 356 |
execsql { SELECT * FROM abc } db2 |
| 357 |
sqlite3_release_memory 3000 |
| 358 |
expr [nPage db] + [nPage db2] |
| 359 |
} {0} |
| 360 |
|
| 361 |
do_test malloc5-6.3.1 { |
| 362 |
# Now open a transaction and update 2 pages in the db2 cache. Then |
| 363 |
# do a SELECT on the db cache so that all the db pages are more recently |
| 364 |
# used than the db2 pages. When we try to free memory, SQLite should |
| 365 |
# free the non-dirty db2 pages, then the db pages, then finally use |
| 366 |
# sync() to free up the dirty db2 pages. The only page that cannot be |
| 367 |
# freed is page1 of db2. Because there is an open transaction, the |
| 368 |
# btree layer holds a reference to page 1 in the db2 cache. |
| 369 |
# |
| 370 |
# UPDATE: No longer. As release_memory() does not cause a sync() |
| 371 |
execsql { |
| 372 |
BEGIN; |
| 373 |
UPDATE abc SET c = randstr(100,100) |
| 374 |
WHERE rowid = 1 OR rowid = (SELECT max(rowid) FROM abc); |
| 375 |
} db2 |
| 376 |
execsql { SELECT * FROM abc } db |
| 377 |
expr [nPage db] + [nPage db2] |
| 378 |
} {4} |
| 379 |
do_test malloc5-6.3.2 { |
| 380 |
# Try to release 7700 bytes. This should release all the |
| 381 |
# non-dirty pages held by db2. |
| 382 |
sqlite3_release_memory [expr 7*1132] |
| 383 |
list [nPage db] [nPage db2] |
| 384 |
} {0 3} |
| 385 |
do_test malloc5-6.3.3 { |
| 386 |
# Try to release another 1000 bytes. This should come fromt the db |
| 387 |
# cache, since all three pages held by db2 are either in-use or diry. |
| 388 |
sqlite3_release_memory 1000 |
| 389 |
list [nPage db] [nPage db2] |
| 390 |
} {0 3} |
| 391 |
do_test malloc5-6.3.4 { |
| 392 |
# Now release 9900 more (about 9 pages worth). This should expunge |
| 393 |
# the rest of the db cache. But the db2 cache remains intact, because |
| 394 |
# SQLite tries to avoid calling sync(). |
| 395 |
if {$::tcl_platform(wordSize)==8} { |
| 396 |
sqlite3_release_memory 10500 |
| 397 |
} else { |
| 398 |
sqlite3_release_memory 9900 |
| 399 |
} |
| 400 |
list [nPage db] [nPage db2] |
| 401 |
} {0 3} |
| 402 |
do_test malloc5-6.3.5 { |
| 403 |
# But if we are really insistent, SQLite will consent to call sync() |
| 404 |
# if there is no other option. UPDATE: As of 3.6.2, SQLite will not |
| 405 |
# call sync() in this scenario. So no further memory can be reclaimed. |
| 406 |
sqlite3_release_memory 1000 |
| 407 |
list [nPage db] [nPage db2] |
| 408 |
} {0 3} |
| 409 |
do_test malloc5-6.3.6 { |
| 410 |
# The referenced page (page 1 of the db2 cache) will not be freed no |
| 411 |
# matter how much memory we ask for: |
| 412 |
sqlite3_release_memory 31459 |
| 413 |
list [nPage db] [nPage db2] |
| 414 |
} {0 3} |
| 415 |
|
| 416 |
db2 close |
| 417 |
|
| 418 |
sqlite3_soft_heap_limit $::soft_limit |
| 419 |
test_restore_config_pagecache |
| 420 |
finish_test |
| 421 |
catch {db close} |