sl@0: # 2005 November 30
sl@0: #
sl@0: # The author disclaims copyright to this source code.  In place of
sl@0: # a legal notice, here is a blessing:
sl@0: #
sl@0: #    May you do good and not evil.
sl@0: #    May you find forgiveness for yourself and forgive others.
sl@0: #    May you share freely, never taking more than you give.
sl@0: #
sl@0: #***********************************************************************
sl@0: #
sl@0: # This file contains test cases focused on the two memory-management APIs, 
sl@0: # sqlite3_soft_heap_limit() and sqlite3_release_memory().
sl@0: #
sl@0: # Prior to version 3.6.2, calling sqlite3_release_memory() or exceeding
sl@0: # the configured soft heap limit could cause sqlite to upgrade database 
sl@0: # locks and flush dirty pages to the file system. As of 3.6.2, this is
sl@0: # no longer the case. In version 3.6.2, sqlite3_release_memory() only
sl@0: # reclaims clean pages. This test file has been updated accordingly.
sl@0: #
sl@0: # $Id: malloc5.test,v 1.20 2008/08/27 16:38:57 danielk1977 Exp $
sl@0: 
sl@0: set testdir [file dirname $argv0]
sl@0: source $testdir/tester.tcl
sl@0: source $testdir/malloc_common.tcl
sl@0: db close
sl@0: 
sl@0: # Only run these tests if memory debugging is turned on.
sl@0: #
sl@0: if {!$MEMDEBUG} {
sl@0:    puts "Skipping malloc5 tests: not compiled with -DSQLITE_MEMDEBUG..."
sl@0:    finish_test
sl@0:    return
sl@0: }
sl@0: 
sl@0: # Skip these tests if OMIT_MEMORY_MANAGEMENT was defined at compile time.
sl@0: ifcapable !memorymanage {
sl@0:    finish_test
sl@0:    return
sl@0: }
sl@0: 
sl@0: sqlite3_soft_heap_limit 0
sl@0: sqlite3 db test.db
sl@0: 
sl@0: do_test malloc5-1.1 {
sl@0:   # Simplest possible test. Call sqlite3_release_memory when there is exactly
sl@0:   # one unused page in a single pager cache. The page cannot be freed, as
sl@0:   # it is dirty. So sqlite3_release_memory() returns 0.
sl@0:   #
sl@0:   execsql {
sl@0:     PRAGMA auto_vacuum=OFF;
sl@0:     BEGIN;
sl@0:     CREATE TABLE abc(a, b, c);
sl@0:   }
sl@0:   sqlite3_release_memory
sl@0: } {0}
sl@0: 
sl@0: do_test malloc5-1.2 {
sl@0:   # Test that the transaction started in the above test is still active.
sl@0:   # The lock on the database file should not have been upgraded (this was
sl@0:   # not the case before version 3.6.2).
sl@0:   #
sl@0:   sqlite3 db2 test.db
sl@0:   execsql { SELECT * FROM sqlite_master } db2
sl@0: } {}
sl@0: do_test malloc5-1.3 {
sl@0:   # Call [sqlite3_release_memory] when there is exactly one unused page 
sl@0:   # in the cache belonging to db2.
sl@0:   #
sl@0:   set ::pgalloc [sqlite3_release_memory]
sl@0:   expr $::pgalloc > 0
sl@0: } {1}
sl@0: 
sl@0: do_test malloc5-1.4 {
sl@0:   # Commit the transaction and open a new one. Read 1 page into the cache.
sl@0:   # Because the page is not dirty, it is eligible for collection even
sl@0:   # before the transaction is concluded.
sl@0:   #
sl@0:   execsql {
sl@0:     COMMIT;
sl@0:     BEGIN;
sl@0:     SELECT * FROM abc;
sl@0:   }
sl@0:   sqlite3_release_memory
sl@0: } $::pgalloc
sl@0: 
sl@0: do_test malloc5-1.5 {
sl@0:   # Conclude the transaction opened in the previous [do_test] block. This
sl@0:   # causes another page (page 1) to become eligible for recycling.
sl@0:   #
sl@0:   execsql { COMMIT }
sl@0:   sqlite3_release_memory
sl@0: } $::pgalloc
sl@0: 
sl@0: do_test malloc5-1.6 {
sl@0:   # Manipulate the cache so that it contains two unused pages. One requires 
sl@0:   # a journal-sync to free, the other does not.
sl@0:   db2 close
sl@0:   execsql {
sl@0:     BEGIN;
sl@0:     SELECT * FROM abc;
sl@0:     CREATE TABLE def(d, e, f);
sl@0:   }
sl@0:   sqlite3_release_memory 500
sl@0: } $::pgalloc
sl@0: 
sl@0: do_test malloc5-1.7 {
sl@0:   # Database should not be locked this time. 
sl@0:   sqlite3 db2 test.db
sl@0:   catchsql { SELECT * FROM abc } db2
sl@0: } {0 {}}
sl@0: do_test malloc5-1.8 {
sl@0:   # Try to release another block of memory. This will fail as the only
sl@0:   # pages currently in the cache are dirty (page 3) or pinned (page 1).
sl@0:   db2 close
sl@0:   sqlite3_release_memory 500
sl@0: } 0
sl@0: do_test malloc5-1.8 {
sl@0:   # Database is still not locked.
sl@0:   #
sl@0:   sqlite3 db2 test.db
sl@0:   catchsql { SELECT * FROM abc } db2
sl@0: } {0 {}}
sl@0: do_test malloc5-1.9 {
sl@0:   execsql {
sl@0:     COMMIT;
sl@0:   }
sl@0: } {}
sl@0: 
sl@0: do_test malloc5-2.1 {
sl@0:   # Put some data in tables abc and def. Both tables are still wholly 
sl@0:   # contained within their root pages.
sl@0:   execsql {
sl@0:     INSERT INTO abc VALUES(1, 2, 3);
sl@0:     INSERT INTO abc VALUES(4, 5, 6);
sl@0:     INSERT INTO def VALUES(7, 8, 9);
sl@0:     INSERT INTO def VALUES(10,11,12);
sl@0:   }
sl@0: } {}
sl@0: do_test malloc5-2.2 {
sl@0:   # Load the root-page for table def into the cache. Then query table abc. 
sl@0:   # Halfway through the query call sqlite3_release_memory(). The goal of this
sl@0:   # test is to make sure we don't free pages that are in use (specifically, 
sl@0:   # the root of table abc).
sl@0:   sqlite3_release_memory
sl@0:   set nRelease 0
sl@0:   execsql { 
sl@0:     BEGIN;
sl@0:     SELECT * FROM def;
sl@0:   }
sl@0:   set data [list]
sl@0:   db eval {SELECT * FROM abc} {
sl@0:     incr nRelease [sqlite3_release_memory]
sl@0:     lappend data $a $b $c
sl@0:   }
sl@0:   execsql {
sl@0:     COMMIT;
sl@0:   }
sl@0:   list $nRelease $data
sl@0: } [list $pgalloc [list 1 2 3 4 5 6]]
sl@0: 
sl@0: do_test malloc5-3.1 {
sl@0:   # Simple test to show that if two pagers are opened from within this
sl@0:   # thread, memory is freed from both when sqlite3_release_memory() is
sl@0:   # called.
sl@0:   execsql {
sl@0:     BEGIN;
sl@0:     SELECT * FROM abc;
sl@0:   }
sl@0:   execsql {
sl@0:     SELECT * FROM sqlite_master;
sl@0:     BEGIN;
sl@0:     SELECT * FROM def;
sl@0:   } db2
sl@0:   sqlite3_release_memory
sl@0: } [expr $::pgalloc * 2]
sl@0: do_test malloc5-3.2 {
sl@0:   concat \
sl@0:     [execsql {SELECT * FROM abc; COMMIT}] \
sl@0:     [execsql {SELECT * FROM def; COMMIT} db2]
sl@0: } {1 2 3 4 5 6 7 8 9 10 11 12}
sl@0: 
sl@0: db2 close
sl@0: puts "Highwater mark: [sqlite3_memory_highwater]"
sl@0: 
sl@0: # The following two test cases each execute a transaction in which 
sl@0: # 10000 rows are inserted into table abc. The first test case is used
sl@0: # to ensure that more than 1MB of dynamic memory is used to perform
sl@0: # the transaction. 
sl@0: #
sl@0: # The second test case sets the "soft-heap-limit" to 100,000 bytes (0.1 MB)
sl@0: # and tests to see that this limit is not exceeded at any point during 
sl@0: # transaction execution.
sl@0: #
sl@0: # Before executing malloc5-4.* we save the value of the current soft heap 
sl@0: # limit in variable ::soft_limit. The original value is restored after 
sl@0: # running the tests.
sl@0: #
sl@0: set ::soft_limit [sqlite3_soft_heap_limit -1]
sl@0: execsql {PRAGMA cache_size=2000}
sl@0: do_test malloc5-4.1 {
sl@0:   execsql {BEGIN;}
sl@0:   execsql {DELETE FROM abc;}
sl@0:   for {set i 0} {$i < 10000} {incr i} {
sl@0:     execsql "INSERT INTO abc VALUES($i, $i, '[string repeat X 100]');"
sl@0:   }
sl@0:   execsql {COMMIT;}
sl@0:   sqlite3_release_memory
sl@0:   sqlite3_memory_highwater 1
sl@0:   execsql {SELECT * FROM abc}
sl@0:   set nMaxBytes [sqlite3_memory_highwater 1]
sl@0:   puts -nonewline " (Highwater mark: $nMaxBytes) "
sl@0:   expr $nMaxBytes > 1000000
sl@0: } {1}
sl@0: do_test malloc5-4.2 {
sl@0:   sqlite3_release_memory
sl@0:   sqlite3_soft_heap_limit 100000
sl@0:   sqlite3_memory_highwater 1
sl@0:   execsql {SELECT * FROM abc}
sl@0:   set nMaxBytes [sqlite3_memory_highwater 1]
sl@0:   puts -nonewline " (Highwater mark: $nMaxBytes) "
sl@0:   expr $nMaxBytes <= 100000
sl@0: } {1}
sl@0: do_test malloc5-4.3 {
sl@0:   # Check that the content of table abc is at least roughly as expected.
sl@0:   execsql {
sl@0:     SELECT count(*), sum(a), sum(b) FROM abc;
sl@0:   }
sl@0: } [list 10000 [expr int(10000.0 * 4999.5)] [expr int(10000.0 * 4999.5)]]
sl@0: 
sl@0: # Restore the soft heap limit.
sl@0: sqlite3_soft_heap_limit $::soft_limit
sl@0: 
sl@0: # Test that there are no problems calling sqlite3_release_memory when
sl@0: # there are open in-memory databases.
sl@0: #
sl@0: # At one point these tests would cause a seg-fault.
sl@0: #
sl@0: do_test malloc5-5.1 {
sl@0:   db close
sl@0:   sqlite3 db :memory:
sl@0:   execsql {
sl@0:     BEGIN;
sl@0:     CREATE TABLE abc(a, b, c);
sl@0:     INSERT INTO abc VALUES('abcdefghi', 1234567890, NULL);
sl@0:     INSERT INTO abc SELECT * FROM abc;
sl@0:     INSERT INTO abc SELECT * FROM abc;
sl@0:     INSERT INTO abc SELECT * FROM abc;
sl@0:     INSERT INTO abc SELECT * FROM abc;
sl@0:     INSERT INTO abc SELECT * FROM abc;
sl@0:     INSERT INTO abc SELECT * FROM abc;
sl@0:     INSERT INTO abc SELECT * FROM abc;
sl@0:   }
sl@0:   sqlite3_release_memory
sl@0: } 0
sl@0: do_test malloc5-5.2 {
sl@0:   sqlite3_soft_heap_limit 5000
sl@0:   execsql {
sl@0:     COMMIT;
sl@0:     PRAGMA temp_store = memory;
sl@0:     SELECT * FROM abc ORDER BY a;
sl@0:   }
sl@0:   expr 1
sl@0: } {1}
sl@0: sqlite3_soft_heap_limit $::soft_limit
sl@0: 
sl@0: #-------------------------------------------------------------------------
sl@0: # The following test cases (malloc5-6.*) test the new global LRU list
sl@0: # used to determine the pages to recycle when sqlite3_release_memory is
sl@0: # called and there is more than one pager open.
sl@0: #
sl@0: proc nPage {db} {
sl@0:   set bt [btree_from_db $db]
sl@0:   array set stats [btree_pager_stats $bt]
sl@0:   set stats(page)
sl@0: }
sl@0: db close
sl@0: file delete -force test.db test.db-journal test2.db test2.db-journal
sl@0: 
sl@0: # This block of test-cases (malloc5-6.1.*) prepares two database files
sl@0: # for the subsequent tests.
sl@0: do_test malloc5-6.1.1 {
sl@0:   sqlite3 db test.db
sl@0:   execsql {
sl@0:     PRAGMA page_size=1024;
sl@0:     PRAGMA default_cache_size=10;
sl@0:   }
sl@0:   execsql {
sl@0:     PRAGMA temp_store = memory;
sl@0:     BEGIN;
sl@0:     CREATE TABLE abc(a PRIMARY KEY, b, c);
sl@0:     INSERT INTO abc VALUES(randstr(50,50), randstr(75,75), randstr(100,100));
sl@0:     INSERT INTO abc 
sl@0:         SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;
sl@0:     INSERT INTO abc 
sl@0:         SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;
sl@0:     INSERT INTO abc 
sl@0:         SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;
sl@0:     INSERT INTO abc 
sl@0:         SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;
sl@0:     INSERT INTO abc 
sl@0:         SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;
sl@0:     INSERT INTO abc 
sl@0:         SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;
sl@0:     COMMIT;
sl@0:   } 
sl@0:   copy_file test.db test2.db
sl@0:   sqlite3 db2 test2.db
sl@0:   list \
sl@0:     [expr ([file size test.db]/1024)>20] [expr ([file size test2.db]/1024)>20]
sl@0: } {1 1}
sl@0: do_test malloc5-6.1.2 {
sl@0:   list [execsql {PRAGMA cache_size}] [execsql {PRAGMA cache_size} db2]
sl@0: } {10 10}
sl@0: 
sl@0: do_test malloc5-6.2.1 {
sl@0: breakpoint
sl@0:   execsql {SELECT * FROM abc} db2
sl@0:   execsql {SELECT * FROM abc} db
sl@0:   expr [nPage db] + [nPage db2]
sl@0: } {20}
sl@0: 
sl@0: do_test malloc5-6.2.2 {
sl@0:   # If we now try to reclaim some memory, it should come from the db2 cache.
sl@0:   sqlite3_release_memory 3000
sl@0:   expr [nPage db] + [nPage db2]
sl@0: } {17}
sl@0: do_test malloc5-6.2.3 {
sl@0:   # Access the db2 cache again, so that all the db2 pages have been used
sl@0:   # more recently than all the db pages. Then try to reclaim 3000 bytes.
sl@0:   # This time, 3 pages should be pulled from the db cache.
sl@0:   execsql { SELECT * FROM abc } db2
sl@0:   sqlite3_release_memory 3000
sl@0:   expr [nPage db] + [nPage db2]
sl@0: } {17}
sl@0: 
sl@0: do_test malloc5-6.3.1 {
sl@0:   # Now open a transaction and update 2 pages in the db2 cache. Then
sl@0:   # do a SELECT on the db cache so that all the db pages are more recently
sl@0:   # used than the db2 pages. When we try to free memory, SQLite should
sl@0:   # free the non-dirty db2 pages, then the db pages, then finally use
sl@0:   # sync() to free up the dirty db2 pages. The only page that cannot be
sl@0:   # freed is page1 of db2. Because there is an open transaction, the
sl@0:   # btree layer holds a reference to page 1 in the db2 cache.
sl@0:   execsql {
sl@0:     BEGIN;
sl@0:     UPDATE abc SET c = randstr(100,100) 
sl@0:     WHERE rowid = 1 OR rowid = (SELECT max(rowid) FROM abc);
sl@0:   } db2
sl@0:   execsql { SELECT * FROM abc } db
sl@0:   expr [nPage db] + [nPage db2]
sl@0: } {20}
sl@0: do_test malloc5-6.3.2 {
sl@0:   # Try to release 7700 bytes. This should release all the 
sl@0:   # non-dirty pages held by db2.
sl@0:   sqlite3_release_memory [expr 7*1100]
sl@0:   list [nPage db] [nPage db2]
sl@0: } {10 3}
sl@0: do_test malloc5-6.3.3 {
sl@0:   # Try to release another 1000 bytes. This should come fromt the db
sl@0:   # cache, since all three pages held by db2 are either in-use or diry.
sl@0:   sqlite3_release_memory 1000
sl@0:   list [nPage db] [nPage db2]
sl@0: } {9 3}
sl@0: do_test malloc5-6.3.4 {
sl@0:   # Now release 9900 more (about 9 pages worth). This should expunge
sl@0:   # the rest of the db cache. But the db2 cache remains intact, because
sl@0:   # SQLite tries to avoid calling sync().
sl@0:   sqlite3_release_memory 9900
sl@0:   list [nPage db] [nPage db2]
sl@0: } {0 3}
sl@0: do_test malloc5-6.3.5 {
sl@0:   # But if we are really insistent, SQLite will consent to call sync()
sl@0:   # if there is no other option. UPDATE: As of 3.6.2, SQLite will not
sl@0:   # call sync() in this scenario. So no further memory can be reclaimed.
sl@0:   sqlite3_release_memory 1000
sl@0:   list [nPage db] [nPage db2]
sl@0: } {0 3}
sl@0: do_test malloc5-6.3.6 {
sl@0:   # The referenced page (page 1 of the db2 cache) will not be freed no
sl@0:   # matter how much memory we ask for:
sl@0:   sqlite3_release_memory 31459
sl@0:   list [nPage db] [nPage db2]
sl@0: } {0 3}
sl@0: 
sl@0: db2 close
sl@0: 
sl@0: sqlite3_soft_heap_limit $::soft_limit
sl@0: finish_test
sl@0: catch {db close}