Symaptic: os/persistentdata/persistentstorage/sqlite3api/SQLite/fts2.c@bde4ae8d615e (annotated)

sl@0	1	/* fts2 has a design flaw which can lead to database corruption (see
sl@0	2	** below). It is recommended not to use it any longer, instead use
sl@0	3	** fts3 (or higher). If you believe that your use of fts2 is safe,
sl@0	4	** add -DSQLITE_ENABLE_BROKEN_FTS2=1 to your CFLAGS.
sl@0	5	*/
sl@0	6	#if (!defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS2)) \
sl@0	7	&& !defined(SQLITE_ENABLE_BROKEN_FTS2)
sl@0	8	#error fts2 has a design flaw and has been deprecated.
sl@0	9	#endif
sl@0	10	/* The flaw is that fts2 uses the content table's unaliased rowid as
sl@0	11	** the unique docid. fts2 embeds the rowid in the index it builds,
sl@0	12	** and expects the rowid to not change. The SQLite VACUUM operation
sl@0	13	** will renumber such rowids, thereby breaking fts2. If you are using
sl@0	14	** fts2 in a system which has disabled VACUUM, then you can continue
sl@0	15	** to use it safely. Note that PRAGMA auto_vacuum does NOT disable
sl@0	16	** VACUUM, though systems using auto_vacuum are unlikely to invoke
sl@0	17	** VACUUM.
sl@0	18	**
sl@0	19	** Unlike fts1, which is safe across VACUUM if you never delete
sl@0	20	** documents, fts2 has a second exposure to this flaw, in the segments
sl@0	21	** table. So fts2 should be considered unsafe across VACUUM in all
sl@0	22	** cases.
sl@0	23	*/
sl@0	24
sl@0	25	/*
sl@0	26	** 2006 Oct 10
sl@0	27	**
sl@0	28	** The author disclaims copyright to this source code. In place of
sl@0	29	** a legal notice, here is a blessing:
sl@0	30	**
sl@0	31	** May you do good and not evil.
sl@0	32	** May you find forgiveness for yourself and forgive others.
sl@0	33	** May you share freely, never taking more than you give.
sl@0	34	**
sl@0	35	******************************************************************************
sl@0	36	**
sl@0	37	** This is an SQLite module implementing full-text search.
sl@0	38	*/
sl@0	39
sl@0	40	/*
sl@0	41	** The code in this file is only compiled if:
sl@0	42	**
sl@0	43	** * The FTS2 module is being built as an extension
sl@0	44	** (in which case SQLITE_CORE is not defined), or
sl@0	45	**
sl@0	46	** * The FTS2 module is being built into the core of
sl@0	47	** SQLite (in which case SQLITE_ENABLE_FTS2 is defined).
sl@0	48	*/
sl@0	49
sl@0	50	/* TODO(shess) Consider exporting this comment to an HTML file or the
sl@0	51	** wiki.
sl@0	52	*/
sl@0	53	/* The full-text index is stored in a series of b+tree (-like)
sl@0	54	** structures called segments which map terms to doclists. The
sl@0	55	** structures are like b+trees in layout, but are constructed from the
sl@0	56	** bottom up in optimal fashion and are not updatable. Since trees
sl@0	57	** are built from the bottom up, things will be described from the
sl@0	58	** bottom up.
sl@0	59	**
sl@0	60	**
sl@0	61	** Varints **
sl@0	62	** The basic unit of encoding is a variable-length integer called a
sl@0	63	** varint. We encode variable-length integers in little-endian order
sl@0	64	** using seven bits * per byte as follows:
sl@0	65	**
sl@0	66	** KEY:
sl@0	67	** A = 0xxxxxxx 7 bits of data and one flag bit
sl@0	68	** B = 1xxxxxxx 7 bits of data and one flag bit
sl@0	69	**
sl@0	70	** 7 bits - A
sl@0	71	** 14 bits - BA
sl@0	72	** 21 bits - BBA
sl@0	73	** and so on.
sl@0	74	**
sl@0	75	** This is identical to how sqlite encodes varints (see util.c).
sl@0	76	**
sl@0	77	**
sl@0	78	** Document lists **
sl@0	79	** A doclist (document list) holds a docid-sorted list of hits for a
sl@0	80	** given term. Doclists hold docids, and can optionally associate
sl@0	81	** token positions and offsets with docids.
sl@0	82	**
sl@0	83	** A DL_POSITIONS_OFFSETS doclist is stored like this:
sl@0	84	**
sl@0	85	** array {
sl@0	86	** varint docid;
sl@0	87	** array { (position list for column 0)
sl@0	88	** varint position; (delta from previous position plus POS_BASE)
sl@0	89	** varint startOffset; (delta from previous startOffset)
sl@0	90	** varint endOffset; (delta from startOffset)
sl@0	91	** }
sl@0	92	** array {
sl@0	93	** varint POS_COLUMN; (marks start of position list for new column)
sl@0	94	** varint column; (index of new column)
sl@0	95	** array {
sl@0	96	** varint position; (delta from previous position plus POS_BASE)
sl@0	97	** varint startOffset;(delta from previous startOffset)
sl@0	98	** varint endOffset; (delta from startOffset)
sl@0	99	** }
sl@0	100	** }
sl@0	101	** varint POS_END; (marks end of positions for this document.
sl@0	102	** }
sl@0	103	**
sl@0	104	** Here, array { X } means zero or more occurrences of X, adjacent in
sl@0	105	** memory. A "position" is an index of a token in the token stream
sl@0	106	** generated by the tokenizer, while an "offset" is a byte offset,
sl@0	107	** both based at 0. Note that POS_END and POS_COLUMN occur in the
sl@0	108	** same logical place as the position element, and act as sentinals
sl@0	109	** ending a position list array.
sl@0	110	**
sl@0	111	** A DL_POSITIONS doclist omits the startOffset and endOffset
sl@0	112	** information. A DL_DOCIDS doclist omits both the position and
sl@0	113	** offset information, becoming an array of varint-encoded docids.
sl@0	114	**
sl@0	115	** On-disk data is stored as type DL_DEFAULT, so we don't serialize
sl@0	116	** the type. Due to how deletion is implemented in the segmentation
sl@0	117	** system, on-disk doclists MUST store at least positions.
sl@0	118	**
sl@0	119	**
sl@0	120	** Segment leaf nodes **
sl@0	121	** Segment leaf nodes store terms and doclists, ordered by term. Leaf
sl@0	122	** nodes are written using LeafWriter, and read using LeafReader (to
sl@0	123	** iterate through a single leaf node's data) and LeavesReader (to
sl@0	124	** iterate through a segment's entire leaf layer). Leaf nodes have
sl@0	125	** the format:
sl@0	126	**
sl@0	127	** varint iHeight; (height from leaf level, always 0)
sl@0	128	** varint nTerm; (length of first term)
sl@0	129	** char pTerm[nTerm]; (content of first term)
sl@0	130	** varint nDoclist; (length of term's associated doclist)
sl@0	131	** char pDoclist[nDoclist]; (content of doclist)
sl@0	132	** array {
sl@0	133	** (further terms are delta-encoded)
sl@0	134	** varint nPrefix; (length of prefix shared with previous term)
sl@0	135	** varint nSuffix; (length of unshared suffix)
sl@0	136	** char pTermSuffix[nSuffix];(unshared suffix of next term)
sl@0	137	** varint nDoclist; (length of term's associated doclist)
sl@0	138	** char pDoclist[nDoclist]; (content of doclist)
sl@0	139	** }
sl@0	140	**
sl@0	141	** Here, array { X } means zero or more occurrences of X, adjacent in
sl@0	142	** memory.
sl@0	143	**
sl@0	144	** Leaf nodes are broken into blocks which are stored contiguously in
sl@0	145	** the %_segments table in sorted order. This means that when the end
sl@0	146	** of a node is reached, the next term is in the node with the next
sl@0	147	** greater node id.
sl@0	148	**
sl@0	149	** New data is spilled to a new leaf node when the current node
sl@0	150	** exceeds LEAF_MAX bytes (default 2048). New data which itself is
sl@0	151	** larger than STANDALONE_MIN (default 1024) is placed in a standalone
sl@0	152	** node (a leaf node with a single term and doclist). The goal of
sl@0	153	** these settings is to pack together groups of small doclists while
sl@0	154	** making it efficient to directly access large doclists. The
sl@0	155	** assumption is that large doclists represent terms which are more
sl@0	156	** likely to be query targets.
sl@0	157	**
sl@0	158	** TODO(shess) It may be useful for blocking decisions to be more
sl@0	159	** dynamic. For instance, it may make more sense to have a 2.5k leaf
sl@0	160	** node rather than splitting into 2k and .5k nodes. My intuition is
sl@0	161	** that this might extend through 2x or 4x the pagesize.
sl@0	162	**
sl@0	163	**
sl@0	164	** Segment interior nodes **
sl@0	165	** Segment interior nodes store blockids for subtree nodes and terms
sl@0	166	** to describe what data is stored by the each subtree. Interior
sl@0	167	** nodes are written using InteriorWriter, and read using
sl@0	168	** InteriorReader. InteriorWriters are created as needed when
sl@0	169	** SegmentWriter creates new leaf nodes, or when an interior node
sl@0	170	** itself grows too big and must be split. The format of interior
sl@0	171	** nodes:
sl@0	172	**
sl@0	173	** varint iHeight; (height from leaf level, always >0)
sl@0	174	** varint iBlockid; (block id of node's leftmost subtree)
sl@0	175	** optional {
sl@0	176	** varint nTerm; (length of first term)
sl@0	177	** char pTerm[nTerm]; (content of first term)
sl@0	178	** array {
sl@0	179	** (further terms are delta-encoded)
sl@0	180	** varint nPrefix; (length of shared prefix with previous term)
sl@0	181	** varint nSuffix; (length of unshared suffix)
sl@0	182	** char pTermSuffix[nSuffix]; (unshared suffix of next term)
sl@0	183	** }
sl@0	184	** }
sl@0	185	**
sl@0	186	** Here, optional { X } means an optional element, while array { X }
sl@0	187	** means zero or more occurrences of X, adjacent in memory.
sl@0	188	**
sl@0	189	** An interior node encodes n terms separating n+1 subtrees. The
sl@0	190	** subtree blocks are contiguous, so only the first subtree's blockid
sl@0	191	** is encoded. The subtree at iBlockid will contain all terms less
sl@0	192	** than the first term encoded (or all terms if no term is encoded).
sl@0	193	** Otherwise, for terms greater than or equal to pTerm[i] but less
sl@0	194	** than pTerm[i+1], the subtree for that term will be rooted at
sl@0	195	** iBlockid+i. Interior nodes only store enough term data to
sl@0	196	** distinguish adjacent children (if the rightmost term of the left
sl@0	197	** child is "something", and the leftmost term of the right child is
sl@0	198	** "wicked", only "w" is stored).
sl@0	199	**
sl@0	200	** New data is spilled to a new interior node at the same height when
sl@0	201	** the current node exceeds INTERIOR_MAX bytes (default 2048).
sl@0	202	** INTERIOR_MIN_TERMS (default 7) keeps large terms from monopolizing
sl@0	203	** interior nodes and making the tree too skinny. The interior nodes
sl@0	204	** at a given height are naturally tracked by interior nodes at
sl@0	205	** height+1, and so on.
sl@0	206	**
sl@0	207	**
sl@0	208	** Segment directory **
sl@0	209	** The segment directory in table %_segdir stores meta-information for
sl@0	210	** merging and deleting segments, and also the root node of the
sl@0	211	** segment's tree.
sl@0	212	**
sl@0	213	** The root node is the top node of the segment's tree after encoding
sl@0	214	** the entire segment, restricted to ROOT_MAX bytes (default 1024).
sl@0	215	** This could be either a leaf node or an interior node. If the top
sl@0	216	** node requires more than ROOT_MAX bytes, it is flushed to %_segments
sl@0	217	** and a new root interior node is generated (which should always fit
sl@0	218	** within ROOT_MAX because it only needs space for 2 varints, the
sl@0	219	** height and the blockid of the previous root).
sl@0	220	**
sl@0	221	** The meta-information in the segment directory is:
sl@0	222	** level - segment level (see below)
sl@0	223	** idx - index within level
sl@0	224	** - (level,idx uniquely identify a segment)
sl@0	225	** start_block - first leaf node
sl@0	226	** leaves_end_block - last leaf node
sl@0	227	** end_block - last block (including interior nodes)
sl@0	228	** root - contents of root node
sl@0	229	**
sl@0	230	** If the root node is a leaf node, then start_block,
sl@0	231	** leaves_end_block, and end_block are all 0.
sl@0	232	**
sl@0	233	**
sl@0	234	** Segment merging **
sl@0	235	** To amortize update costs, segments are groups into levels and
sl@0	236	** merged in matches. Each increase in level represents exponentially
sl@0	237	** more documents.
sl@0	238	**
sl@0	239	** New documents (actually, document updates) are tokenized and
sl@0	240	** written individually (using LeafWriter) to a level 0 segment, with
sl@0	241	** incrementing idx. When idx reaches MERGE_COUNT (default 16), all
sl@0	242	** level 0 segments are merged into a single level 1 segment. Level 1
sl@0	243	** is populated like level 0, and eventually MERGE_COUNT level 1
sl@0	244	** segments are merged to a single level 2 segment (representing
sl@0	245	** MERGE_COUNT^2 updates), and so on.
sl@0	246	**
sl@0	247	** A segment merge traverses all segments at a given level in
sl@0	248	** parallel, performing a straightforward sorted merge. Since segment
sl@0	249	** leaf nodes are written in to the %_segments table in order, this
sl@0	250	** merge traverses the underlying sqlite disk structures efficiently.
sl@0	251	** After the merge, all segment blocks from the merged level are
sl@0	252	** deleted.
sl@0	253	**
sl@0	254	** MERGE_COUNT controls how often we merge segments. 16 seems to be
sl@0	255	** somewhat of a sweet spot for insertion performance. 32 and 64 show
sl@0	256	** very similar performance numbers to 16 on insertion, though they're
sl@0	257	** a tiny bit slower (perhaps due to more overhead in merge-time
sl@0	258	** sorting). 8 is about 20% slower than 16, 4 about 50% slower than
sl@0	259	** 16, 2 about 66% slower than 16.
sl@0	260	**
sl@0	261	** At query time, high MERGE_COUNT increases the number of segments
sl@0	262	** which need to be scanned and merged. For instance, with 100k docs
sl@0	263	** inserted:
sl@0	264	**
sl@0	265	** MERGE_COUNT segments
sl@0	266	** 16 25
sl@0	267	** 8 12
sl@0	268	** 4 10
sl@0	269	** 2 6
sl@0	270	**
sl@0	271	** This appears to have only a moderate impact on queries for very
sl@0	272	** frequent terms (which are somewhat dominated by segment merge
sl@0	273	** costs), and infrequent and non-existent terms still seem to be fast
sl@0	274	** even with many segments.
sl@0	275	**
sl@0	276	** TODO(shess) That said, it would be nice to have a better query-side
sl@0	277	** argument for MERGE_COUNT of 16. Also, it is possible/likely that
sl@0	278	** optimizations to things like doclist merging will swing the sweet
sl@0	279	** spot around.
sl@0	280	**
sl@0	281	**
sl@0	282	**
sl@0	283	** Handling of deletions and updates **
sl@0	284	** Since we're using a segmented structure, with no docid-oriented
sl@0	285	** index into the term index, we clearly cannot simply update the term
sl@0	286	** index when a document is deleted or updated. For deletions, we
sl@0	287	** write an empty doclist (varint(docid) varint(POS_END)), for updates
sl@0	288	** we simply write the new doclist. Segment merges overwrite older
sl@0	289	** data for a particular docid with newer data, so deletes or updates
sl@0	290	** will eventually overtake the earlier data and knock it out. The
sl@0	291	** query logic likewise merges doclists so that newer data knocks out
sl@0	292	** older data.
sl@0	293	**
sl@0	294	** TODO(shess) Provide a VACUUM type operation to clear out all
sl@0	295	** deletions and duplications. This would basically be a forced merge
sl@0	296	** into a single segment.
sl@0	297	*/
sl@0	298
sl@0	299	#if !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS2)
sl@0	300
sl@0	301	#if defined(SQLITE_ENABLE_FTS2) && !defined(SQLITE_CORE)
sl@0	302	# define SQLITE_CORE 1
sl@0	303	#endif
sl@0	304
sl@0	305	#include <assert.h>
sl@0	306	#include <stdlib.h>
sl@0	307	#include <stdio.h>
sl@0	308	#include <string.h>
sl@0	309	#include <ctype.h>
sl@0	310
sl@0	311	#include "fts2.h"
sl@0	312	#include "fts2_hash.h"
sl@0	313	#include "fts2_tokenizer.h"
sl@0	314	#include "sqlite3.h"
sl@0	315	#include "sqlite3ext.h"
sl@0	316	SQLITE_EXTENSION_INIT1
sl@0	317
sl@0	318
sl@0	319	/* TODO(shess) MAN, this thing needs some refactoring. At minimum, it
sl@0	320	** would be nice to order the file better, perhaps something along the
sl@0	321	** lines of:
sl@0	322	**
sl@0	323	** - utility functions
sl@0	324	** - table setup functions
sl@0	325	** - table update functions
sl@0	326	** - table query functions
sl@0	327	**
sl@0	328	** Put the query functions last because they're likely to reference
sl@0	329	** typedefs or functions from the table update section.
sl@0	330	*/
sl@0	331
sl@0	332	#if 0
sl@0	333	# define TRACE(A) printf A; fflush(stdout)
sl@0	334	#else
sl@0	335	# define TRACE(A)
sl@0	336	#endif
sl@0	337
sl@0	338	/* It is not safe to call isspace(), tolower(), or isalnum() on
sl@0	339	** hi-bit-set characters. This is the same solution used in the
sl@0	340	** tokenizer.
sl@0	341	*/
sl@0	342	/* TODO(shess) The snippet-generation code should be using the
sl@0	343	** tokenizer-generated tokens rather than doing its own local
sl@0	344	** tokenization.
sl@0	345	*/
sl@0	346	/* TODO(shess) Is __isascii() a portable version of (c&0x80)==0? */
sl@0	347	static int safe_isspace(char c){
sl@0	348	return (c&0x80)==0 ? isspace(c) : 0;
sl@0	349	}
sl@0	350	static int safe_tolower(char c){
sl@0	351	return (c&0x80)==0 ? tolower(c) : c;
sl@0	352	}
sl@0	353	static int safe_isalnum(char c){
sl@0	354	return (c&0x80)==0 ? isalnum(c) : 0;
sl@0	355	}
sl@0	356
sl@0	357	typedef enum DocListType {
sl@0	358	DL_DOCIDS, /* docids only */
sl@0	359	DL_POSITIONS, /* docids + positions */
sl@0	360	DL_POSITIONS_OFFSETS /* docids + positions + offsets */
sl@0	361	} DocListType;
sl@0	362
sl@0	363	/*
sl@0	364	** By default, only positions and not offsets are stored in the doclists.
sl@0	365	** To change this so that offsets are stored too, compile with
sl@0	366	**
sl@0	367	** -DDL_DEFAULT=DL_POSITIONS_OFFSETS
sl@0	368	**
sl@0	369	** If DL_DEFAULT is set to DL_DOCIDS, your table can only be inserted
sl@0	370	** into (no deletes or updates).
sl@0	371	*/
sl@0	372	#ifndef DL_DEFAULT
sl@0	373	# define DL_DEFAULT DL_POSITIONS
sl@0	374	#endif
sl@0	375
sl@0	376	enum {
sl@0	377	POS_END = 0, /* end of this position list */
sl@0	378	POS_COLUMN, /* followed by new column number */
sl@0	379	POS_BASE
sl@0	380	};
sl@0	381
sl@0	382	/* MERGE_COUNT controls how often we merge segments (see comment at
sl@0	383	** top of file).
sl@0	384	*/
sl@0	385	#define MERGE_COUNT 16
sl@0	386
sl@0	387	/* utility functions */
sl@0	388
sl@0	389	/* CLEAR() and SCRAMBLE() abstract memset() on a pointer to a single
sl@0	390	** record to prevent errors of the form:
sl@0	391	**
sl@0	392	** my_function(SomeType *b){
sl@0	393	** memset(b, '\0', sizeof(b)); // sizeof(b)!=sizeof(*b)
sl@0	394	** }
sl@0	395	*/
sl@0	396	/* TODO(shess) Obvious candidates for a header file. */
sl@0	397	#define CLEAR(b) memset(b, '\0', sizeof(*(b)))
sl@0	398
sl@0	399	#ifndef NDEBUG
sl@0	400	# define SCRAMBLE(b) memset(b, 0x55, sizeof(*(b)))
sl@0	401	#else
sl@0	402	# define SCRAMBLE(b)
sl@0	403	#endif
sl@0	404
sl@0	405	/* We may need up to VARINT_MAX bytes to store an encoded 64-bit integer. */
sl@0	406	#define VARINT_MAX 10
sl@0	407
sl@0	408	/* Write a 64-bit variable-length integer to memory starting at p[0].
sl@0	409	* The length of data written will be between 1 and VARINT_MAX bytes.
sl@0	410	* The number of bytes written is returned. */
sl@0	411	static int putVarint(char *p, sqlite_int64 v){
sl@0	412	unsigned char q = (unsigned char ) p;
sl@0	413	sqlite_uint64 vu = v;
sl@0	414	do{
sl@0	415	*q++ = (unsigned char) ((vu & 0x7f) \| 0x80);
sl@0	416	vu >>= 7;
sl@0	417	}while( vu!=0 );
sl@0	418	q[-1] &= 0x7f; /* turn off high bit in final byte */
sl@0	419	assert( q - (unsigned char *)p <= VARINT_MAX );
sl@0	420	return (int) (q - (unsigned char *)p);
sl@0	421	}
sl@0	422
sl@0	423	/* Read a 64-bit variable-length integer from memory starting at p[0].
sl@0	424	* Return the number of bytes read, or 0 on error.
sl@0	425	* The value is stored in v. /
sl@0	426	static int getVarint(const char p, sqlite_int64 v){
sl@0	427	const unsigned char q = (const unsigned char ) p;
sl@0	428	sqlite_uint64 x = 0, y = 1;
sl@0	429	while( (*q & 0x80) == 0x80 ){
sl@0	430	x += y * (*q++ & 0x7f);
sl@0	431	y <<= 7;
sl@0	432	if( q - (unsigned char )p >= VARINT_MAX ){ / bad data */
sl@0	433	assert( 0 );
sl@0	434	return 0;
sl@0	435	}
sl@0	436	}
sl@0	437	x += y * (*q++);
sl@0	438	*v = (sqlite_int64) x;
sl@0	439	return (int) (q - (unsigned char *)p);
sl@0	440	}
sl@0	441
sl@0	442	static int getVarint32(const char p, int pi){
sl@0	443	sqlite_int64 i;
sl@0	444	int ret = getVarint(p, &i);
sl@0	445	*pi = (int) i;
sl@0	446	assert( *pi==i );
sl@0	447	return ret;
sl@0	448	}
sl@0	449
sl@0	450	/*******************************************************************/
sl@0	451	/* DataBuffer is used to collect data into a buffer in piecemeal
sl@0	452	** fashion. It implements the usual distinction between amount of
sl@0	453	** data currently stored (nData) and buffer capacity (nCapacity).
sl@0	454	**
sl@0	455	** dataBufferInit - create a buffer with given initial capacity.
sl@0	456	** dataBufferReset - forget buffer's data, retaining capacity.
sl@0	457	** dataBufferDestroy - free buffer's data.
sl@0	458	** dataBufferSwap - swap contents of two buffers.
sl@0	459	** dataBufferExpand - expand capacity without adding data.
sl@0	460	** dataBufferAppend - append data.
sl@0	461	** dataBufferAppend2 - append two pieces of data at once.
sl@0	462	** dataBufferReplace - replace buffer's data.
sl@0	463	*/
sl@0	464	typedef struct DataBuffer {
sl@0	465	char pData; / Pointer to malloc'ed buffer. */
sl@0	466	int nCapacity; /* Size of pData buffer. */
sl@0	467	int nData; /* End of data loaded into pData. */
sl@0	468	} DataBuffer;
sl@0	469
sl@0	470	static void dataBufferInit(DataBuffer *pBuffer, int nCapacity){
sl@0	471	assert( nCapacity>=0 );
sl@0	472	pBuffer->nData = 0;
sl@0	473	pBuffer->nCapacity = nCapacity;
sl@0	474	pBuffer->pData = nCapacity==0 ? NULL : sqlite3_malloc(nCapacity);
sl@0	475	}
sl@0	476	static void dataBufferReset(DataBuffer *pBuffer){
sl@0	477	pBuffer->nData = 0;
sl@0	478	}
sl@0	479	static void dataBufferDestroy(DataBuffer *pBuffer){
sl@0	480	if( pBuffer->pData!=NULL ) sqlite3_free(pBuffer->pData);
sl@0	481	SCRAMBLE(pBuffer);
sl@0	482	}
sl@0	483	static void dataBufferSwap(DataBuffer pBuffer1, DataBuffer pBuffer2){
sl@0	484	DataBuffer tmp = *pBuffer1;
sl@0	485	pBuffer1 = pBuffer2;
sl@0	486	*pBuffer2 = tmp;
sl@0	487	}
sl@0	488	static void dataBufferExpand(DataBuffer *pBuffer, int nAddCapacity){
sl@0	489	assert( nAddCapacity>0 );
sl@0	490	/* TODO(shess) Consider expanding more aggressively. Note that the
sl@0	491	** underlying malloc implementation may take care of such things for
sl@0	492	** us already.
sl@0	493	*/
sl@0	494	if( pBuffer->nData+nAddCapacity>pBuffer->nCapacity ){
sl@0	495	pBuffer->nCapacity = pBuffer->nData+nAddCapacity;
sl@0	496	pBuffer->pData = sqlite3_realloc(pBuffer->pData, pBuffer->nCapacity);
sl@0	497	}
sl@0	498	}
sl@0	499	static void dataBufferAppend(DataBuffer *pBuffer,
sl@0	500	const char *pSource, int nSource){
sl@0	501	assert( nSource>0 && pSource!=NULL );
sl@0	502	dataBufferExpand(pBuffer, nSource);
sl@0	503	memcpy(pBuffer->pData+pBuffer->nData, pSource, nSource);
sl@0	504	pBuffer->nData += nSource;
sl@0	505	}
sl@0	506	static void dataBufferAppend2(DataBuffer *pBuffer,
sl@0	507	const char *pSource1, int nSource1,
sl@0	508	const char *pSource2, int nSource2){
sl@0	509	assert( nSource1>0 && pSource1!=NULL );
sl@0	510	assert( nSource2>0 && pSource2!=NULL );
sl@0	511	dataBufferExpand(pBuffer, nSource1+nSource2);
sl@0	512	memcpy(pBuffer->pData+pBuffer->nData, pSource1, nSource1);
sl@0	513	memcpy(pBuffer->pData+pBuffer->nData+nSource1, pSource2, nSource2);
sl@0	514	pBuffer->nData += nSource1+nSource2;
sl@0	515	}
sl@0	516	static void dataBufferReplace(DataBuffer *pBuffer,
sl@0	517	const char *pSource, int nSource){
sl@0	518	dataBufferReset(pBuffer);
sl@0	519	dataBufferAppend(pBuffer, pSource, nSource);
sl@0	520	}
sl@0	521
sl@0	522	/* StringBuffer is a null-terminated version of DataBuffer. */
sl@0	523	typedef struct StringBuffer {
sl@0	524	DataBuffer b; /* Includes null terminator. */
sl@0	525	} StringBuffer;
sl@0	526
sl@0	527	static void initStringBuffer(StringBuffer *sb){
sl@0	528	dataBufferInit(&sb->b, 100);
sl@0	529	dataBufferReplace(&sb->b, "", 1);
sl@0	530	}
sl@0	531	static int stringBufferLength(StringBuffer *sb){
sl@0	532	return sb->b.nData-1;
sl@0	533	}
sl@0	534	static char stringBufferData(StringBuffer sb){
sl@0	535	return sb->b.pData;
sl@0	536	}
sl@0	537	static void stringBufferDestroy(StringBuffer *sb){
sl@0	538	dataBufferDestroy(&sb->b);
sl@0	539	}
sl@0	540
sl@0	541	static void nappend(StringBuffer sb, const char zFrom, int nFrom){
sl@0	542	assert( sb->b.nData>0 );
sl@0	543	if( nFrom>0 ){
sl@0	544	sb->b.nData--;
sl@0	545	dataBufferAppend2(&sb->b, zFrom, nFrom, "", 1);
sl@0	546	}
sl@0	547	}
sl@0	548	static void append(StringBuffer sb, const char zFrom){
sl@0	549	nappend(sb, zFrom, strlen(zFrom));
sl@0	550	}
sl@0	551
sl@0	552	/* Append a list of strings separated by commas. */
sl@0	553	static void appendList(StringBuffer sb, int nString, char *azString){
sl@0	554	int i;
sl@0	555	for(i=0; i<nString; ++i){
sl@0	556	if( i>0 ) append(sb, ", ");
sl@0	557	append(sb, azString[i]);
sl@0	558	}
sl@0	559	}
sl@0	560
sl@0	561	static int endsInWhiteSpace(StringBuffer *p){
sl@0	562	return stringBufferLength(p)>0 &&
sl@0	563	safe_isspace(stringBufferData(p)[stringBufferLength(p)-1]);
sl@0	564	}
sl@0	565
sl@0	566	/* If the StringBuffer ends in something other than white space, add a
sl@0	567	** single space character to the end.
sl@0	568	*/
sl@0	569	static void appendWhiteSpace(StringBuffer *p){
sl@0	570	if( stringBufferLength(p)==0 ) return;
sl@0	571	if( !endsInWhiteSpace(p) ) append(p, " ");
sl@0	572	}
sl@0	573
sl@0	574	/* Remove white space from the end of the StringBuffer */
sl@0	575	static void trimWhiteSpace(StringBuffer *p){
sl@0	576	while( endsInWhiteSpace(p) ){
sl@0	577	p->b.pData[--p->b.nData-1] = '\0';
sl@0	578	}
sl@0	579	}
sl@0	580
sl@0	581	/*******************************************************************/
sl@0	582	/* DLReader is used to read document elements from a doclist. The
sl@0	583	** current docid is cached, so dlrDocid() is fast. DLReader does not
sl@0	584	** own the doclist buffer.
sl@0	585	**
sl@0	586	** dlrAtEnd - true if there's no more data to read.
sl@0	587	** dlrDocid - docid of current document.
sl@0	588	** dlrDocData - doclist data for current document (including docid).
sl@0	589	** dlrDocDataBytes - length of same.
sl@0	590	** dlrAllDataBytes - length of all remaining data.
sl@0	591	** dlrPosData - position data for current document.
sl@0	592	** dlrPosDataLen - length of pos data for current document (incl POS_END).
sl@0	593	** dlrStep - step to current document.
sl@0	594	** dlrInit - initial for doclist of given type against given data.
sl@0	595	** dlrDestroy - clean up.
sl@0	596	**
sl@0	597	** Expected usage is something like:
sl@0	598	**
sl@0	599	** DLReader reader;
sl@0	600	** dlrInit(&reader, pData, nData);
sl@0	601	** while( !dlrAtEnd(&reader) ){
sl@0	602	** // calls to dlrDocid() and kin.
sl@0	603	** dlrStep(&reader);
sl@0	604	** }
sl@0	605	** dlrDestroy(&reader);
sl@0	606	*/
sl@0	607	typedef struct DLReader {
sl@0	608	DocListType iType;
sl@0	609	const char *pData;
sl@0	610	int nData;
sl@0	611
sl@0	612	sqlite_int64 iDocid;
sl@0	613	int nElement;
sl@0	614	} DLReader;
sl@0	615
sl@0	616	static int dlrAtEnd(DLReader *pReader){
sl@0	617	assert( pReader->nData>=0 );
sl@0	618	return pReader->nData==0;
sl@0	619	}
sl@0	620	static sqlite_int64 dlrDocid(DLReader *pReader){
sl@0	621	assert( !dlrAtEnd(pReader) );
sl@0	622	return pReader->iDocid;
sl@0	623	}
sl@0	624	static const char dlrDocData(DLReader pReader){
sl@0	625	assert( !dlrAtEnd(pReader) );
sl@0	626	return pReader->pData;
sl@0	627	}
sl@0	628	static int dlrDocDataBytes(DLReader *pReader){
sl@0	629	assert( !dlrAtEnd(pReader) );
sl@0	630	return pReader->nElement;
sl@0	631	}
sl@0	632	static int dlrAllDataBytes(DLReader *pReader){
sl@0	633	assert( !dlrAtEnd(pReader) );
sl@0	634	return pReader->nData;
sl@0	635	}
sl@0	636	/* TODO(shess) Consider adding a field to track iDocid varint length
sl@0	637	** to make these two functions faster. This might matter (a tiny bit)
sl@0	638	** for queries.
sl@0	639	*/
sl@0	640	static const char dlrPosData(DLReader pReader){
sl@0	641	sqlite_int64 iDummy;
sl@0	642	int n = getVarint(pReader->pData, &iDummy);
sl@0	643	assert( !dlrAtEnd(pReader) );
sl@0	644	return pReader->pData+n;
sl@0	645	}
sl@0	646	static int dlrPosDataLen(DLReader *pReader){
sl@0	647	sqlite_int64 iDummy;
sl@0	648	int n = getVarint(pReader->pData, &iDummy);
sl@0	649	assert( !dlrAtEnd(pReader) );
sl@0	650	return pReader->nElement-n;
sl@0	651	}
sl@0	652	static void dlrStep(DLReader *pReader){
sl@0	653	assert( !dlrAtEnd(pReader) );
sl@0	654
sl@0	655	/* Skip past current doclist element. */
sl@0	656	assert( pReader->nElement<=pReader->nData );
sl@0	657	pReader->pData += pReader->nElement;
sl@0	658	pReader->nData -= pReader->nElement;
sl@0	659
sl@0	660	/* If there is more data, read the next doclist element. */
sl@0	661	if( pReader->nData!=0 ){
sl@0	662	sqlite_int64 iDocidDelta;
sl@0	663	int iDummy, n = getVarint(pReader->pData, &iDocidDelta);
sl@0	664	pReader->iDocid += iDocidDelta;
sl@0	665	if( pReader->iType>=DL_POSITIONS ){
sl@0	666	assert( n<pReader->nData );
sl@0	667	while( 1 ){
sl@0	668	n += getVarint32(pReader->pData+n, &iDummy);
sl@0	669	assert( n<=pReader->nData );
sl@0	670	if( iDummy==POS_END ) break;
sl@0	671	if( iDummy==POS_COLUMN ){
sl@0	672	n += getVarint32(pReader->pData+n, &iDummy);
sl@0	673	assert( n<pReader->nData );
sl@0	674	}else if( pReader->iType==DL_POSITIONS_OFFSETS ){
sl@0	675	n += getVarint32(pReader->pData+n, &iDummy);
sl@0	676	n += getVarint32(pReader->pData+n, &iDummy);
sl@0	677	assert( n<pReader->nData );
sl@0	678	}
sl@0	679	}
sl@0	680	}
sl@0	681	pReader->nElement = n;
sl@0	682	assert( pReader->nElement<=pReader->nData );
sl@0	683	}
sl@0	684	}
sl@0	685	static void dlrInit(DLReader *pReader, DocListType iType,
sl@0	686	const char *pData, int nData){
sl@0	687	assert( pData!=NULL && nData!=0 );
sl@0	688	pReader->iType = iType;
sl@0	689	pReader->pData = pData;
sl@0	690	pReader->nData = nData;
sl@0	691	pReader->nElement = 0;
sl@0	692	pReader->iDocid = 0;
sl@0	693
sl@0	694	/* Load the first element's data. There must be a first element. */
sl@0	695	dlrStep(pReader);
sl@0	696	}
sl@0	697	static void dlrDestroy(DLReader *pReader){
sl@0	698	SCRAMBLE(pReader);
sl@0	699	}
sl@0	700
sl@0	701	#ifndef NDEBUG
sl@0	702	/* Verify that the doclist can be validly decoded. Also returns the
sl@0	703	** last docid found because it is convenient in other assertions for
sl@0	704	** DLWriter.
sl@0	705	*/
sl@0	706	static void docListValidate(DocListType iType, const char *pData, int nData,
sl@0	707	sqlite_int64 *pLastDocid){
sl@0	708	sqlite_int64 iPrevDocid = 0;
sl@0	709	assert( nData>0 );
sl@0	710	assert( pData!=0 );
sl@0	711	assert( pData+nData>pData );
sl@0	712	while( nData!=0 ){
sl@0	713	sqlite_int64 iDocidDelta;
sl@0	714	int n = getVarint(pData, &iDocidDelta);
sl@0	715	iPrevDocid += iDocidDelta;
sl@0	716	if( iType>DL_DOCIDS ){
sl@0	717	int iDummy;
sl@0	718	while( 1 ){
sl@0	719	n += getVarint32(pData+n, &iDummy);
sl@0	720	if( iDummy==POS_END ) break;
sl@0	721	if( iDummy==POS_COLUMN ){
sl@0	722	n += getVarint32(pData+n, &iDummy);
sl@0	723	}else if( iType>DL_POSITIONS ){
sl@0	724	n += getVarint32(pData+n, &iDummy);
sl@0	725	n += getVarint32(pData+n, &iDummy);
sl@0	726	}
sl@0	727	assert( n<=nData );
sl@0	728	}
sl@0	729	}
sl@0	730	assert( n<=nData );
sl@0	731	pData += n;
sl@0	732	nData -= n;
sl@0	733	}
sl@0	734	if( pLastDocid ) *pLastDocid = iPrevDocid;
sl@0	735	}
sl@0	736	#define ASSERT_VALID_DOCLIST(i, p, n, o) docListValidate(i, p, n, o)
sl@0	737	#else
sl@0	738	#define ASSERT_VALID_DOCLIST(i, p, n, o) assert( 1 )
sl@0	739	#endif
sl@0	740
sl@0	741	/*******************************************************************/
sl@0	742	/* DLWriter is used to write doclist data to a DataBuffer. DLWriter
sl@0	743	** always appends to the buffer and does not own it.
sl@0	744	**
sl@0	745	** dlwInit - initialize to write a given type doclistto a buffer.
sl@0	746	** dlwDestroy - clear the writer's memory. Does not free buffer.
sl@0	747	** dlwAppend - append raw doclist data to buffer.
sl@0	748	** dlwCopy - copy next doclist from reader to writer.
sl@0	749	** dlwAdd - construct doclist element and append to buffer.
sl@0	750	** Only apply dlwAdd() to DL_DOCIDS doclists (else use PLWriter).
sl@0	751	*/
sl@0	752	typedef struct DLWriter {
sl@0	753	DocListType iType;
sl@0	754	DataBuffer *b;
sl@0	755	sqlite_int64 iPrevDocid;
sl@0	756	#ifndef NDEBUG
sl@0	757	int has_iPrevDocid;
sl@0	758	#endif
sl@0	759	} DLWriter;
sl@0	760
sl@0	761	static void dlwInit(DLWriter pWriter, DocListType iType, DataBuffer b){
sl@0	762	pWriter->b = b;
sl@0	763	pWriter->iType = iType;
sl@0	764	pWriter->iPrevDocid = 0;
sl@0	765	#ifndef NDEBUG
sl@0	766	pWriter->has_iPrevDocid = 0;
sl@0	767	#endif
sl@0	768	}
sl@0	769	static void dlwDestroy(DLWriter *pWriter){
sl@0	770	SCRAMBLE(pWriter);
sl@0	771	}
sl@0	772	/* iFirstDocid is the first docid in the doclist in pData. It is
sl@0	773	** needed because pData may point within a larger doclist, in which
sl@0	774	** case the first item would be delta-encoded.
sl@0	775	**
sl@0	776	** iLastDocid is the final docid in the doclist in pData. It is
sl@0	777	** needed to create the new iPrevDocid for future delta-encoding. The
sl@0	778	** code could decode the passed doclist to recreate iLastDocid, but
sl@0	779	** the only current user (docListMerge) already has decoded this
sl@0	780	** information.
sl@0	781	*/
sl@0	782	/* TODO(shess) This has become just a helper for docListMerge.
sl@0	783	** Consider a refactor to make this cleaner.
sl@0	784	*/
sl@0	785	static void dlwAppend(DLWriter *pWriter,
sl@0	786	const char *pData, int nData,
sl@0	787	sqlite_int64 iFirstDocid, sqlite_int64 iLastDocid){
sl@0	788	sqlite_int64 iDocid = 0;
sl@0	789	char c[VARINT_MAX];
sl@0	790	int nFirstOld, nFirstNew; /* Old and new varint len of first docid. */
sl@0	791	#ifndef NDEBUG
sl@0	792	sqlite_int64 iLastDocidDelta;
sl@0	793	#endif
sl@0	794
sl@0	795	/* Recode the initial docid as delta from iPrevDocid. */
sl@0	796	nFirstOld = getVarint(pData, &iDocid);
sl@0	797	assert( nFirstOld<nData \|\| (nFirstOld==nData && pWriter->iType==DL_DOCIDS) );
sl@0	798	nFirstNew = putVarint(c, iFirstDocid-pWriter->iPrevDocid);
sl@0	799
sl@0	800	/* Verify that the incoming doclist is valid AND that it ends with
sl@0	801	** the expected docid. This is essential because we'll trust this
sl@0	802	** docid in future delta-encoding.
sl@0	803	*/
sl@0	804	ASSERT_VALID_DOCLIST(pWriter->iType, pData, nData, &iLastDocidDelta);
sl@0	805	assert( iLastDocid==iFirstDocid-iDocid+iLastDocidDelta );
sl@0	806
sl@0	807	/* Append recoded initial docid and everything else. Rest of docids
sl@0	808	** should have been delta-encoded from previous initial docid.
sl@0	809	*/
sl@0	810	if( nFirstOld<nData ){
sl@0	811	dataBufferAppend2(pWriter->b, c, nFirstNew,
sl@0	812	pData+nFirstOld, nData-nFirstOld);
sl@0	813	}else{
sl@0	814	dataBufferAppend(pWriter->b, c, nFirstNew);
sl@0	815	}
sl@0	816	pWriter->iPrevDocid = iLastDocid;
sl@0	817	}
sl@0	818	static void dlwCopy(DLWriter pWriter, DLReader pReader){
sl@0	819	dlwAppend(pWriter, dlrDocData(pReader), dlrDocDataBytes(pReader),
sl@0	820	dlrDocid(pReader), dlrDocid(pReader));
sl@0	821	}
sl@0	822	static void dlwAdd(DLWriter *pWriter, sqlite_int64 iDocid){
sl@0	823	char c[VARINT_MAX];
sl@0	824	int n = putVarint(c, iDocid-pWriter->iPrevDocid);
sl@0	825
sl@0	826	/* Docids must ascend. */
sl@0	827	assert( !pWriter->has_iPrevDocid \|\| iDocid>pWriter->iPrevDocid );
sl@0	828	assert( pWriter->iType==DL_DOCIDS );
sl@0	829
sl@0	830	dataBufferAppend(pWriter->b, c, n);
sl@0	831	pWriter->iPrevDocid = iDocid;
sl@0	832	#ifndef NDEBUG
sl@0	833	pWriter->has_iPrevDocid = 1;
sl@0	834	#endif
sl@0	835	}
sl@0	836
sl@0	837	/*******************************************************************/
sl@0	838	/* PLReader is used to read data from a document's position list. As
sl@0	839	** the caller steps through the list, data is cached so that varints
sl@0	840	** only need to be decoded once.
sl@0	841	**
sl@0	842	** plrInit, plrDestroy - create/destroy a reader.
sl@0	843	** plrColumn, plrPosition, plrStartOffset, plrEndOffset - accessors
sl@0	844	** plrAtEnd - at end of stream, only call plrDestroy once true.
sl@0	845	** plrStep - step to the next element.
sl@0	846	*/
sl@0	847	typedef struct PLReader {
sl@0	848	/* These refer to the next position's data. nData will reach 0 when
sl@0	849	** reading the last position, so plrStep() signals EOF by setting
sl@0	850	** pData to NULL.
sl@0	851	*/
sl@0	852	const char *pData;
sl@0	853	int nData;
sl@0	854
sl@0	855	DocListType iType;
sl@0	856	int iColumn; /* the last column read */
sl@0	857	int iPosition; /* the last position read */
sl@0	858	int iStartOffset; /* the last start offset read */
sl@0	859	int iEndOffset; /* the last end offset read */
sl@0	860	} PLReader;
sl@0	861
sl@0	862	static int plrAtEnd(PLReader *pReader){
sl@0	863	return pReader->pData==NULL;
sl@0	864	}
sl@0	865	static int plrColumn(PLReader *pReader){
sl@0	866	assert( !plrAtEnd(pReader) );
sl@0	867	return pReader->iColumn;
sl@0	868	}
sl@0	869	static int plrPosition(PLReader *pReader){
sl@0	870	assert( !plrAtEnd(pReader) );
sl@0	871	return pReader->iPosition;
sl@0	872	}
sl@0	873	static int plrStartOffset(PLReader *pReader){
sl@0	874	assert( !plrAtEnd(pReader) );
sl@0	875	return pReader->iStartOffset;
sl@0	876	}
sl@0	877	static int plrEndOffset(PLReader *pReader){
sl@0	878	assert( !plrAtEnd(pReader) );
sl@0	879	return pReader->iEndOffset;
sl@0	880	}
sl@0	881	static void plrStep(PLReader *pReader){
sl@0	882	int i, n;
sl@0	883
sl@0	884	assert( !plrAtEnd(pReader) );
sl@0	885
sl@0	886	if( pReader->nData==0 ){
sl@0	887	pReader->pData = NULL;
sl@0	888	return;
sl@0	889	}
sl@0	890
sl@0	891	n = getVarint32(pReader->pData, &i);
sl@0	892	if( i==POS_COLUMN ){
sl@0	893	n += getVarint32(pReader->pData+n, &pReader->iColumn);
sl@0	894	pReader->iPosition = 0;
sl@0	895	pReader->iStartOffset = 0;
sl@0	896	n += getVarint32(pReader->pData+n, &i);
sl@0	897	}
sl@0	898	/* Should never see adjacent column changes. */
sl@0	899	assert( i!=POS_COLUMN );
sl@0	900
sl@0	901	if( i==POS_END ){
sl@0	902	pReader->nData = 0;
sl@0	903	pReader->pData = NULL;
sl@0	904	return;
sl@0	905	}
sl@0	906
sl@0	907	pReader->iPosition += i-POS_BASE;
sl@0	908	if( pReader->iType==DL_POSITIONS_OFFSETS ){
sl@0	909	n += getVarint32(pReader->pData+n, &i);
sl@0	910	pReader->iStartOffset += i;
sl@0	911	n += getVarint32(pReader->pData+n, &i);
sl@0	912	pReader->iEndOffset = pReader->iStartOffset+i;
sl@0	913	}
sl@0	914	assert( n<=pReader->nData );
sl@0	915	pReader->pData += n;
sl@0	916	pReader->nData -= n;
sl@0	917	}
sl@0	918
sl@0	919	static void plrInit(PLReader pReader, DLReader pDLReader){
sl@0	920	pReader->pData = dlrPosData(pDLReader);
sl@0	921	pReader->nData = dlrPosDataLen(pDLReader);
sl@0	922	pReader->iType = pDLReader->iType;
sl@0	923	pReader->iColumn = 0;
sl@0	924	pReader->iPosition = 0;
sl@0	925	pReader->iStartOffset = 0;
sl@0	926	pReader->iEndOffset = 0;
sl@0	927	plrStep(pReader);
sl@0	928	}
sl@0	929	static void plrDestroy(PLReader *pReader){
sl@0	930	SCRAMBLE(pReader);
sl@0	931	}
sl@0	932
sl@0	933	/*******************************************************************/
sl@0	934	/* PLWriter is used in constructing a document's position list. As a
sl@0	935	** convenience, if iType is DL_DOCIDS, PLWriter becomes a no-op.
sl@0	936	** PLWriter writes to the associated DLWriter's buffer.
sl@0	937	**
sl@0	938	** plwInit - init for writing a document's poslist.
sl@0	939	** plwDestroy - clear a writer.
sl@0	940	** plwAdd - append position and offset information.
sl@0	941	** plwCopy - copy next position's data from reader to writer.
sl@0	942	** plwTerminate - add any necessary doclist terminator.
sl@0	943	**
sl@0	944	** Calling plwAdd() after plwTerminate() may result in a corrupt
sl@0	945	** doclist.
sl@0	946	*/
sl@0	947	/* TODO(shess) Until we've written the second item, we can cache the
sl@0	948	** first item's information. Then we'd have three states:
sl@0	949	**
sl@0	950	** - initialized with docid, no positions.
sl@0	951	** - docid and one position.
sl@0	952	** - docid and multiple positions.
sl@0	953	**
sl@0	954	** Only the last state needs to actually write to dlw->b, which would
sl@0	955	** be an improvement in the DLCollector case.
sl@0	956	*/
sl@0	957	typedef struct PLWriter {
sl@0	958	DLWriter *dlw;
sl@0	959
sl@0	960	int iColumn; /* the last column written */
sl@0	961	int iPos; /* the last position written */
sl@0	962	int iOffset; /* the last start offset written */
sl@0	963	} PLWriter;
sl@0	964
sl@0	965	/* TODO(shess) In the case where the parent is reading these values
sl@0	966	** from a PLReader, we could optimize to a copy if that PLReader has
sl@0	967	** the same type as pWriter.
sl@0	968	*/
sl@0	969	static void plwAdd(PLWriter *pWriter, int iColumn, int iPos,
sl@0	970	int iStartOffset, int iEndOffset){
sl@0	971	/* Worst-case space for POS_COLUMN, iColumn, iPosDelta,
sl@0	972	** iStartOffsetDelta, and iEndOffsetDelta.
sl@0	973	*/
sl@0	974	char c[5*VARINT_MAX];
sl@0	975	int n = 0;
sl@0	976
sl@0	977	/* Ban plwAdd() after plwTerminate(). */
sl@0	978	assert( pWriter->iPos!=-1 );
sl@0	979
sl@0	980	if( pWriter->dlw->iType==DL_DOCIDS ) return;
sl@0	981
sl@0	982	if( iColumn!=pWriter->iColumn ){
sl@0	983	n += putVarint(c+n, POS_COLUMN);
sl@0	984	n += putVarint(c+n, iColumn);
sl@0	985	pWriter->iColumn = iColumn;
sl@0	986	pWriter->iPos = 0;
sl@0	987	pWriter->iOffset = 0;
sl@0	988	}
sl@0	989	assert( iPos>=pWriter->iPos );
sl@0	990	n += putVarint(c+n, POS_BASE+(iPos-pWriter->iPos));
sl@0	991	pWriter->iPos = iPos;
sl@0	992	if( pWriter->dlw->iType==DL_POSITIONS_OFFSETS ){
sl@0	993	assert( iStartOffset>=pWriter->iOffset );
sl@0	994	n += putVarint(c+n, iStartOffset-pWriter->iOffset);
sl@0	995	pWriter->iOffset = iStartOffset;
sl@0	996	assert( iEndOffset>=iStartOffset );
sl@0	997	n += putVarint(c+n, iEndOffset-iStartOffset);
sl@0	998	}
sl@0	999	dataBufferAppend(pWriter->dlw->b, c, n);
sl@0	1000	}
sl@0	1001	static void plwCopy(PLWriter pWriter, PLReader pReader){
sl@0	1002	plwAdd(pWriter, plrColumn(pReader), plrPosition(pReader),
sl@0	1003	plrStartOffset(pReader), plrEndOffset(pReader));
sl@0	1004	}
sl@0	1005	static void plwInit(PLWriter pWriter, DLWriter dlw, sqlite_int64 iDocid){
sl@0	1006	char c[VARINT_MAX];
sl@0	1007	int n;
sl@0	1008
sl@0	1009	pWriter->dlw = dlw;
sl@0	1010
sl@0	1011	/* Docids must ascend. */
sl@0	1012	assert( !pWriter->dlw->has_iPrevDocid \|\| iDocid>pWriter->dlw->iPrevDocid );
sl@0	1013	n = putVarint(c, iDocid-pWriter->dlw->iPrevDocid);
sl@0	1014	dataBufferAppend(pWriter->dlw->b, c, n);
sl@0	1015	pWriter->dlw->iPrevDocid = iDocid;
sl@0	1016	#ifndef NDEBUG
sl@0	1017	pWriter->dlw->has_iPrevDocid = 1;
sl@0	1018	#endif
sl@0	1019
sl@0	1020	pWriter->iColumn = 0;
sl@0	1021	pWriter->iPos = 0;
sl@0	1022	pWriter->iOffset = 0;
sl@0	1023	}
sl@0	1024	/* TODO(shess) Should plwDestroy() also terminate the doclist? But
sl@0	1025	** then plwDestroy() would no longer be just a destructor, it would
sl@0	1026	** also be doing work, which isn't consistent with the overall idiom.
sl@0	1027	** Another option would be for plwAdd() to always append any necessary
sl@0	1028	** terminator, so that the output is always correct. But that would
sl@0	1029	** add incremental work to the common case with the only benefit being
sl@0	1030	** API elegance. Punt for now.
sl@0	1031	*/
sl@0	1032	static void plwTerminate(PLWriter *pWriter){
sl@0	1033	if( pWriter->dlw->iType>DL_DOCIDS ){
sl@0	1034	char c[VARINT_MAX];
sl@0	1035	int n = putVarint(c, POS_END);
sl@0	1036	dataBufferAppend(pWriter->dlw->b, c, n);
sl@0	1037	}
sl@0	1038	#ifndef NDEBUG
sl@0	1039	/* Mark as terminated for assert in plwAdd(). */
sl@0	1040	pWriter->iPos = -1;
sl@0	1041	#endif
sl@0	1042	}
sl@0	1043	static void plwDestroy(PLWriter *pWriter){
sl@0	1044	SCRAMBLE(pWriter);
sl@0	1045	}
sl@0	1046
sl@0	1047	/*******************************************************************/
sl@0	1048	/* DLCollector wraps PLWriter and DLWriter to provide a
sl@0	1049	** dynamically-allocated doclist area to use during tokenization.
sl@0	1050	**
sl@0	1051	** dlcNew - malloc up and initialize a collector.
sl@0	1052	** dlcDelete - destroy a collector and all contained items.
sl@0	1053	** dlcAddPos - append position and offset information.
sl@0	1054	** dlcAddDoclist - add the collected doclist to the given buffer.
sl@0	1055	** dlcNext - terminate the current document and open another.
sl@0	1056	*/
sl@0	1057	typedef struct DLCollector {
sl@0	1058	DataBuffer b;
sl@0	1059	DLWriter dlw;
sl@0	1060	PLWriter plw;
sl@0	1061	} DLCollector;
sl@0	1062
sl@0	1063	/* TODO(shess) This could also be done by calling plwTerminate() and
sl@0	1064	** dataBufferAppend(). I tried that, expecting nominal performance
sl@0	1065	** differences, but it seemed to pretty reliably be worth 1% to code
sl@0	1066	** it this way. I suspect it is the incremental malloc overhead (some
sl@0	1067	** percentage of the plwTerminate() calls will cause a realloc), so
sl@0	1068	** this might be worth revisiting if the DataBuffer implementation
sl@0	1069	** changes.
sl@0	1070	*/
sl@0	1071	static void dlcAddDoclist(DLCollector pCollector, DataBuffer b){
sl@0	1072	if( pCollector->dlw.iType>DL_DOCIDS ){
sl@0	1073	char c[VARINT_MAX];
sl@0	1074	int n = putVarint(c, POS_END);
sl@0	1075	dataBufferAppend2(b, pCollector->b.pData, pCollector->b.nData, c, n);
sl@0	1076	}else{
sl@0	1077	dataBufferAppend(b, pCollector->b.pData, pCollector->b.nData);
sl@0	1078	}
sl@0	1079	}
sl@0	1080	static void dlcNext(DLCollector *pCollector, sqlite_int64 iDocid){
sl@0	1081	plwTerminate(&pCollector->plw);
sl@0	1082	plwDestroy(&pCollector->plw);
sl@0	1083	plwInit(&pCollector->plw, &pCollector->dlw, iDocid);
sl@0	1084	}
sl@0	1085	static void dlcAddPos(DLCollector *pCollector, int iColumn, int iPos,
sl@0	1086	int iStartOffset, int iEndOffset){
sl@0	1087	plwAdd(&pCollector->plw, iColumn, iPos, iStartOffset, iEndOffset);
sl@0	1088	}
sl@0	1089
sl@0	1090	static DLCollector *dlcNew(sqlite_int64 iDocid, DocListType iType){
sl@0	1091	DLCollector *pCollector = sqlite3_malloc(sizeof(DLCollector));
sl@0	1092	dataBufferInit(&pCollector->b, 0);
sl@0	1093	dlwInit(&pCollector->dlw, iType, &pCollector->b);
sl@0	1094	plwInit(&pCollector->plw, &pCollector->dlw, iDocid);
sl@0	1095	return pCollector;
sl@0	1096	}
sl@0	1097	static void dlcDelete(DLCollector *pCollector){
sl@0	1098	plwDestroy(&pCollector->plw);
sl@0	1099	dlwDestroy(&pCollector->dlw);
sl@0	1100	dataBufferDestroy(&pCollector->b);
sl@0	1101	SCRAMBLE(pCollector);
sl@0	1102	sqlite3_free(pCollector);
sl@0	1103	}
sl@0	1104
sl@0	1105
sl@0	1106	/* Copy the doclist data of iType in pData/nData into *out, trimming
sl@0	1107	** unnecessary data as we go. Only columns matching iColumn are
sl@0	1108	** copied, all columns copied if iColumn is -1. Elements with no
sl@0	1109	** matching columns are dropped. The output is an iOutType doclist.
sl@0	1110	*/
sl@0	1111	/* NOTE(shess) This code is only valid after all doclists are merged.
sl@0	1112	** If this is run before merges, then doclist items which represent
sl@0	1113	** deletion will be trimmed, and will thus not effect a deletion
sl@0	1114	** during the merge.
sl@0	1115	*/
sl@0	1116	static void docListTrim(DocListType iType, const char *pData, int nData,
sl@0	1117	int iColumn, DocListType iOutType, DataBuffer *out){
sl@0	1118	DLReader dlReader;
sl@0	1119	DLWriter dlWriter;
sl@0	1120
sl@0	1121	assert( iOutType<=iType );
sl@0	1122
sl@0	1123	dlrInit(&dlReader, iType, pData, nData);
sl@0	1124	dlwInit(&dlWriter, iOutType, out);
sl@0	1125
sl@0	1126	while( !dlrAtEnd(&dlReader) ){
sl@0	1127	PLReader plReader;
sl@0	1128	PLWriter plWriter;
sl@0	1129	int match = 0;
sl@0	1130
sl@0	1131	plrInit(&plReader, &dlReader);
sl@0	1132
sl@0	1133	while( !plrAtEnd(&plReader) ){
sl@0	1134	if( iColumn==-1 \|\| plrColumn(&plReader)==iColumn ){
sl@0	1135	if( !match ){
sl@0	1136	plwInit(&plWriter, &dlWriter, dlrDocid(&dlReader));
sl@0	1137	match = 1;
sl@0	1138	}
sl@0	1139	plwAdd(&plWriter, plrColumn(&plReader), plrPosition(&plReader),
sl@0	1140	plrStartOffset(&plReader), plrEndOffset(&plReader));
sl@0	1141	}
sl@0	1142	plrStep(&plReader);
sl@0	1143	}
sl@0	1144	if( match ){
sl@0	1145	plwTerminate(&plWriter);
sl@0	1146	plwDestroy(&plWriter);
sl@0	1147	}
sl@0	1148
sl@0	1149	plrDestroy(&plReader);
sl@0	1150	dlrStep(&dlReader);
sl@0	1151	}
sl@0	1152	dlwDestroy(&dlWriter);
sl@0	1153	dlrDestroy(&dlReader);
sl@0	1154	}
sl@0	1155
sl@0	1156	/* Used by docListMerge() to keep doclists in the ascending order by
sl@0	1157	** docid, then ascending order by age (so the newest comes first).
sl@0	1158	*/
sl@0	1159	typedef struct OrderedDLReader {
sl@0	1160	DLReader *pReader;
sl@0	1161
sl@0	1162	/* TODO(shess) If we assume that docListMerge pReaders is ordered by
sl@0	1163	** age (which we do), then we could use pReader comparisons to break
sl@0	1164	** ties.
sl@0	1165	*/
sl@0	1166	int idx;
sl@0	1167	} OrderedDLReader;
sl@0	1168
sl@0	1169	/* Order eof to end, then by docid asc, idx desc. */
sl@0	1170	static int orderedDLReaderCmp(OrderedDLReader r1, OrderedDLReader r2){
sl@0	1171	if( dlrAtEnd(r1->pReader) ){
sl@0	1172	if( dlrAtEnd(r2->pReader) ) return 0; /* Both atEnd(). */
sl@0	1173	return 1; /* Only r1 atEnd(). */
sl@0	1174	}
sl@0	1175	if( dlrAtEnd(r2->pReader) ) return -1; /* Only r2 atEnd(). */
sl@0	1176
sl@0	1177	if( dlrDocid(r1->pReader)<dlrDocid(r2->pReader) ) return -1;
sl@0	1178	if( dlrDocid(r1->pReader)>dlrDocid(r2->pReader) ) return 1;
sl@0	1179
sl@0	1180	/* Descending on idx. */
sl@0	1181	return r2->idx-r1->idx;
sl@0	1182	}
sl@0	1183
sl@0	1184	/* Bubble p[0] to appropriate place in p[1..n-1]. Assumes that
sl@0	1185	** p[1..n-1] is already sorted.
sl@0	1186	*/
sl@0	1187	/* TODO(shess) Is this frequent enough to warrant a binary search?
sl@0	1188	** Before implementing that, instrument the code to check. In most
sl@0	1189	** current usage, I expect that p[0] will be less than p[1] a very
sl@0	1190	** high proportion of the time.
sl@0	1191	*/
sl@0	1192	static void orderedDLReaderReorder(OrderedDLReader *p, int n){
sl@0	1193	while( n>1 && orderedDLReaderCmp(p, p+1)>0 ){
sl@0	1194	OrderedDLReader tmp = p[0];
sl@0	1195	p[0] = p[1];
sl@0	1196	p[1] = tmp;
sl@0	1197	n--;
sl@0	1198	p++;
sl@0	1199	}
sl@0	1200	}
sl@0	1201
sl@0	1202	/* Given an array of doclist readers, merge their doclist elements
sl@0	1203	** into out in sorted order (by docid), dropping elements from older
sl@0	1204	** readers when there is a duplicate docid. pReaders is assumed to be
sl@0	1205	** ordered by age, oldest first.
sl@0	1206	*/
sl@0	1207	/* TODO(shess) nReaders must be <= MERGE_COUNT. This should probably
sl@0	1208	** be fixed.
sl@0	1209	*/
sl@0	1210	static void docListMerge(DataBuffer *out,
sl@0	1211	DLReader *pReaders, int nReaders){
sl@0	1212	OrderedDLReader readers[MERGE_COUNT];
sl@0	1213	DLWriter writer;
sl@0	1214	int i, n;
sl@0	1215	const char *pStart = 0;
sl@0	1216	int nStart = 0;
sl@0	1217	sqlite_int64 iFirstDocid = 0, iLastDocid = 0;
sl@0	1218
sl@0	1219	assert( nReaders>0 );
sl@0	1220	if( nReaders==1 ){
sl@0	1221	dataBufferAppend(out, dlrDocData(pReaders), dlrAllDataBytes(pReaders));
sl@0	1222	return;
sl@0	1223	}
sl@0	1224
sl@0	1225	assert( nReaders<=MERGE_COUNT );
sl@0	1226	n = 0;
sl@0	1227	for(i=0; i<nReaders; i++){
sl@0	1228	assert( pReaders[i].iType==pReaders[0].iType );
sl@0	1229	readers[i].pReader = pReaders+i;
sl@0	1230	readers[i].idx = i;
sl@0	1231	n += dlrAllDataBytes(&pReaders[i]);
sl@0	1232	}
sl@0	1233	/* Conservatively size output to sum of inputs. Output should end
sl@0	1234	** up strictly smaller than input.
sl@0	1235	*/
sl@0	1236	dataBufferExpand(out, n);
sl@0	1237
sl@0	1238	/* Get the readers into sorted order. */
sl@0	1239	while( i-->0 ){
sl@0	1240	orderedDLReaderReorder(readers+i, nReaders-i);
sl@0	1241	}
sl@0	1242
sl@0	1243	dlwInit(&writer, pReaders[0].iType, out);
sl@0	1244	while( !dlrAtEnd(readers[0].pReader) ){
sl@0	1245	sqlite_int64 iDocid = dlrDocid(readers[0].pReader);
sl@0	1246
sl@0	1247	/* If this is a continuation of the current buffer to copy, extend
sl@0	1248	** that buffer. memcpy() seems to be more efficient if it has a
sl@0	1249	** lots of data to copy.
sl@0	1250	*/
sl@0	1251	if( dlrDocData(readers[0].pReader)==pStart+nStart ){
sl@0	1252	nStart += dlrDocDataBytes(readers[0].pReader);
sl@0	1253	}else{
sl@0	1254	if( pStart!=0 ){
sl@0	1255	dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid);
sl@0	1256	}
sl@0	1257	pStart = dlrDocData(readers[0].pReader);
sl@0	1258	nStart = dlrDocDataBytes(readers[0].pReader);
sl@0	1259	iFirstDocid = iDocid;
sl@0	1260	}
sl@0	1261	iLastDocid = iDocid;
sl@0	1262	dlrStep(readers[0].pReader);
sl@0	1263
sl@0	1264	/* Drop all of the older elements with the same docid. */
sl@0	1265	for(i=1; i<nReaders &&
sl@0	1266	!dlrAtEnd(readers[i].pReader) &&
sl@0	1267	dlrDocid(readers[i].pReader)==iDocid; i++){
sl@0	1268	dlrStep(readers[i].pReader);
sl@0	1269	}
sl@0	1270
sl@0	1271	/* Get the readers back into order. */
sl@0	1272	while( i-->0 ){
sl@0	1273	orderedDLReaderReorder(readers+i, nReaders-i);
sl@0	1274	}
sl@0	1275	}
sl@0	1276
sl@0	1277	/* Copy over any remaining elements. */
sl@0	1278	if( nStart>0 ) dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid);
sl@0	1279	dlwDestroy(&writer);
sl@0	1280	}
sl@0	1281
sl@0	1282	/* Helper function for posListUnion(). Compares the current position
sl@0	1283	** between left and right, returning as standard C idiom of <0 if
sl@0	1284	** left<right, >0 if left>right, and 0 if left==right. "End" always
sl@0	1285	** compares greater.
sl@0	1286	*/
sl@0	1287	static int posListCmp(PLReader pLeft, PLReader pRight){
sl@0	1288	assert( pLeft->iType==pRight->iType );
sl@0	1289	if( pLeft->iType==DL_DOCIDS ) return 0;
sl@0	1290
sl@0	1291	if( plrAtEnd(pLeft) ) return plrAtEnd(pRight) ? 0 : 1;
sl@0	1292	if( plrAtEnd(pRight) ) return -1;
sl@0	1293
sl@0	1294	if( plrColumn(pLeft)<plrColumn(pRight) ) return -1;
sl@0	1295	if( plrColumn(pLeft)>plrColumn(pRight) ) return 1;
sl@0	1296
sl@0	1297	if( plrPosition(pLeft)<plrPosition(pRight) ) return -1;
sl@0	1298	if( plrPosition(pLeft)>plrPosition(pRight) ) return 1;
sl@0	1299	if( pLeft->iType==DL_POSITIONS ) return 0;
sl@0	1300
sl@0	1301	if( plrStartOffset(pLeft)<plrStartOffset(pRight) ) return -1;
sl@0	1302	if( plrStartOffset(pLeft)>plrStartOffset(pRight) ) return 1;
sl@0	1303
sl@0	1304	if( plrEndOffset(pLeft)<plrEndOffset(pRight) ) return -1;
sl@0	1305	if( plrEndOffset(pLeft)>plrEndOffset(pRight) ) return 1;
sl@0	1306
sl@0	1307	return 0;
sl@0	1308	}
sl@0	1309
sl@0	1310	/* Write the union of position lists in pLeft and pRight to pOut.
sl@0	1311	** "Union" in this case meaning "All unique position tuples". Should
sl@0	1312	** work with any doclist type, though both inputs and the output
sl@0	1313	** should be the same type.
sl@0	1314	*/
sl@0	1315	static void posListUnion(DLReader pLeft, DLReader pRight, DLWriter *pOut){
sl@0	1316	PLReader left, right;
sl@0	1317	PLWriter writer;
sl@0	1318
sl@0	1319	assert( dlrDocid(pLeft)==dlrDocid(pRight) );
sl@0	1320	assert( pLeft->iType==pRight->iType );
sl@0	1321	assert( pLeft->iType==pOut->iType );
sl@0	1322
sl@0	1323	plrInit(&left, pLeft);
sl@0	1324	plrInit(&right, pRight);
sl@0	1325	plwInit(&writer, pOut, dlrDocid(pLeft));
sl@0	1326
sl@0	1327	while( !plrAtEnd(&left) \|\| !plrAtEnd(&right) ){
sl@0	1328	int c = posListCmp(&left, &right);
sl@0	1329	if( c<0 ){
sl@0	1330	plwCopy(&writer, &left);
sl@0	1331	plrStep(&left);
sl@0	1332	}else if( c>0 ){
sl@0	1333	plwCopy(&writer, &right);
sl@0	1334	plrStep(&right);
sl@0	1335	}else{
sl@0	1336	plwCopy(&writer, &left);
sl@0	1337	plrStep(&left);
sl@0	1338	plrStep(&right);
sl@0	1339	}
sl@0	1340	}
sl@0	1341
sl@0	1342	plwTerminate(&writer);
sl@0	1343	plwDestroy(&writer);
sl@0	1344	plrDestroy(&left);
sl@0	1345	plrDestroy(&right);
sl@0	1346	}
sl@0	1347
sl@0	1348	/* Write the union of doclists in pLeft and pRight to pOut. For
sl@0	1349	** docids in common between the inputs, the union of the position
sl@0	1350	** lists is written. Inputs and outputs are always type DL_DEFAULT.
sl@0	1351	*/
sl@0	1352	static void docListUnion(
sl@0	1353	const char *pLeft, int nLeft,
sl@0	1354	const char *pRight, int nRight,
sl@0	1355	DataBuffer pOut / Write the combined doclist here */
sl@0	1356	){
sl@0	1357	DLReader left, right;
sl@0	1358	DLWriter writer;
sl@0	1359
sl@0	1360	if( nLeft==0 ){
sl@0	1361	if( nRight!=0) dataBufferAppend(pOut, pRight, nRight);
sl@0	1362	return;
sl@0	1363	}
sl@0	1364	if( nRight==0 ){
sl@0	1365	dataBufferAppend(pOut, pLeft, nLeft);
sl@0	1366	return;
sl@0	1367	}
sl@0	1368
sl@0	1369	dlrInit(&left, DL_DEFAULT, pLeft, nLeft);
sl@0	1370	dlrInit(&right, DL_DEFAULT, pRight, nRight);
sl@0	1371	dlwInit(&writer, DL_DEFAULT, pOut);
sl@0	1372
sl@0	1373	while( !dlrAtEnd(&left) \|\| !dlrAtEnd(&right) ){
sl@0	1374	if( dlrAtEnd(&right) ){
sl@0	1375	dlwCopy(&writer, &left);
sl@0	1376	dlrStep(&left);
sl@0	1377	}else if( dlrAtEnd(&left) ){
sl@0	1378	dlwCopy(&writer, &right);
sl@0	1379	dlrStep(&right);
sl@0	1380	}else if( dlrDocid(&left)<dlrDocid(&right) ){
sl@0	1381	dlwCopy(&writer, &left);
sl@0	1382	dlrStep(&left);
sl@0	1383	}else if( dlrDocid(&left)>dlrDocid(&right) ){
sl@0	1384	dlwCopy(&writer, &right);
sl@0	1385	dlrStep(&right);
sl@0	1386	}else{
sl@0	1387	posListUnion(&left, &right, &writer);
sl@0	1388	dlrStep(&left);
sl@0	1389	dlrStep(&right);
sl@0	1390	}
sl@0	1391	}
sl@0	1392
sl@0	1393	dlrDestroy(&left);
sl@0	1394	dlrDestroy(&right);
sl@0	1395	dlwDestroy(&writer);
sl@0	1396	}
sl@0	1397
sl@0	1398	/* pLeft and pRight are DLReaders positioned to the same docid.
sl@0	1399	**
sl@0	1400	** If there are no instances in pLeft or pRight where the position
sl@0	1401	** of pLeft is one less than the position of pRight, then this
sl@0	1402	** routine adds nothing to pOut.
sl@0	1403	**
sl@0	1404	** If there are one or more instances where positions from pLeft
sl@0	1405	** are exactly one less than positions from pRight, then add a new
sl@0	1406	** document record to pOut. If pOut wants to hold positions, then
sl@0	1407	** include the positions from pRight that are one more than a
sl@0	1408	** position in pLeft. In other words: pRight.iPos==pLeft.iPos+1.
sl@0	1409	*/
sl@0	1410	static void posListPhraseMerge(DLReader pLeft, DLReader pRight,
sl@0	1411	DLWriter *pOut){
sl@0	1412	PLReader left, right;
sl@0	1413	PLWriter writer;
sl@0	1414	int match = 0;
sl@0	1415
sl@0	1416	assert( dlrDocid(pLeft)==dlrDocid(pRight) );
sl@0	1417	assert( pOut->iType!=DL_POSITIONS_OFFSETS );
sl@0	1418
sl@0	1419	plrInit(&left, pLeft);
sl@0	1420	plrInit(&right, pRight);
sl@0	1421
sl@0	1422	while( !plrAtEnd(&left) && !plrAtEnd(&right) ){
sl@0	1423	if( plrColumn(&left)<plrColumn(&right) ){
sl@0	1424	plrStep(&left);
sl@0	1425	}else if( plrColumn(&left)>plrColumn(&right) ){
sl@0	1426	plrStep(&right);
sl@0	1427	}else if( plrPosition(&left)+1<plrPosition(&right) ){
sl@0	1428	plrStep(&left);
sl@0	1429	}else if( plrPosition(&left)+1>plrPosition(&right) ){
sl@0	1430	plrStep(&right);
sl@0	1431	}else{
sl@0	1432	if( !match ){
sl@0	1433	plwInit(&writer, pOut, dlrDocid(pLeft));
sl@0	1434	match = 1;
sl@0	1435	}
sl@0	1436	plwAdd(&writer, plrColumn(&right), plrPosition(&right), 0, 0);
sl@0	1437	plrStep(&left);
sl@0	1438	plrStep(&right);
sl@0	1439	}
sl@0	1440	}
sl@0	1441
sl@0	1442	if( match ){
sl@0	1443	plwTerminate(&writer);
sl@0	1444	plwDestroy(&writer);
sl@0	1445	}
sl@0	1446
sl@0	1447	plrDestroy(&left);
sl@0	1448	plrDestroy(&right);
sl@0	1449	}
sl@0	1450
sl@0	1451	/* We have two doclists with positions: pLeft and pRight.
sl@0	1452	** Write the phrase intersection of these two doclists into pOut.
sl@0	1453	**
sl@0	1454	** A phrase intersection means that two documents only match
sl@0	1455	** if pLeft.iPos+1==pRight.iPos.
sl@0	1456	**
sl@0	1457	** iType controls the type of data written to pOut. If iType is
sl@0	1458	** DL_POSITIONS, the positions are those from pRight.
sl@0	1459	*/
sl@0	1460	static void docListPhraseMerge(
sl@0	1461	const char *pLeft, int nLeft,
sl@0	1462	const char *pRight, int nRight,
sl@0	1463	DocListType iType,
sl@0	1464	DataBuffer pOut / Write the combined doclist here */
sl@0	1465	){
sl@0	1466	DLReader left, right;
sl@0	1467	DLWriter writer;
sl@0	1468
sl@0	1469	if( nLeft==0 \|\| nRight==0 ) return;
sl@0	1470
sl@0	1471	assert( iType!=DL_POSITIONS_OFFSETS );
sl@0	1472
sl@0	1473	dlrInit(&left, DL_POSITIONS, pLeft, nLeft);
sl@0	1474	dlrInit(&right, DL_POSITIONS, pRight, nRight);
sl@0	1475	dlwInit(&writer, iType, pOut);
sl@0	1476
sl@0	1477	while( !dlrAtEnd(&left) && !dlrAtEnd(&right) ){
sl@0	1478	if( dlrDocid(&left)<dlrDocid(&right) ){
sl@0	1479	dlrStep(&left);
sl@0	1480	}else if( dlrDocid(&right)<dlrDocid(&left) ){
sl@0	1481	dlrStep(&right);
sl@0	1482	}else{
sl@0	1483	posListPhraseMerge(&left, &right, &writer);
sl@0	1484	dlrStep(&left);
sl@0	1485	dlrStep(&right);
sl@0	1486	}
sl@0	1487	}
sl@0	1488
sl@0	1489	dlrDestroy(&left);
sl@0	1490	dlrDestroy(&right);
sl@0	1491	dlwDestroy(&writer);
sl@0	1492	}
sl@0	1493
sl@0	1494	/* We have two DL_DOCIDS doclists: pLeft and pRight.
sl@0	1495	** Write the intersection of these two doclists into pOut as a
sl@0	1496	** DL_DOCIDS doclist.
sl@0	1497	*/
sl@0	1498	static void docListAndMerge(
sl@0	1499	const char *pLeft, int nLeft,
sl@0	1500	const char *pRight, int nRight,
sl@0	1501	DataBuffer pOut / Write the combined doclist here */
sl@0	1502	){
sl@0	1503	DLReader left, right;
sl@0	1504	DLWriter writer;
sl@0	1505
sl@0	1506	if( nLeft==0 \|\| nRight==0 ) return;
sl@0	1507
sl@0	1508	dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
sl@0	1509	dlrInit(&right, DL_DOCIDS, pRight, nRight);
sl@0	1510	dlwInit(&writer, DL_DOCIDS, pOut);
sl@0	1511
sl@0	1512	while( !dlrAtEnd(&left) && !dlrAtEnd(&right) ){
sl@0	1513	if( dlrDocid(&left)<dlrDocid(&right) ){
sl@0	1514	dlrStep(&left);
sl@0	1515	}else if( dlrDocid(&right)<dlrDocid(&left) ){
sl@0	1516	dlrStep(&right);
sl@0	1517	}else{
sl@0	1518	dlwAdd(&writer, dlrDocid(&left));
sl@0	1519	dlrStep(&left);
sl@0	1520	dlrStep(&right);
sl@0	1521	}
sl@0	1522	}
sl@0	1523
sl@0	1524	dlrDestroy(&left);
sl@0	1525	dlrDestroy(&right);
sl@0	1526	dlwDestroy(&writer);
sl@0	1527	}
sl@0	1528
sl@0	1529	/* We have two DL_DOCIDS doclists: pLeft and pRight.
sl@0	1530	** Write the union of these two doclists into pOut as a
sl@0	1531	** DL_DOCIDS doclist.
sl@0	1532	*/
sl@0	1533	static void docListOrMerge(
sl@0	1534	const char *pLeft, int nLeft,
sl@0	1535	const char *pRight, int nRight,
sl@0	1536	DataBuffer pOut / Write the combined doclist here */
sl@0	1537	){
sl@0	1538	DLReader left, right;
sl@0	1539	DLWriter writer;
sl@0	1540
sl@0	1541	if( nLeft==0 ){
sl@0	1542	if( nRight!=0 ) dataBufferAppend(pOut, pRight, nRight);
sl@0	1543	return;
sl@0	1544	}
sl@0	1545	if( nRight==0 ){
sl@0	1546	dataBufferAppend(pOut, pLeft, nLeft);
sl@0	1547	return;
sl@0	1548	}
sl@0	1549
sl@0	1550	dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
sl@0	1551	dlrInit(&right, DL_DOCIDS, pRight, nRight);
sl@0	1552	dlwInit(&writer, DL_DOCIDS, pOut);
sl@0	1553
sl@0	1554	while( !dlrAtEnd(&left) \|\| !dlrAtEnd(&right) ){
sl@0	1555	if( dlrAtEnd(&right) ){
sl@0	1556	dlwAdd(&writer, dlrDocid(&left));
sl@0	1557	dlrStep(&left);
sl@0	1558	}else if( dlrAtEnd(&left) ){
sl@0	1559	dlwAdd(&writer, dlrDocid(&right));
sl@0	1560	dlrStep(&right);
sl@0	1561	}else if( dlrDocid(&left)<dlrDocid(&right) ){
sl@0	1562	dlwAdd(&writer, dlrDocid(&left));
sl@0	1563	dlrStep(&left);
sl@0	1564	}else if( dlrDocid(&right)<dlrDocid(&left) ){
sl@0	1565	dlwAdd(&writer, dlrDocid(&right));
sl@0	1566	dlrStep(&right);
sl@0	1567	}else{
sl@0	1568	dlwAdd(&writer, dlrDocid(&left));
sl@0	1569	dlrStep(&left);
sl@0	1570	dlrStep(&right);
sl@0	1571	}
sl@0	1572	}
sl@0	1573
sl@0	1574	dlrDestroy(&left);
sl@0	1575	dlrDestroy(&right);
sl@0	1576	dlwDestroy(&writer);
sl@0	1577	}
sl@0	1578
sl@0	1579	/* We have two DL_DOCIDS doclists: pLeft and pRight.
sl@0	1580	** Write into pOut as DL_DOCIDS doclist containing all documents that
sl@0	1581	** occur in pLeft but not in pRight.
sl@0	1582	*/
sl@0	1583	static void docListExceptMerge(
sl@0	1584	const char *pLeft, int nLeft,
sl@0	1585	const char *pRight, int nRight,
sl@0	1586	DataBuffer pOut / Write the combined doclist here */
sl@0	1587	){
sl@0	1588	DLReader left, right;
sl@0	1589	DLWriter writer;
sl@0	1590
sl@0	1591	if( nLeft==0 ) return;
sl@0	1592	if( nRight==0 ){
sl@0	1593	dataBufferAppend(pOut, pLeft, nLeft);
sl@0	1594	return;
sl@0	1595	}
sl@0	1596
sl@0	1597	dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
sl@0	1598	dlrInit(&right, DL_DOCIDS, pRight, nRight);
sl@0	1599	dlwInit(&writer, DL_DOCIDS, pOut);
sl@0	1600
sl@0	1601	while( !dlrAtEnd(&left) ){
sl@0	1602	while( !dlrAtEnd(&right) && dlrDocid(&right)<dlrDocid(&left) ){
sl@0	1603	dlrStep(&right);
sl@0	1604	}
sl@0	1605	if( dlrAtEnd(&right) \|\| dlrDocid(&left)<dlrDocid(&right) ){
sl@0	1606	dlwAdd(&writer, dlrDocid(&left));
sl@0	1607	}
sl@0	1608	dlrStep(&left);
sl@0	1609	}
sl@0	1610
sl@0	1611	dlrDestroy(&left);
sl@0	1612	dlrDestroy(&right);
sl@0	1613	dlwDestroy(&writer);
sl@0	1614	}
sl@0	1615
sl@0	1616	static char string_dup_n(const char s, int n){
sl@0	1617	char *str = sqlite3_malloc(n + 1);
sl@0	1618	memcpy(str, s, n);
sl@0	1619	str[n] = '\0';
sl@0	1620	return str;
sl@0	1621	}
sl@0	1622
sl@0	1623	/* Duplicate a string; the caller must free() the returned string.
sl@0	1624	* (We don't use strdup() since it is not part of the standard C library and
sl@0	1625	* may not be available everywhere.) */
sl@0	1626	static char string_dup(const char s){
sl@0	1627	return string_dup_n(s, strlen(s));
sl@0	1628	}
sl@0	1629
sl@0	1630	/* Format a string, replacing each occurrence of the % character with
sl@0	1631	* zDb.zName. This may be more convenient than sqlite_mprintf()
sl@0	1632	* when one string is used repeatedly in a format string.
sl@0	1633	* The caller must free() the returned string. */
sl@0	1634	static char string_format(const char zFormat,
sl@0	1635	const char zDb, const char zName){
sl@0	1636	const char *p;
sl@0	1637	size_t len = 0;
sl@0	1638	size_t nDb = strlen(zDb);
sl@0	1639	size_t nName = strlen(zName);
sl@0	1640	size_t nFullTableName = nDb+1+nName;
sl@0	1641	char *result;
sl@0	1642	char *r;
sl@0	1643
sl@0	1644	/* first compute length needed */
sl@0	1645	for(p = zFormat ; *p ; ++p){
sl@0	1646	len += (*p=='%' ? nFullTableName : 1);
sl@0	1647	}
sl@0	1648	len += 1; /* for null terminator */
sl@0	1649
sl@0	1650	r = result = sqlite3_malloc(len);
sl@0	1651	for(p = zFormat; *p; ++p){
sl@0	1652	if( *p=='%' ){
sl@0	1653	memcpy(r, zDb, nDb);
sl@0	1654	r += nDb;
sl@0	1655	*r++ = '.';
sl@0	1656	memcpy(r, zName, nName);
sl@0	1657	r += nName;
sl@0	1658	} else {
sl@0	1659	r++ = p;
sl@0	1660	}
sl@0	1661	}
sl@0	1662	*r++ = '\0';
sl@0	1663	assert( r == result + len );
sl@0	1664	return result;
sl@0	1665	}
sl@0	1666
sl@0	1667	static int sql_exec(sqlite3 db, const char zDb, const char *zName,
sl@0	1668	const char *zFormat){
sl@0	1669	char *zCommand = string_format(zFormat, zDb, zName);
sl@0	1670	int rc;
sl@0	1671	TRACE(("FTS2 sql: %s\n", zCommand));
sl@0	1672	rc = sqlite3_exec(db, zCommand, NULL, 0, NULL);
sl@0	1673	sqlite3_free(zCommand);
sl@0	1674	return rc;
sl@0	1675	}
sl@0	1676
sl@0	1677	static int sql_prepare(sqlite3 db, const char zDb, const char *zName,
sl@0	1678	sqlite3_stmt *ppStmt, const char zFormat){
sl@0	1679	char *zCommand = string_format(zFormat, zDb, zName);
sl@0	1680	int rc;
sl@0	1681	TRACE(("FTS2 prepare: %s\n", zCommand));
sl@0	1682	rc = sqlite3_prepare_v2(db, zCommand, -1, ppStmt, NULL);
sl@0	1683	sqlite3_free(zCommand);
sl@0	1684	return rc;
sl@0	1685	}
sl@0	1686
sl@0	1687	/* end utility functions */
sl@0	1688
sl@0	1689	/* Forward reference */
sl@0	1690	typedef struct fulltext_vtab fulltext_vtab;
sl@0	1691
sl@0	1692	/* A single term in a query is represented by an instances of
sl@0	1693	** the following structure.
sl@0	1694	*/
sl@0	1695	typedef struct QueryTerm {
sl@0	1696	short int nPhrase; /* How many following terms are part of the same phrase */
sl@0	1697	short int iPhrase; /* This is the i-th term of a phrase. */
sl@0	1698	short int iColumn; /* Column of the index that must match this term */
sl@0	1699	signed char isOr; /* this term is preceded by "OR" */
sl@0	1700	signed char isNot; /* this term is preceded by "-" */
sl@0	1701	signed char isPrefix; /* this term is followed by "" /
sl@0	1702	char pTerm; / text of the term. '\000' terminated. malloced */
sl@0	1703	int nTerm; /* Number of bytes in pTerm[] */
sl@0	1704	} QueryTerm;
sl@0	1705
sl@0	1706
sl@0	1707	/* A query string is parsed into a Query structure.
sl@0	1708	*
sl@0	1709	* We could, in theory, allow query strings to be complicated
sl@0	1710	* nested expressions with precedence determined by parentheses.
sl@0	1711	* But none of the major search engines do this. (Perhaps the
sl@0	1712	* feeling is that an parenthesized expression is two complex of
sl@0	1713	* an idea for the average user to grasp.) Taking our lead from
sl@0	1714	* the major search engines, we will allow queries to be a list
sl@0	1715	* of terms (with an implied AND operator) or phrases in double-quotes,
sl@0	1716	* with a single optional "-" before each non-phrase term to designate
sl@0	1717	* negation and an optional OR connector.
sl@0	1718	*
sl@0	1719	* OR binds more tightly than the implied AND, which is what the
sl@0	1720	* major search engines seem to do. So, for example:
sl@0	1721	*
sl@0	1722	* [one two OR three] ==> one AND (two OR three)
sl@0	1723	* [one OR two three] ==> (one OR two) AND three
sl@0	1724	*
sl@0	1725	* A "-" before a term matches all entries that lack that term.
sl@0	1726	* The "-" must occur immediately before the term with in intervening
sl@0	1727	* space. This is how the search engines do it.
sl@0	1728	*
sl@0	1729	* A NOT term cannot be the right-hand operand of an OR. If this
sl@0	1730	* occurs in the query string, the NOT is ignored:
sl@0	1731	*
sl@0	1732	* [one OR -two] ==> one OR two
sl@0	1733	*
sl@0	1734	*/
sl@0	1735	typedef struct Query {
sl@0	1736	fulltext_vtab pFts; / The full text index */
sl@0	1737	int nTerms; /* Number of terms in the query */
sl@0	1738	QueryTerm pTerms; / Array of terms. Space obtained from malloc() */
sl@0	1739	int nextIsOr; /* Set the isOr flag on the next inserted term */
sl@0	1740	int nextColumn; /* Next word parsed must be in this column */
sl@0	1741	int dfltColumn; /* The default column */
sl@0	1742	} Query;
sl@0	1743
sl@0	1744
sl@0	1745	/*
sl@0	1746	** An instance of the following structure keeps track of generated
sl@0	1747	** matching-word offset information and snippets.
sl@0	1748	*/
sl@0	1749	typedef struct Snippet {
sl@0	1750	int nMatch; /* Total number of matches */
sl@0	1751	int nAlloc; /* Space allocated for aMatch[] */
sl@0	1752	struct snippetMatch { /* One entry for each matching term */
sl@0	1753	char snStatus; /* Status flag for use while constructing snippets */
sl@0	1754	short int iCol; /* The column that contains the match */
sl@0	1755	short int iTerm; /* The index in Query.pTerms[] of the matching term */
sl@0	1756	short int nByte; /* Number of bytes in the term */
sl@0	1757	int iStart; /* The offset to the first character of the term */
sl@0	1758	} aMatch; / Points to space obtained from malloc */
sl@0	1759	char zOffset; / Text rendering of aMatch[] */
sl@0	1760	int nOffset; /* strlen(zOffset) */
sl@0	1761	char zSnippet; / Snippet text */
sl@0	1762	int nSnippet; /* strlen(zSnippet) */
sl@0	1763	} Snippet;
sl@0	1764
sl@0	1765
sl@0	1766	typedef enum QueryType {
sl@0	1767	QUERY_GENERIC, /* table scan */
sl@0	1768	QUERY_ROWID, /* lookup by rowid */
sl@0	1769	QUERY_FULLTEXT /* QUERY_FULLTEXT + [i] is a full-text search for column i*/
sl@0	1770	} QueryType;
sl@0	1771
sl@0	1772	typedef enum fulltext_statement {
sl@0	1773	CONTENT_INSERT_STMT,
sl@0	1774	CONTENT_SELECT_STMT,
sl@0	1775	CONTENT_UPDATE_STMT,
sl@0	1776	CONTENT_DELETE_STMT,
sl@0	1777	CONTENT_EXISTS_STMT,
sl@0	1778
sl@0	1779	BLOCK_INSERT_STMT,
sl@0	1780	BLOCK_SELECT_STMT,
sl@0	1781	BLOCK_DELETE_STMT,
sl@0	1782	BLOCK_DELETE_ALL_STMT,
sl@0	1783
sl@0	1784	SEGDIR_MAX_INDEX_STMT,
sl@0	1785	SEGDIR_SET_STMT,
sl@0	1786	SEGDIR_SELECT_LEVEL_STMT,
sl@0	1787	SEGDIR_SPAN_STMT,
sl@0	1788	SEGDIR_DELETE_STMT,
sl@0	1789	SEGDIR_SELECT_SEGMENT_STMT,
sl@0	1790	SEGDIR_SELECT_ALL_STMT,
sl@0	1791	SEGDIR_DELETE_ALL_STMT,
sl@0	1792	SEGDIR_COUNT_STMT,
sl@0	1793
sl@0	1794	MAX_STMT /* Always at end! */
sl@0	1795	} fulltext_statement;
sl@0	1796
sl@0	1797	/* These must exactly match the enum above. */
sl@0	1798	/* TODO(shess): Is there some risk that a statement will be used in two
sl@0	1799	** cursors at once, e.g. if a query joins a virtual table to itself?
sl@0	1800	** If so perhaps we should move some of these to the cursor object.
sl@0	1801	*/
sl@0	1802	static const char *const fulltext_zStatement[MAX_STMT] = {
sl@0	1803	/* CONTENT_INSERT / NULL, / generated in contentInsertStatement() */
sl@0	1804	/* CONTENT_SELECT / "select from %_content where rowid = ?",
sl@0	1805	/* CONTENT_UPDATE / NULL, / generated in contentUpdateStatement() */
sl@0	1806	/* CONTENT_DELETE */ "delete from %_content where rowid = ?",
sl@0	1807	/* CONTENT_EXISTS */ "select rowid from %_content limit 1",
sl@0	1808
sl@0	1809	/* BLOCK_INSERT */ "insert into %_segments values (?)",
sl@0	1810	/* BLOCK_SELECT */ "select block from %_segments where rowid = ?",
sl@0	1811	/* BLOCK_DELETE */ "delete from %_segments where rowid between ? and ?",
sl@0	1812	/* BLOCK_DELETE_ALL */ "delete from %_segments",
sl@0	1813
sl@0	1814	/* SEGDIR_MAX_INDEX */ "select max(idx) from %_segdir where level = ?",
sl@0	1815	/* SEGDIR_SET */ "insert into %_segdir values (?, ?, ?, ?, ?, ?)",
sl@0	1816	/* SEGDIR_SELECT_LEVEL */
sl@0	1817	"select start_block, leaves_end_block, root from %_segdir "
sl@0	1818	" where level = ? order by idx",
sl@0	1819	/* SEGDIR_SPAN */
sl@0	1820	"select min(start_block), max(end_block) from %_segdir "
sl@0	1821	" where level = ? and start_block <> 0",
sl@0	1822	/* SEGDIR_DELETE */ "delete from %_segdir where level = ?",
sl@0	1823
sl@0	1824	/* NOTE(shess): The first three results of the following two
sl@0	1825	** statements must match.
sl@0	1826	*/
sl@0	1827	/* SEGDIR_SELECT_SEGMENT */
sl@0	1828	"select start_block, leaves_end_block, root from %_segdir "
sl@0	1829	" where level = ? and idx = ?",
sl@0	1830	/* SEGDIR_SELECT_ALL */
sl@0	1831	"select start_block, leaves_end_block, root from %_segdir "
sl@0	1832	" order by level desc, idx asc",
sl@0	1833	/* SEGDIR_DELETE_ALL */ "delete from %_segdir",
sl@0	1834	/* SEGDIR_COUNT / "select count(), ifnull(max(level),0) from %_segdir",
sl@0	1835	};
sl@0	1836
sl@0	1837	/*
sl@0	1838	** A connection to a fulltext index is an instance of the following
sl@0	1839	** structure. The xCreate and xConnect methods create an instance
sl@0	1840	** of this structure and xDestroy and xDisconnect free that instance.
sl@0	1841	** All other methods receive a pointer to the structure as one of their
sl@0	1842	** arguments.
sl@0	1843	*/
sl@0	1844	struct fulltext_vtab {
sl@0	1845	sqlite3_vtab base; /* Base class used by SQLite core */
sl@0	1846	sqlite3 db; / The database connection */
sl@0	1847	const char zDb; / logical database name */
sl@0	1848	const char zName; / virtual table name */
sl@0	1849	int nColumn; /* number of columns in virtual table */
sl@0	1850	char *azColumn; / column names. malloced */
sl@0	1851	char *azContentColumn; / column names in content table; malloced */
sl@0	1852	sqlite3_tokenizer pTokenizer; / tokenizer for inserts and queries */
sl@0	1853
sl@0	1854	/* Precompiled statements which we keep as long as the table is
sl@0	1855	** open.
sl@0	1856	*/
sl@0	1857	sqlite3_stmt *pFulltextStatements[MAX_STMT];
sl@0	1858
sl@0	1859	/* Precompiled statements used for segment merges. We run a
sl@0	1860	** separate select across the leaf level of each tree being merged.
sl@0	1861	*/
sl@0	1862	sqlite3_stmt *pLeafSelectStmts[MERGE_COUNT];
sl@0	1863	/* The statement used to prepare pLeafSelectStmts. */
sl@0	1864	#define LEAF_SELECT \
sl@0	1865	"select block from %_segments where rowid between ? and ? order by rowid"
sl@0	1866
sl@0	1867	/* These buffer pending index updates during transactions.
sl@0	1868	** nPendingData estimates the memory size of the pending data. It
sl@0	1869	** doesn't include the hash-bucket overhead, nor any malloc
sl@0	1870	** overhead. When nPendingData exceeds kPendingThreshold, the
sl@0	1871	** buffer is flushed even before the transaction closes.
sl@0	1872	** pendingTerms stores the data, and is only valid when nPendingData
sl@0	1873	** is >=0 (nPendingData<0 means pendingTerms has not been
sl@0	1874	** initialized). iPrevDocid is the last docid written, used to make
sl@0	1875	** certain we're inserting in sorted order.
sl@0	1876	*/
sl@0	1877	int nPendingData;
sl@0	1878	#define kPendingThreshold (110241024)
sl@0	1879	sqlite_int64 iPrevDocid;
sl@0	1880	fts2Hash pendingTerms;
sl@0	1881	};
sl@0	1882
sl@0	1883	/*
sl@0	1884	** When the core wants to do a query, it create a cursor using a
sl@0	1885	** call to xOpen. This structure is an instance of a cursor. It
sl@0	1886	** is destroyed by xClose.
sl@0	1887	*/
sl@0	1888	typedef struct fulltext_cursor {
sl@0	1889	sqlite3_vtab_cursor base; /* Base class used by SQLite core */
sl@0	1890	QueryType iCursorType; /* Copy of sqlite3_index_info.idxNum */
sl@0	1891	sqlite3_stmt pStmt; / Prepared statement in use by the cursor */
sl@0	1892	int eof; /* True if at End Of Results */
sl@0	1893	Query q; /* Parsed query string */
sl@0	1894	Snippet snippet; /* Cached snippet for the current row */
sl@0	1895	int iColumn; /* Column being searched */
sl@0	1896	DataBuffer result; /* Doclist results from fulltextQuery */
sl@0	1897	DLReader reader; /* Result reader if result not empty */
sl@0	1898	} fulltext_cursor;
sl@0	1899
sl@0	1900	static struct fulltext_vtab cursor_vtab(fulltext_cursor c){
sl@0	1901	return (fulltext_vtab *) c->base.pVtab;
sl@0	1902	}
sl@0	1903
sl@0	1904	static const sqlite3_module fts2Module; /* forward declaration */
sl@0	1905
sl@0	1906	/* Return a dynamically generated statement of the form
sl@0	1907	* insert into %_content (rowid, ...) values (?, ...)
sl@0	1908	*/
sl@0	1909	static const char contentInsertStatement(fulltext_vtab v){
sl@0	1910	StringBuffer sb;
sl@0	1911	int i;
sl@0	1912
sl@0	1913	initStringBuffer(&sb);
sl@0	1914	append(&sb, "insert into %_content (rowid, ");
sl@0	1915	appendList(&sb, v->nColumn, v->azContentColumn);
sl@0	1916	append(&sb, ") values (?");
sl@0	1917	for(i=0; i<v->nColumn; ++i)
sl@0	1918	append(&sb, ", ?");
sl@0	1919	append(&sb, ")");
sl@0	1920	return stringBufferData(&sb);
sl@0	1921	}
sl@0	1922
sl@0	1923	/* Return a dynamically generated statement of the form
sl@0	1924	* update %_content set [col_0] = ?, [col_1] = ?, ...
sl@0	1925	* where rowid = ?
sl@0	1926	*/
sl@0	1927	static const char contentUpdateStatement(fulltext_vtab v){
sl@0	1928	StringBuffer sb;
sl@0	1929	int i;
sl@0	1930
sl@0	1931	initStringBuffer(&sb);
sl@0	1932	append(&sb, "update %_content set ");
sl@0	1933	for(i=0; i<v->nColumn; ++i) {
sl@0	1934	if( i>0 ){
sl@0	1935	append(&sb, ", ");
sl@0	1936	}
sl@0	1937	append(&sb, v->azContentColumn[i]);
sl@0	1938	append(&sb, " = ?");
sl@0	1939	}
sl@0	1940	append(&sb, " where rowid = ?");
sl@0	1941	return stringBufferData(&sb);
sl@0	1942	}
sl@0	1943
sl@0	1944	/* Puts a freshly-prepared statement determined by iStmt in *ppStmt.
sl@0	1945	** If the indicated statement has never been prepared, it is prepared
sl@0	1946	** and cached, otherwise the cached version is reset.
sl@0	1947	*/
sl@0	1948	static int sql_get_statement(fulltext_vtab *v, fulltext_statement iStmt,
sl@0	1949	sqlite3_stmt **ppStmt){
sl@0	1950	assert( iStmt<MAX_STMT );
sl@0	1951	if( v->pFulltextStatements[iStmt]==NULL ){
sl@0	1952	const char *zStmt;
sl@0	1953	int rc;
sl@0	1954	switch( iStmt ){
sl@0	1955	case CONTENT_INSERT_STMT:
sl@0	1956	zStmt = contentInsertStatement(v); break;
sl@0	1957	case CONTENT_UPDATE_STMT:
sl@0	1958	zStmt = contentUpdateStatement(v); break;
sl@0	1959	default:
sl@0	1960	zStmt = fulltext_zStatement[iStmt];
sl@0	1961	}
sl@0	1962	rc = sql_prepare(v->db, v->zDb, v->zName, &v->pFulltextStatements[iStmt],
sl@0	1963	zStmt);
sl@0	1964	if( zStmt != fulltext_zStatement[iStmt]) sqlite3_free((void *) zStmt);
sl@0	1965	if( rc!=SQLITE_OK ) return rc;
sl@0	1966	} else {
sl@0	1967	int rc = sqlite3_reset(v->pFulltextStatements[iStmt]);
sl@0	1968	if( rc!=SQLITE_OK ) return rc;
sl@0	1969	}
sl@0	1970
sl@0	1971	*ppStmt = v->pFulltextStatements[iStmt];
sl@0	1972	return SQLITE_OK;
sl@0	1973	}
sl@0	1974
sl@0	1975	/* Like sqlite3_step(), but convert SQLITE_DONE to SQLITE_OK and
sl@0	1976	** SQLITE_ROW to SQLITE_ERROR. Useful for statements like UPDATE,
sl@0	1977	** where we expect no results.
sl@0	1978	*/
sl@0	1979	static int sql_single_step(sqlite3_stmt *s){
sl@0	1980	int rc = sqlite3_step(s);
sl@0	1981	return (rc==SQLITE_DONE) ? SQLITE_OK : rc;
sl@0	1982	}
sl@0	1983
sl@0	1984	/* Like sql_get_statement(), but for special replicated LEAF_SELECT
sl@0	1985	** statements. idx -1 is a special case for an uncached version of
sl@0	1986	** the statement (used in the optimize implementation).
sl@0	1987	*/
sl@0	1988	/* TODO(shess) Write version for generic statements and then share
sl@0	1989	** that between the cached-statement functions.
sl@0	1990	*/
sl@0	1991	static int sql_get_leaf_statement(fulltext_vtab *v, int idx,
sl@0	1992	sqlite3_stmt **ppStmt){
sl@0	1993	assert( idx>=-1 && idx<MERGE_COUNT );
sl@0	1994	if( idx==-1 ){
sl@0	1995	return sql_prepare(v->db, v->zDb, v->zName, ppStmt, LEAF_SELECT);
sl@0	1996	}else if( v->pLeafSelectStmts[idx]==NULL ){
sl@0	1997	int rc = sql_prepare(v->db, v->zDb, v->zName, &v->pLeafSelectStmts[idx],
sl@0	1998	LEAF_SELECT);
sl@0	1999	if( rc!=SQLITE_OK ) return rc;
sl@0	2000	}else{
sl@0	2001	int rc = sqlite3_reset(v->pLeafSelectStmts[idx]);
sl@0	2002	if( rc!=SQLITE_OK ) return rc;
sl@0	2003	}
sl@0	2004
sl@0	2005	*ppStmt = v->pLeafSelectStmts[idx];
sl@0	2006	return SQLITE_OK;
sl@0	2007	}
sl@0	2008
sl@0	2009	/* insert into %_content (rowid, ...) values ([rowid], [pValues]) */
sl@0	2010	static int content_insert(fulltext_vtab v, sqlite3_value rowid,
sl@0	2011	sqlite3_value **pValues){
sl@0	2012	sqlite3_stmt *s;
sl@0	2013	int i;
sl@0	2014	int rc = sql_get_statement(v, CONTENT_INSERT_STMT, &s);
sl@0	2015	if( rc!=SQLITE_OK ) return rc;
sl@0	2016
sl@0	2017	rc = sqlite3_bind_value(s, 1, rowid);
sl@0	2018	if( rc!=SQLITE_OK ) return rc;
sl@0	2019
sl@0	2020	for(i=0; i<v->nColumn; ++i){
sl@0	2021	rc = sqlite3_bind_value(s, 2+i, pValues[i]);
sl@0	2022	if( rc!=SQLITE_OK ) return rc;
sl@0	2023	}
sl@0	2024
sl@0	2025	return sql_single_step(s);
sl@0	2026	}
sl@0	2027
sl@0	2028	/* update %_content set col0 = pValues[0], col1 = pValues[1], ...
sl@0	2029	* where rowid = [iRowid] */
sl@0	2030	static int content_update(fulltext_vtab v, sqlite3_value *pValues,
sl@0	2031	sqlite_int64 iRowid){
sl@0	2032	sqlite3_stmt *s;
sl@0	2033	int i;
sl@0	2034	int rc = sql_get_statement(v, CONTENT_UPDATE_STMT, &s);
sl@0	2035	if( rc!=SQLITE_OK ) return rc;
sl@0	2036
sl@0	2037	for(i=0; i<v->nColumn; ++i){
sl@0	2038	rc = sqlite3_bind_value(s, 1+i, pValues[i]);
sl@0	2039	if( rc!=SQLITE_OK ) return rc;
sl@0	2040	}
sl@0	2041
sl@0	2042	rc = sqlite3_bind_int64(s, 1+v->nColumn, iRowid);
sl@0	2043	if( rc!=SQLITE_OK ) return rc;
sl@0	2044
sl@0	2045	return sql_single_step(s);
sl@0	2046	}
sl@0	2047
sl@0	2048	static void freeStringArray(int nString, const char **pString){
sl@0	2049	int i;
sl@0	2050
sl@0	2051	for (i=0 ; i < nString ; ++i) {
sl@0	2052	if( pString[i]!=NULL ) sqlite3_free((void *) pString[i]);
sl@0	2053	}
sl@0	2054	sqlite3_free((void *) pString);
sl@0	2055	}
sl@0	2056
sl@0	2057	/* select * from %_content where rowid = [iRow]
sl@0	2058	* The caller must delete the returned array and all strings in it.
sl@0	2059	* null fields will be NULL in the returned array.
sl@0	2060	*
sl@0	2061	* TODO: Perhaps we should return pointer/length strings here for consistency
sl@0	2062	* with other code which uses pointer/length. */
sl@0	2063	static int content_select(fulltext_vtab *v, sqlite_int64 iRow,
sl@0	2064	const char ***pValues){
sl@0	2065	sqlite3_stmt *s;
sl@0	2066	const char **values;
sl@0	2067	int i;
sl@0	2068	int rc;
sl@0	2069
sl@0	2070	*pValues = NULL;
sl@0	2071
sl@0	2072	rc = sql_get_statement(v, CONTENT_SELECT_STMT, &s);
sl@0	2073	if( rc!=SQLITE_OK ) return rc;
sl@0	2074
sl@0	2075	rc = sqlite3_bind_int64(s, 1, iRow);
sl@0	2076	if( rc!=SQLITE_OK ) return rc;
sl@0	2077
sl@0	2078	rc = sqlite3_step(s);
sl@0	2079	if( rc!=SQLITE_ROW ) return rc;
sl@0	2080
sl@0	2081	values = (const char *) sqlite3_malloc(v->nColumn sizeof(const char *));
sl@0	2082	for(i=0; i<v->nColumn; ++i){
sl@0	2083	if( sqlite3_column_type(s, i)==SQLITE_NULL ){
sl@0	2084	values[i] = NULL;
sl@0	2085	}else{
sl@0	2086	values[i] = string_dup((char*)sqlite3_column_text(s, i));
sl@0	2087	}
sl@0	2088	}
sl@0	2089
sl@0	2090	/* We expect only one row. We must execute another sqlite3_step()
sl@0	2091	* to complete the iteration; otherwise the table will remain locked. */
sl@0	2092	rc = sqlite3_step(s);
sl@0	2093	if( rc==SQLITE_DONE ){
sl@0	2094	*pValues = values;
sl@0	2095	return SQLITE_OK;
sl@0	2096	}
sl@0	2097
sl@0	2098	freeStringArray(v->nColumn, values);
sl@0	2099	return rc;
sl@0	2100	}
sl@0	2101
sl@0	2102	/* delete from %_content where rowid = [iRow ] */
sl@0	2103	static int content_delete(fulltext_vtab *v, sqlite_int64 iRow){
sl@0	2104	sqlite3_stmt *s;
sl@0	2105	int rc = sql_get_statement(v, CONTENT_DELETE_STMT, &s);
sl@0	2106	if( rc!=SQLITE_OK ) return rc;
sl@0	2107
sl@0	2108	rc = sqlite3_bind_int64(s, 1, iRow);
sl@0	2109	if( rc!=SQLITE_OK ) return rc;
sl@0	2110
sl@0	2111	return sql_single_step(s);
sl@0	2112	}
sl@0	2113
sl@0	2114	/* Returns SQLITE_ROW if any rows exist in %_content, SQLITE_DONE if
sl@0	2115	** no rows exist, and any error in case of failure.
sl@0	2116	*/
sl@0	2117	static int content_exists(fulltext_vtab *v){
sl@0	2118	sqlite3_stmt *s;
sl@0	2119	int rc = sql_get_statement(v, CONTENT_EXISTS_STMT, &s);
sl@0	2120	if( rc!=SQLITE_OK ) return rc;
sl@0	2121
sl@0	2122	rc = sqlite3_step(s);
sl@0	2123	if( rc!=SQLITE_ROW ) return rc;
sl@0	2124
sl@0	2125	/* We expect only one row. We must execute another sqlite3_step()
sl@0	2126	* to complete the iteration; otherwise the table will remain locked. */
sl@0	2127	rc = sqlite3_step(s);
sl@0	2128	if( rc==SQLITE_DONE ) return SQLITE_ROW;
sl@0	2129	if( rc==SQLITE_ROW ) return SQLITE_ERROR;
sl@0	2130	return rc;
sl@0	2131	}
sl@0	2132
sl@0	2133	/* insert into %_segments values ([pData])
sl@0	2134	** returns assigned rowid in *piBlockid
sl@0	2135	*/
sl@0	2136	static int block_insert(fulltext_vtab v, const char pData, int nData,
sl@0	2137	sqlite_int64 *piBlockid){
sl@0	2138	sqlite3_stmt *s;
sl@0	2139	int rc = sql_get_statement(v, BLOCK_INSERT_STMT, &s);
sl@0	2140	if( rc!=SQLITE_OK ) return rc;
sl@0	2141
sl@0	2142	rc = sqlite3_bind_blob(s, 1, pData, nData, SQLITE_STATIC);
sl@0	2143	if( rc!=SQLITE_OK ) return rc;
sl@0	2144
sl@0	2145	rc = sqlite3_step(s);
sl@0	2146	if( rc==SQLITE_ROW ) return SQLITE_ERROR;
sl@0	2147	if( rc!=SQLITE_DONE ) return rc;
sl@0	2148
sl@0	2149	*piBlockid = sqlite3_last_insert_rowid(v->db);
sl@0	2150	return SQLITE_OK;
sl@0	2151	}
sl@0	2152
sl@0	2153	/* delete from %_segments
sl@0	2154	** where rowid between [iStartBlockid] and [iEndBlockid]
sl@0	2155	**
sl@0	2156	** Deletes the range of blocks, inclusive, used to delete the blocks
sl@0	2157	** which form a segment.
sl@0	2158	*/
sl@0	2159	static int block_delete(fulltext_vtab *v,
sl@0	2160	sqlite_int64 iStartBlockid, sqlite_int64 iEndBlockid){
sl@0	2161	sqlite3_stmt *s;
sl@0	2162	int rc = sql_get_statement(v, BLOCK_DELETE_STMT, &s);
sl@0	2163	if( rc!=SQLITE_OK ) return rc;
sl@0	2164
sl@0	2165	rc = sqlite3_bind_int64(s, 1, iStartBlockid);
sl@0	2166	if( rc!=SQLITE_OK ) return rc;
sl@0	2167
sl@0	2168	rc = sqlite3_bind_int64(s, 2, iEndBlockid);
sl@0	2169	if( rc!=SQLITE_OK ) return rc;
sl@0	2170
sl@0	2171	return sql_single_step(s);
sl@0	2172	}
sl@0	2173
sl@0	2174	/* Returns SQLITE_ROW with *pidx set to the maximum segment idx found
sl@0	2175	** at iLevel. Returns SQLITE_DONE if there are no segments at
sl@0	2176	** iLevel. Otherwise returns an error.
sl@0	2177	*/
sl@0	2178	static int segdir_max_index(fulltext_vtab v, int iLevel, int pidx){
sl@0	2179	sqlite3_stmt *s;
sl@0	2180	int rc = sql_get_statement(v, SEGDIR_MAX_INDEX_STMT, &s);
sl@0	2181	if( rc!=SQLITE_OK ) return rc;
sl@0	2182
sl@0	2183	rc = sqlite3_bind_int(s, 1, iLevel);
sl@0	2184	if( rc!=SQLITE_OK ) return rc;
sl@0	2185
sl@0	2186	rc = sqlite3_step(s);
sl@0	2187	/* Should always get at least one row due to how max() works. */
sl@0	2188	if( rc==SQLITE_DONE ) return SQLITE_DONE;
sl@0	2189	if( rc!=SQLITE_ROW ) return rc;
sl@0	2190
sl@0	2191	/* NULL means that there were no inputs to max(). */
sl@0	2192	if( SQLITE_NULL==sqlite3_column_type(s, 0) ){
sl@0	2193	rc = sqlite3_step(s);
sl@0	2194	if( rc==SQLITE_ROW ) return SQLITE_ERROR;
sl@0	2195	return rc;
sl@0	2196	}
sl@0	2197
sl@0	2198	*pidx = sqlite3_column_int(s, 0);
sl@0	2199
sl@0	2200	/* We expect only one row. We must execute another sqlite3_step()
sl@0	2201	* to complete the iteration; otherwise the table will remain locked. */
sl@0	2202	rc = sqlite3_step(s);
sl@0	2203	if( rc==SQLITE_ROW ) return SQLITE_ERROR;
sl@0	2204	if( rc!=SQLITE_DONE ) return rc;
sl@0	2205	return SQLITE_ROW;
sl@0	2206	}
sl@0	2207
sl@0	2208	/* insert into %_segdir values (
sl@0	2209	** [iLevel], [idx],
sl@0	2210	** [iStartBlockid], [iLeavesEndBlockid], [iEndBlockid],
sl@0	2211	** [pRootData]
sl@0	2212	** )
sl@0	2213	*/
sl@0	2214	static int segdir_set(fulltext_vtab *v, int iLevel, int idx,
sl@0	2215	sqlite_int64 iStartBlockid,
sl@0	2216	sqlite_int64 iLeavesEndBlockid,
sl@0	2217	sqlite_int64 iEndBlockid,
sl@0	2218	const char *pRootData, int nRootData){
sl@0	2219	sqlite3_stmt *s;
sl@0	2220	int rc = sql_get_statement(v, SEGDIR_SET_STMT, &s);
sl@0	2221	if( rc!=SQLITE_OK ) return rc;
sl@0	2222
sl@0	2223	rc = sqlite3_bind_int(s, 1, iLevel);
sl@0	2224	if( rc!=SQLITE_OK ) return rc;
sl@0	2225
sl@0	2226	rc = sqlite3_bind_int(s, 2, idx);
sl@0	2227	if( rc!=SQLITE_OK ) return rc;
sl@0	2228
sl@0	2229	rc = sqlite3_bind_int64(s, 3, iStartBlockid);
sl@0	2230	if( rc!=SQLITE_OK ) return rc;
sl@0	2231
sl@0	2232	rc = sqlite3_bind_int64(s, 4, iLeavesEndBlockid);
sl@0	2233	if( rc!=SQLITE_OK ) return rc;
sl@0	2234
sl@0	2235	rc = sqlite3_bind_int64(s, 5, iEndBlockid);
sl@0	2236	if( rc!=SQLITE_OK ) return rc;
sl@0	2237
sl@0	2238	rc = sqlite3_bind_blob(s, 6, pRootData, nRootData, SQLITE_STATIC);
sl@0	2239	if( rc!=SQLITE_OK ) return rc;
sl@0	2240
sl@0	2241	return sql_single_step(s);
sl@0	2242	}
sl@0	2243
sl@0	2244	/* Queries %_segdir for the block span of the segments in level
sl@0	2245	** iLevel. Returns SQLITE_DONE if there are no blocks for iLevel,
sl@0	2246	** SQLITE_ROW if there are blocks, else an error.
sl@0	2247	*/
sl@0	2248	static int segdir_span(fulltext_vtab *v, int iLevel,
sl@0	2249	sqlite_int64 *piStartBlockid,
sl@0	2250	sqlite_int64 *piEndBlockid){
sl@0	2251	sqlite3_stmt *s;
sl@0	2252	int rc = sql_get_statement(v, SEGDIR_SPAN_STMT, &s);
sl@0	2253	if( rc!=SQLITE_OK ) return rc;
sl@0	2254
sl@0	2255	rc = sqlite3_bind_int(s, 1, iLevel);
sl@0	2256	if( rc!=SQLITE_OK ) return rc;
sl@0	2257
sl@0	2258	rc = sqlite3_step(s);
sl@0	2259	if( rc==SQLITE_DONE ) return SQLITE_DONE; /* Should never happen */
sl@0	2260	if( rc!=SQLITE_ROW ) return rc;
sl@0	2261
sl@0	2262	/* This happens if all segments at this level are entirely inline. */
sl@0	2263	if( SQLITE_NULL==sqlite3_column_type(s, 0) ){
sl@0	2264	/* We expect only one row. We must execute another sqlite3_step()
sl@0	2265	* to complete the iteration; otherwise the table will remain locked. */
sl@0	2266	int rc2 = sqlite3_step(s);
sl@0	2267	if( rc2==SQLITE_ROW ) return SQLITE_ERROR;
sl@0	2268	return rc2;
sl@0	2269	}
sl@0	2270
sl@0	2271	*piStartBlockid = sqlite3_column_int64(s, 0);
sl@0	2272	*piEndBlockid = sqlite3_column_int64(s, 1);
sl@0	2273
sl@0	2274	/* We expect only one row. We must execute another sqlite3_step()
sl@0	2275	* to complete the iteration; otherwise the table will remain locked. */
sl@0	2276	rc = sqlite3_step(s);
sl@0	2277	if( rc==SQLITE_ROW ) return SQLITE_ERROR;
sl@0	2278	if( rc!=SQLITE_DONE ) return rc;
sl@0	2279	return SQLITE_ROW;
sl@0	2280	}
sl@0	2281
sl@0	2282	/* Delete the segment blocks and segment directory records for all
sl@0	2283	** segments at iLevel.
sl@0	2284	*/
sl@0	2285	static int segdir_delete(fulltext_vtab *v, int iLevel){
sl@0	2286	sqlite3_stmt *s;
sl@0	2287	sqlite_int64 iStartBlockid, iEndBlockid;
sl@0	2288	int rc = segdir_span(v, iLevel, &iStartBlockid, &iEndBlockid);
sl@0	2289	if( rc!=SQLITE_ROW && rc!=SQLITE_DONE ) return rc;
sl@0	2290
sl@0	2291	if( rc==SQLITE_ROW ){
sl@0	2292	rc = block_delete(v, iStartBlockid, iEndBlockid);
sl@0	2293	if( rc!=SQLITE_OK ) return rc;
sl@0	2294	}
sl@0	2295
sl@0	2296	/* Delete the segment directory itself. */
sl@0	2297	rc = sql_get_statement(v, SEGDIR_DELETE_STMT, &s);
sl@0	2298	if( rc!=SQLITE_OK ) return rc;
sl@0	2299
sl@0	2300	rc = sqlite3_bind_int64(s, 1, iLevel);
sl@0	2301	if( rc!=SQLITE_OK ) return rc;
sl@0	2302
sl@0	2303	return sql_single_step(s);
sl@0	2304	}
sl@0	2305
sl@0	2306	/* Delete entire fts index, SQLITE_OK on success, relevant error on
sl@0	2307	** failure.
sl@0	2308	*/
sl@0	2309	static int segdir_delete_all(fulltext_vtab *v){
sl@0	2310	sqlite3_stmt *s;
sl@0	2311	int rc = sql_get_statement(v, SEGDIR_DELETE_ALL_STMT, &s);
sl@0	2312	if( rc!=SQLITE_OK ) return rc;
sl@0	2313
sl@0	2314	rc = sql_single_step(s);
sl@0	2315	if( rc!=SQLITE_OK ) return rc;
sl@0	2316
sl@0	2317	rc = sql_get_statement(v, BLOCK_DELETE_ALL_STMT, &s);
sl@0	2318	if( rc!=SQLITE_OK ) return rc;
sl@0	2319
sl@0	2320	return sql_single_step(s);
sl@0	2321	}
sl@0	2322
sl@0	2323	/* Returns SQLITE_OK with *pnSegments set to the number of entries in
sl@0	2324	** %_segdir and *piMaxLevel set to the highest level which has a
sl@0	2325	** segment. Otherwise returns the SQLite error which caused failure.
sl@0	2326	*/
sl@0	2327	static int segdir_count(fulltext_vtab v, int pnSegments, int *piMaxLevel){
sl@0	2328	sqlite3_stmt *s;
sl@0	2329	int rc = sql_get_statement(v, SEGDIR_COUNT_STMT, &s);
sl@0	2330	if( rc!=SQLITE_OK ) return rc;
sl@0	2331
sl@0	2332	rc = sqlite3_step(s);
sl@0	2333	/* TODO(shess): This case should not be possible? Should stronger
sl@0	2334	** measures be taken if it happens?
sl@0	2335	*/
sl@0	2336	if( rc==SQLITE_DONE ){
sl@0	2337	*pnSegments = 0;
sl@0	2338	*piMaxLevel = 0;
sl@0	2339	return SQLITE_OK;
sl@0	2340	}
sl@0	2341	if( rc!=SQLITE_ROW ) return rc;
sl@0	2342
sl@0	2343	*pnSegments = sqlite3_column_int(s, 0);
sl@0	2344	*piMaxLevel = sqlite3_column_int(s, 1);
sl@0	2345
sl@0	2346	/* We expect only one row. We must execute another sqlite3_step()
sl@0	2347	* to complete the iteration; otherwise the table will remain locked. */
sl@0	2348	rc = sqlite3_step(s);
sl@0	2349	if( rc==SQLITE_DONE ) return SQLITE_OK;
sl@0	2350	if( rc==SQLITE_ROW ) return SQLITE_ERROR;
sl@0	2351	return rc;
sl@0	2352	}
sl@0	2353
sl@0	2354	/* TODO(shess) clearPendingTerms() is far down the file because
sl@0	2355	** writeZeroSegment() is far down the file because LeafWriter is far
sl@0	2356	** down the file. Consider refactoring the code to move the non-vtab
sl@0	2357	** code above the vtab code so that we don't need this forward
sl@0	2358	** reference.
sl@0	2359	*/
sl@0	2360	static int clearPendingTerms(fulltext_vtab *v);
sl@0	2361
sl@0	2362	/*
sl@0	2363	** Free the memory used to contain a fulltext_vtab structure.
sl@0	2364	*/
sl@0	2365	static void fulltext_vtab_destroy(fulltext_vtab *v){
sl@0	2366	int iStmt, i;
sl@0	2367
sl@0	2368	TRACE(("FTS2 Destroy %p\n", v));
sl@0	2369	for( iStmt=0; iStmt<MAX_STMT; iStmt++ ){
sl@0	2370	if( v->pFulltextStatements[iStmt]!=NULL ){
sl@0	2371	sqlite3_finalize(v->pFulltextStatements[iStmt]);
sl@0	2372	v->pFulltextStatements[iStmt] = NULL;
sl@0	2373	}
sl@0	2374	}
sl@0	2375
sl@0	2376	for( i=0; i<MERGE_COUNT; i++ ){
sl@0	2377	if( v->pLeafSelectStmts[i]!=NULL ){
sl@0	2378	sqlite3_finalize(v->pLeafSelectStmts[i]);
sl@0	2379	v->pLeafSelectStmts[i] = NULL;
sl@0	2380	}
sl@0	2381	}
sl@0	2382
sl@0	2383	if( v->pTokenizer!=NULL ){
sl@0	2384	v->pTokenizer->pModule->xDestroy(v->pTokenizer);
sl@0	2385	v->pTokenizer = NULL;
sl@0	2386	}
sl@0	2387
sl@0	2388	clearPendingTerms(v);
sl@0	2389
sl@0	2390	sqlite3_free(v->azColumn);
sl@0	2391	for(i = 0; i < v->nColumn; ++i) {
sl@0	2392	sqlite3_free(v->azContentColumn[i]);
sl@0	2393	}
sl@0	2394	sqlite3_free(v->azContentColumn);
sl@0	2395	sqlite3_free(v);
sl@0	2396	}
sl@0	2397
sl@0	2398	/*
sl@0	2399	** Token types for parsing the arguments to xConnect or xCreate.
sl@0	2400	*/
sl@0	2401	#define TOKEN_EOF 0 /* End of file */
sl@0	2402	#define TOKEN_SPACE 1 /* Any kind of whitespace */
sl@0	2403	#define TOKEN_ID 2 /* An identifier */
sl@0	2404	#define TOKEN_STRING 3 /* A string literal */
sl@0	2405	#define TOKEN_PUNCT 4 /* A single punctuation character */
sl@0	2406
sl@0	2407	/*
sl@0	2408	** If X is a character that can be used in an identifier then
sl@0	2409	** IdChar(X) will be true. Otherwise it is false.
sl@0	2410	**
sl@0	2411	** For ASCII, any character with the high-order bit set is
sl@0	2412	** allowed in an identifier. For 7-bit characters,
sl@0	2413	** sqlite3IsIdChar[X] must be 1.
sl@0	2414	**
sl@0	2415	** Ticket #1066. the SQL standard does not allow '$' in the
sl@0	2416	** middle of identfiers. But many SQL implementations do.
sl@0	2417	** SQLite will allow '$' in identifiers for compatibility.
sl@0	2418	** But the feature is undocumented.
sl@0	2419	*/
sl@0	2420	static const char isIdChar[] = {
sl@0	2421	/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
sl@0	2422	0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
sl@0	2423	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
sl@0	2424	0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
sl@0	2425	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
sl@0	2426	0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
sl@0	2427	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
sl@0	2428	};
sl@0	2429	#define IdChar(C) (((c=C)&0x80)!=0 \|\| (c>0x1f && isIdChar[c-0x20]))
sl@0	2430
sl@0	2431
sl@0	2432	/*
sl@0	2433	** Return the length of the token that begins at z[0].
sl@0	2434	** Store the token type in *tokenType before returning.
sl@0	2435	*/
sl@0	2436	static int getToken(const char z, int tokenType){
sl@0	2437	int i, c;
sl@0	2438	switch( *z ){
sl@0	2439	case 0: {
sl@0	2440	*tokenType = TOKEN_EOF;
sl@0	2441	return 0;
sl@0	2442	}
sl@0	2443	case ' ': case '\t': case '\n': case '\f': case '\r': {
sl@0	2444	for(i=1; safe_isspace(z[i]); i++){}
sl@0	2445	*tokenType = TOKEN_SPACE;
sl@0	2446	return i;
sl@0	2447	}
sl@0	2448	case '`':
sl@0	2449	case '\'':
sl@0	2450	case '"': {
sl@0	2451	int delim = z[0];
sl@0	2452	for(i=1; (c=z[i])!=0; i++){
sl@0	2453	if( c==delim ){
sl@0	2454	if( z[i+1]==delim ){
sl@0	2455	i++;
sl@0	2456	}else{
sl@0	2457	break;
sl@0	2458	}
sl@0	2459	}
sl@0	2460	}
sl@0	2461	*tokenType = TOKEN_STRING;
sl@0	2462	return i + (c!=0);
sl@0	2463	}
sl@0	2464	case '[': {
sl@0	2465	for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){}
sl@0	2466	*tokenType = TOKEN_ID;
sl@0	2467	return i;
sl@0	2468	}
sl@0	2469	default: {
sl@0	2470	if( !IdChar(*z) ){
sl@0	2471	break;
sl@0	2472	}
sl@0	2473	for(i=1; IdChar(z[i]); i++){}
sl@0	2474	*tokenType = TOKEN_ID;
sl@0	2475	return i;
sl@0	2476	}
sl@0	2477	}
sl@0	2478	*tokenType = TOKEN_PUNCT;
sl@0	2479	return 1;
sl@0	2480	}
sl@0	2481
sl@0	2482	/*
sl@0	2483	** A token extracted from a string is an instance of the following
sl@0	2484	** structure.
sl@0	2485	*/
sl@0	2486	typedef struct Token {
sl@0	2487	const char z; / Pointer to token text. Not '\000' terminated */
sl@0	2488	short int n; /* Length of the token text in bytes. */
sl@0	2489	} Token;
sl@0	2490
sl@0	2491	/*
sl@0	2492	** Given a input string (which is really one of the argv[] parameters
sl@0	2493	** passed into xConnect or xCreate) split the string up into tokens.
sl@0	2494	** Return an array of pointers to '\000' terminated strings, one string
sl@0	2495	** for each non-whitespace token.
sl@0	2496	**
sl@0	2497	** The returned array is terminated by a single NULL pointer.
sl@0	2498	**
sl@0	2499	** Space to hold the returned array is obtained from a single
sl@0	2500	** malloc and should be freed by passing the return value to free().
sl@0	2501	** The individual strings within the token list are all a part of
sl@0	2502	** the single memory allocation and will all be freed at once.
sl@0	2503	*/
sl@0	2504	static char *tokenizeString(const char z, int *pnToken){
sl@0	2505	int nToken = 0;
sl@0	2506	Token aToken = sqlite3_malloc( strlen(z) sizeof(aToken[0]) );
sl@0	2507	int n = 1;
sl@0	2508	int e, i;
sl@0	2509	int totalSize = 0;
sl@0	2510	char **azToken;
sl@0	2511	char *zCopy;
sl@0	2512	while( n>0 ){
sl@0	2513	n = getToken(z, &e);
sl@0	2514	if( e!=TOKEN_SPACE ){
sl@0	2515	aToken[nToken].z = z;
sl@0	2516	aToken[nToken].n = n;
sl@0	2517	nToken++;
sl@0	2518	totalSize += n+1;
sl@0	2519	}
sl@0	2520	z += n;
sl@0	2521	}
sl@0	2522	azToken = (char*)sqlite3_malloc( nTokensizeof(char*) + totalSize );
sl@0	2523	zCopy = (char*)&azToken[nToken];
sl@0	2524	nToken--;
sl@0	2525	for(i=0; i<nToken; i++){
sl@0	2526	azToken[i] = zCopy;
sl@0	2527	n = aToken[i].n;
sl@0	2528	memcpy(zCopy, aToken[i].z, n);
sl@0	2529	zCopy[n] = 0;
sl@0	2530	zCopy += n+1;
sl@0	2531	}
sl@0	2532	azToken[nToken] = 0;
sl@0	2533	sqlite3_free(aToken);
sl@0	2534	*pnToken = nToken;
sl@0	2535	return azToken;
sl@0	2536	}
sl@0	2537
sl@0	2538	/*
sl@0	2539	** Convert an SQL-style quoted string into a normal string by removing
sl@0	2540	** the quote characters. The conversion is done in-place. If the
sl@0	2541	** input does not begin with a quote character, then this routine
sl@0	2542	** is a no-op.
sl@0	2543	**
sl@0	2544	** Examples:
sl@0	2545	**
sl@0	2546	** "abc" becomes abc
sl@0	2547	** 'xyz' becomes xyz
sl@0	2548	** [pqr] becomes pqr
sl@0	2549	** `mno` becomes mno
sl@0	2550	*/
sl@0	2551	static void dequoteString(char *z){
sl@0	2552	int quote;
sl@0	2553	int i, j;
sl@0	2554	if( z==0 ) return;
sl@0	2555	quote = z[0];
sl@0	2556	switch( quote ){
sl@0	2557	case '\'': break;
sl@0	2558	case '"': break;
sl@0	2559	case '`': break; /* For MySQL compatibility */
sl@0	2560	case '[': quote = ']'; break; /* For MS SqlServer compatibility */
sl@0	2561	default: return;
sl@0	2562	}
sl@0	2563	for(i=1, j=0; z[i]; i++){
sl@0	2564	if( z[i]==quote ){
sl@0	2565	if( z[i+1]==quote ){
sl@0	2566	z[j++] = quote;
sl@0	2567	i++;
sl@0	2568	}else{
sl@0	2569	z[j++] = 0;
sl@0	2570	break;
sl@0	2571	}
sl@0	2572	}else{
sl@0	2573	z[j++] = z[i];
sl@0	2574	}
sl@0	2575	}
sl@0	2576	}
sl@0	2577
sl@0	2578	/*
sl@0	2579	** The input azIn is a NULL-terminated list of tokens. Remove the first
sl@0	2580	** token and all punctuation tokens. Remove the quotes from
sl@0	2581	** around string literal tokens.
sl@0	2582	**
sl@0	2583	** Example:
sl@0	2584	**
sl@0	2585	** input: tokenize chinese ( 'simplifed' , 'mixed' )
sl@0	2586	** output: chinese simplifed mixed
sl@0	2587	**
sl@0	2588	** Another example:
sl@0	2589	**
sl@0	2590	** input: delimiters ( '[' , ']' , '...' )
sl@0	2591	** output: [ ] ...
sl@0	2592	*/
sl@0	2593	static void tokenListToIdList(char **azIn){
sl@0	2594	int i, j;
sl@0	2595	if( azIn ){
sl@0	2596	for(i=0, j=-1; azIn[i]; i++){
sl@0	2597	if( safe_isalnum(azIn[i][0]) \|\| azIn[i][1] ){
sl@0	2598	dequoteString(azIn[i]);
sl@0	2599	if( j>=0 ){
sl@0	2600	azIn[j] = azIn[i];
sl@0	2601	}
sl@0	2602	j++;
sl@0	2603	}
sl@0	2604	}
sl@0	2605	azIn[j] = 0;
sl@0	2606	}
sl@0	2607	}
sl@0	2608
sl@0	2609
sl@0	2610	/*
sl@0	2611	** Find the first alphanumeric token in the string zIn. Null-terminate
sl@0	2612	** this token. Remove any quotation marks. And return a pointer to
sl@0	2613	** the result.
sl@0	2614	*/
sl@0	2615	static char firstToken(char zIn, char **pzTail){
sl@0	2616	int n, ttype;
sl@0	2617	while(1){
sl@0	2618	n = getToken(zIn, &ttype);
sl@0	2619	if( ttype==TOKEN_SPACE ){
sl@0	2620	zIn += n;
sl@0	2621	}else if( ttype==TOKEN_EOF ){
sl@0	2622	*pzTail = zIn;
sl@0	2623	return 0;
sl@0	2624	}else{
sl@0	2625	zIn[n] = 0;
sl@0	2626	*pzTail = &zIn[1];
sl@0	2627	dequoteString(zIn);
sl@0	2628	return zIn;
sl@0	2629	}
sl@0	2630	}
sl@0	2631	/NOTREACHED/
sl@0	2632	}
sl@0	2633
sl@0	2634	/* Return true if...
sl@0	2635	**
sl@0	2636	** * s begins with the string t, ignoring case
sl@0	2637	** * s is longer than t
sl@0	2638	** * The first character of s beyond t is not a alphanumeric
sl@0	2639	**
sl@0	2640	** Ignore leading space in *s.
sl@0	2641	**
sl@0	2642	** To put it another way, return true if the first token of
sl@0	2643	** s[] is t[].
sl@0	2644	*/
sl@0	2645	static int startsWith(const char s, const char t){
sl@0	2646	while( safe_isspace(*s) ){ s++; }
sl@0	2647	while( *t ){
sl@0	2648	if( safe_tolower(s++)!=safe_tolower(t++) ) return 0;
sl@0	2649	}
sl@0	2650	return s!='_' && !safe_isalnum(s);
sl@0	2651	}
sl@0	2652
sl@0	2653	/*
sl@0	2654	** An instance of this structure defines the "spec" of a
sl@0	2655	** full text index. This structure is populated by parseSpec
sl@0	2656	** and use by fulltextConnect and fulltextCreate.
sl@0	2657	*/
sl@0	2658	typedef struct TableSpec {
sl@0	2659	const char zDb; / Logical database name */
sl@0	2660	const char zName; / Name of the full-text index */
sl@0	2661	int nColumn; /* Number of columns to be indexed */
sl@0	2662	char *azColumn; / Original names of columns to be indexed */
sl@0	2663	char *azContentColumn; / Column names for %_content */
sl@0	2664	char *azTokenizer; / Name of tokenizer and its arguments */
sl@0	2665	} TableSpec;
sl@0	2666
sl@0	2667	/*
sl@0	2668	** Reclaim all of the memory used by a TableSpec
sl@0	2669	*/
sl@0	2670	static void clearTableSpec(TableSpec *p) {
sl@0	2671	sqlite3_free(p->azColumn);
sl@0	2672	sqlite3_free(p->azContentColumn);
sl@0	2673	sqlite3_free(p->azTokenizer);
sl@0	2674	}
sl@0	2675
sl@0	2676	/* Parse a CREATE VIRTUAL TABLE statement, which looks like this:
sl@0	2677	*
sl@0	2678	* CREATE VIRTUAL TABLE email
sl@0	2679	* USING fts2(subject, body, tokenize mytokenizer(myarg))
sl@0	2680	*
sl@0	2681	* We return parsed information in a TableSpec structure.
sl@0	2682	*
sl@0	2683	*/
sl@0	2684	static int parseSpec(TableSpec pSpec, int argc, const char const*argv,
sl@0	2685	char**pzErr){
sl@0	2686	int i, n;
sl@0	2687	char z, zDummy;
sl@0	2688	char **azArg;
sl@0	2689	const char zTokenizer = 0; / argv[] entry describing the tokenizer */
sl@0	2690
sl@0	2691	assert( argc>=3 );
sl@0	2692	/* Current interface:
sl@0	2693	** argv[0] - module name
sl@0	2694	** argv[1] - database name
sl@0	2695	** argv[2] - table name
sl@0	2696	** argv[3..] - columns, optionally followed by tokenizer specification
sl@0	2697	** and snippet delimiters specification.
sl@0	2698	*/
sl@0	2699
sl@0	2700	/* Make a copy of the complete argv[][] array in a single allocation.
sl@0	2701	** The argv[][] array is read-only and transient. We can write to the
sl@0	2702	** copy in order to modify things and the copy is persistent.
sl@0	2703	*/
sl@0	2704	CLEAR(pSpec);
sl@0	2705	for(i=n=0; i<argc; i++){
sl@0	2706	n += strlen(argv[i]) + 1;
sl@0	2707	}
sl@0	2708	azArg = sqlite3_malloc( sizeof(char)argc + n );
sl@0	2709	if( azArg==0 ){
sl@0	2710	return SQLITE_NOMEM;
sl@0	2711	}
sl@0	2712	z = (char*)&azArg[argc];
sl@0	2713	for(i=0; i<argc; i++){
sl@0	2714	azArg[i] = z;
sl@0	2715	strcpy(z, argv[i]);
sl@0	2716	z += strlen(z)+1;
sl@0	2717	}
sl@0	2718
sl@0	2719	/* Identify the column names and the tokenizer and delimiter arguments
sl@0	2720	** in the argv[][] array.
sl@0	2721	*/
sl@0	2722	pSpec->zDb = azArg[1];
sl@0	2723	pSpec->zName = azArg[2];
sl@0	2724	pSpec->nColumn = 0;
sl@0	2725	pSpec->azColumn = azArg;
sl@0	2726	zTokenizer = "tokenize simple";
sl@0	2727	for(i=3; i<argc; ++i){
sl@0	2728	if( startsWith(azArg[i],"tokenize") ){
sl@0	2729	zTokenizer = azArg[i];
sl@0	2730	}else{
sl@0	2731	z = azArg[pSpec->nColumn] = firstToken(azArg[i], &zDummy);
sl@0	2732	pSpec->nColumn++;
sl@0	2733	}
sl@0	2734	}
sl@0	2735	if( pSpec->nColumn==0 ){
sl@0	2736	azArg[0] = "content";
sl@0	2737	pSpec->nColumn = 1;
sl@0	2738	}
sl@0	2739
sl@0	2740	/*
sl@0	2741	** Construct the list of content column names.
sl@0	2742	**
sl@0	2743	** Each content column name will be of the form cNNAAAA
sl@0	2744	** where NN is the column number and AAAA is the sanitized
sl@0	2745	** column name. "sanitized" means that special characters are
sl@0	2746	** converted to "_". The cNN prefix guarantees that all column
sl@0	2747	** names are unique.
sl@0	2748	**
sl@0	2749	** The AAAA suffix is not strictly necessary. It is included
sl@0	2750	** for the convenience of people who might examine the generated
sl@0	2751	** %_content table and wonder what the columns are used for.
sl@0	2752	*/
sl@0	2753	pSpec->azContentColumn = sqlite3_malloc( pSpec->nColumn * sizeof(char *) );
sl@0	2754	if( pSpec->azContentColumn==0 ){
sl@0	2755	clearTableSpec(pSpec);
sl@0	2756	return SQLITE_NOMEM;
sl@0	2757	}
sl@0	2758	for(i=0; i<pSpec->nColumn; i++){
sl@0	2759	char *p;
sl@0	2760	pSpec->azContentColumn[i] = sqlite3_mprintf("c%d%s", i, azArg[i]);
sl@0	2761	for (p = pSpec->azContentColumn[i]; *p ; ++p) {
sl@0	2762	if( !safe_isalnum(p) ) p = '_';
sl@0	2763	}
sl@0	2764	}
sl@0	2765
sl@0	2766	/*
sl@0	2767	** Parse the tokenizer specification string.
sl@0	2768	*/
sl@0	2769	pSpec->azTokenizer = tokenizeString(zTokenizer, &n);
sl@0	2770	tokenListToIdList(pSpec->azTokenizer);
sl@0	2771
sl@0	2772	return SQLITE_OK;
sl@0	2773	}
sl@0	2774
sl@0	2775	/*
sl@0	2776	** Generate a CREATE TABLE statement that describes the schema of
sl@0	2777	** the virtual table. Return a pointer to this schema string.
sl@0	2778	**
sl@0	2779	** Space is obtained from sqlite3_mprintf() and should be freed
sl@0	2780	** using sqlite3_free().
sl@0	2781	*/
sl@0	2782	static char *fulltextSchema(
sl@0	2783	int nColumn, /* Number of columns */
sl@0	2784	const char const azColumn, /* List of columns */
sl@0	2785	const char zTableName / Name of the table */
sl@0	2786	){
sl@0	2787	int i;
sl@0	2788	char zSchema, zNext;
sl@0	2789	const char *zSep = "(";
sl@0	2790	zSchema = sqlite3_mprintf("CREATE TABLE x");
sl@0	2791	for(i=0; i<nColumn; i++){
sl@0	2792	zNext = sqlite3_mprintf("%s%s%Q", zSchema, zSep, azColumn[i]);
sl@0	2793	sqlite3_free(zSchema);
sl@0	2794	zSchema = zNext;
sl@0	2795	zSep = ",";
sl@0	2796	}
sl@0	2797	zNext = sqlite3_mprintf("%s,%Q)", zSchema, zTableName);
sl@0	2798	sqlite3_free(zSchema);
sl@0	2799	return zNext;
sl@0	2800	}
sl@0	2801
sl@0	2802	/*
sl@0	2803	** Build a new sqlite3_vtab structure that will describe the
sl@0	2804	** fulltext index defined by spec.
sl@0	2805	*/
sl@0	2806	static int constructVtab(
sl@0	2807	sqlite3 db, / The SQLite database connection */
sl@0	2808	fts2Hash pHash, / Hash table containing tokenizers */
sl@0	2809	TableSpec spec, / Parsed spec information from parseSpec() */
sl@0	2810	sqlite3_vtab *ppVTab, / Write the resulting vtab structure here */
sl@0	2811	char *pzErr / Write any error message here */
sl@0	2812	){
sl@0	2813	int rc;
sl@0	2814	int n;
sl@0	2815	fulltext_vtab *v = 0;
sl@0	2816	const sqlite3_tokenizer_module *m = NULL;
sl@0	2817	char *schema;
sl@0	2818
sl@0	2819	char const zTok; / Name of tokenizer to use for this fts table */
sl@0	2820	int nTok; /* Length of zTok, including nul terminator */
sl@0	2821
sl@0	2822	v = (fulltext_vtab *) sqlite3_malloc(sizeof(fulltext_vtab));
sl@0	2823	if( v==0 ) return SQLITE_NOMEM;
sl@0	2824	CLEAR(v);
sl@0	2825	/* sqlite will initialize v->base */
sl@0	2826	v->db = db;
sl@0	2827	v->zDb = spec->zDb; /* Freed when azColumn is freed */
sl@0	2828	v->zName = spec->zName; /* Freed when azColumn is freed */
sl@0	2829	v->nColumn = spec->nColumn;
sl@0	2830	v->azContentColumn = spec->azContentColumn;
sl@0	2831	spec->azContentColumn = 0;
sl@0	2832	v->azColumn = spec->azColumn;
sl@0	2833	spec->azColumn = 0;
sl@0	2834
sl@0	2835	if( spec->azTokenizer==0 ){
sl@0	2836	return SQLITE_NOMEM;
sl@0	2837	}
sl@0	2838
sl@0	2839	zTok = spec->azTokenizer[0];
sl@0	2840	if( !zTok ){
sl@0	2841	zTok = "simple";
sl@0	2842	}
sl@0	2843	nTok = strlen(zTok)+1;
sl@0	2844
sl@0	2845	m = (sqlite3_tokenizer_module *)sqlite3Fts2HashFind(pHash, zTok, nTok);
sl@0	2846	if( !m ){
sl@0	2847	*pzErr = sqlite3_mprintf("unknown tokenizer: %s", spec->azTokenizer[0]);
sl@0	2848	rc = SQLITE_ERROR;
sl@0	2849	goto err;
sl@0	2850	}
sl@0	2851
sl@0	2852	for(n=0; spec->azTokenizer[n]; n++){}
sl@0	2853	if( n ){
sl@0	2854	rc = m->xCreate(n-1, (const charconst)&spec->azTokenizer[1],
sl@0	2855	&v->pTokenizer);
sl@0	2856	}else{
sl@0	2857	rc = m->xCreate(0, 0, &v->pTokenizer);
sl@0	2858	}
sl@0	2859	if( rc!=SQLITE_OK ) goto err;
sl@0	2860	v->pTokenizer->pModule = m;
sl@0	2861
sl@0	2862	/* TODO: verify the existence of backing tables foo_content, foo_term */
sl@0	2863
sl@0	2864	schema = fulltextSchema(v->nColumn, (const charconst)v->azColumn,
sl@0	2865	spec->zName);
sl@0	2866	rc = sqlite3_declare_vtab(db, schema);
sl@0	2867	sqlite3_free(schema);
sl@0	2868	if( rc!=SQLITE_OK ) goto err;
sl@0	2869
sl@0	2870	memset(v->pFulltextStatements, 0, sizeof(v->pFulltextStatements));
sl@0	2871
sl@0	2872	/* Indicate that the buffer is not live. */
sl@0	2873	v->nPendingData = -1;
sl@0	2874
sl@0	2875	*ppVTab = &v->base;
sl@0	2876	TRACE(("FTS2 Connect %p\n", v));
sl@0	2877
sl@0	2878	return rc;
sl@0	2879
sl@0	2880	err:
sl@0	2881	fulltext_vtab_destroy(v);
sl@0	2882	return rc;
sl@0	2883	}
sl@0	2884
sl@0	2885	static int fulltextConnect(
sl@0	2886	sqlite3 *db,
sl@0	2887	void *pAux,
sl@0	2888	int argc, const char constargv,
sl@0	2889	sqlite3_vtab **ppVTab,
sl@0	2890	char **pzErr
sl@0	2891	){
sl@0	2892	TableSpec spec;
sl@0	2893	int rc = parseSpec(&spec, argc, argv, pzErr);
sl@0	2894	if( rc!=SQLITE_OK ) return rc;
sl@0	2895
sl@0	2896	rc = constructVtab(db, (fts2Hash *)pAux, &spec, ppVTab, pzErr);
sl@0	2897	clearTableSpec(&spec);
sl@0	2898	return rc;
sl@0	2899	}
sl@0	2900
sl@0	2901	/* The %_content table holds the text of each document, with
sl@0	2902	** the rowid used as the docid.
sl@0	2903	*/
sl@0	2904	/* TODO(shess) This comment needs elaboration to match the updated
sl@0	2905	** code. Work it into the top-of-file comment at that time.
sl@0	2906	*/
sl@0	2907	static int fulltextCreate(sqlite3 db, void pAux,
sl@0	2908	int argc, const char * const *argv,
sl@0	2909	sqlite3_vtab ppVTab, char pzErr){
sl@0	2910	int rc;
sl@0	2911	TableSpec spec;
sl@0	2912	StringBuffer schema;
sl@0	2913	TRACE(("FTS2 Create\n"));
sl@0	2914
sl@0	2915	rc = parseSpec(&spec, argc, argv, pzErr);
sl@0	2916	if( rc!=SQLITE_OK ) return rc;
sl@0	2917
sl@0	2918	initStringBuffer(&schema);
sl@0	2919	append(&schema, "CREATE TABLE %_content(");
sl@0	2920	appendList(&schema, spec.nColumn, spec.azContentColumn);
sl@0	2921	append(&schema, ")");
sl@0	2922	rc = sql_exec(db, spec.zDb, spec.zName, stringBufferData(&schema));
sl@0	2923	stringBufferDestroy(&schema);
sl@0	2924	if( rc!=SQLITE_OK ) goto out;
sl@0	2925
sl@0	2926	rc = sql_exec(db, spec.zDb, spec.zName,
sl@0	2927	"create table %_segments(block blob);");
sl@0	2928	if( rc!=SQLITE_OK ) goto out;
sl@0	2929
sl@0	2930	rc = sql_exec(db, spec.zDb, spec.zName,
sl@0	2931	"create table %_segdir("
sl@0	2932	" level integer,"
sl@0	2933	" idx integer,"
sl@0	2934	" start_block integer,"
sl@0	2935	" leaves_end_block integer,"
sl@0	2936	" end_block integer,"
sl@0	2937	" root blob,"
sl@0	2938	" primary key(level, idx)"
sl@0	2939	");");
sl@0	2940	if( rc!=SQLITE_OK ) goto out;
sl@0	2941
sl@0	2942	rc = constructVtab(db, (fts2Hash *)pAux, &spec, ppVTab, pzErr);
sl@0	2943
sl@0	2944	out:
sl@0	2945	clearTableSpec(&spec);
sl@0	2946	return rc;
sl@0	2947	}
sl@0	2948
sl@0	2949	/* Decide how to handle an SQL query. */
sl@0	2950	static int fulltextBestIndex(sqlite3_vtab pVTab, sqlite3_index_info pInfo){
sl@0	2951	int i;
sl@0	2952	TRACE(("FTS2 BestIndex\n"));
sl@0	2953
sl@0	2954	for(i=0; i<pInfo->nConstraint; ++i){
sl@0	2955	const struct sqlite3_index_constraint *pConstraint;
sl@0	2956	pConstraint = &pInfo->aConstraint[i];
sl@0	2957	if( pConstraint->usable ) {
sl@0	2958	if( pConstraint->iColumn==-1 &&
sl@0	2959	pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){
sl@0	2960	pInfo->idxNum = QUERY_ROWID; /* lookup by rowid */
sl@0	2961	TRACE(("FTS2 QUERY_ROWID\n"));
sl@0	2962	} else if( pConstraint->iColumn>=0 &&
sl@0	2963	pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH ){
sl@0	2964	/* full-text search */
sl@0	2965	pInfo->idxNum = QUERY_FULLTEXT + pConstraint->iColumn;
sl@0	2966	TRACE(("FTS2 QUERY_FULLTEXT %d\n", pConstraint->iColumn));
sl@0	2967	} else continue;
sl@0	2968
sl@0	2969	pInfo->aConstraintUsage[i].argvIndex = 1;
sl@0	2970	pInfo->aConstraintUsage[i].omit = 1;
sl@0	2971
sl@0	2972	/* An arbitrary value for now.
sl@0	2973	* TODO: Perhaps rowid matches should be considered cheaper than
sl@0	2974	* full-text searches. */
sl@0	2975	pInfo->estimatedCost = 1.0;
sl@0	2976
sl@0	2977	return SQLITE_OK;
sl@0	2978	}
sl@0	2979	}
sl@0	2980	pInfo->idxNum = QUERY_GENERIC;
sl@0	2981	return SQLITE_OK;
sl@0	2982	}
sl@0	2983
sl@0	2984	static int fulltextDisconnect(sqlite3_vtab *pVTab){
sl@0	2985	TRACE(("FTS2 Disconnect %p\n", pVTab));
sl@0	2986	fulltext_vtab_destroy((fulltext_vtab *)pVTab);
sl@0	2987	return SQLITE_OK;
sl@0	2988	}
sl@0	2989
sl@0	2990	static int fulltextDestroy(sqlite3_vtab *pVTab){
sl@0	2991	fulltext_vtab v = (fulltext_vtab )pVTab;
sl@0	2992	int rc;
sl@0	2993
sl@0	2994	TRACE(("FTS2 Destroy %p\n", pVTab));
sl@0	2995	rc = sql_exec(v->db, v->zDb, v->zName,
sl@0	2996	"drop table if exists %_content;"
sl@0	2997	"drop table if exists %_segments;"
sl@0	2998	"drop table if exists %_segdir;"
sl@0	2999	);
sl@0	3000	if( rc!=SQLITE_OK ) return rc;
sl@0	3001
sl@0	3002	fulltext_vtab_destroy((fulltext_vtab *)pVTab);
sl@0	3003	return SQLITE_OK;
sl@0	3004	}
sl@0	3005
sl@0	3006	static int fulltextOpen(sqlite3_vtab pVTab, sqlite3_vtab_cursor *ppCursor){
sl@0	3007	fulltext_cursor *c;
sl@0	3008
sl@0	3009	c = (fulltext_cursor *) sqlite3_malloc(sizeof(fulltext_cursor));
sl@0	3010	if( c ){
sl@0	3011	memset(c, 0, sizeof(fulltext_cursor));
sl@0	3012	/* sqlite will initialize c->base */
sl@0	3013	*ppCursor = &c->base;
sl@0	3014	TRACE(("FTS2 Open %p: %p\n", pVTab, c));
sl@0	3015	return SQLITE_OK;
sl@0	3016	}else{
sl@0	3017	return SQLITE_NOMEM;
sl@0	3018	}
sl@0	3019	}
sl@0	3020
sl@0	3021
sl@0	3022	/* Free all of the dynamically allocated memory held by *q
sl@0	3023	*/
sl@0	3024	static void queryClear(Query *q){
sl@0	3025	int i;
sl@0	3026	for(i = 0; i < q->nTerms; ++i){
sl@0	3027	sqlite3_free(q->pTerms[i].pTerm);
sl@0	3028	}
sl@0	3029	sqlite3_free(q->pTerms);
sl@0	3030	CLEAR(q);
sl@0	3031	}
sl@0	3032
sl@0	3033	/* Free all of the dynamically allocated memory held by the
sl@0	3034	** Snippet
sl@0	3035	*/
sl@0	3036	static void snippetClear(Snippet *p){
sl@0	3037	sqlite3_free(p->aMatch);
sl@0	3038	sqlite3_free(p->zOffset);
sl@0	3039	sqlite3_free(p->zSnippet);
sl@0	3040	CLEAR(p);
sl@0	3041	}
sl@0	3042	/*
sl@0	3043	** Append a single entry to the p->aMatch[] log.
sl@0	3044	*/
sl@0	3045	static void snippetAppendMatch(
sl@0	3046	Snippet p, / Append the entry to this snippet */
sl@0	3047	int iCol, int iTerm, /* The column and query term */
sl@0	3048	int iStart, int nByte /* Offset and size of the match */
sl@0	3049	){
sl@0	3050	int i;
sl@0	3051	struct snippetMatch *pMatch;
sl@0	3052	if( p->nMatch+1>=p->nAlloc ){
sl@0	3053	p->nAlloc = p->nAlloc*2 + 10;
sl@0	3054	p->aMatch = sqlite3_realloc(p->aMatch, p->nAlloc*sizeof(p->aMatch[0]) );
sl@0	3055	if( p->aMatch==0 ){
sl@0	3056	p->nMatch = 0;
sl@0	3057	p->nAlloc = 0;
sl@0	3058	return;
sl@0	3059	}
sl@0	3060	}
sl@0	3061	i = p->nMatch++;
sl@0	3062	pMatch = &p->aMatch[i];
sl@0	3063	pMatch->iCol = iCol;
sl@0	3064	pMatch->iTerm = iTerm;
sl@0	3065	pMatch->iStart = iStart;
sl@0	3066	pMatch->nByte = nByte;
sl@0	3067	}
sl@0	3068
sl@0	3069	/*
sl@0	3070	** Sizing information for the circular buffer used in snippetOffsetsOfColumn()
sl@0	3071	*/
sl@0	3072	#define FTS2_ROTOR_SZ (32)
sl@0	3073	#define FTS2_ROTOR_MASK (FTS2_ROTOR_SZ-1)
sl@0	3074
sl@0	3075	/*
sl@0	3076	** Add entries to pSnippet->aMatch[] for every match that occurs against
sl@0	3077	** document zDoc[0..nDoc-1] which is stored in column iColumn.
sl@0	3078	*/
sl@0	3079	static void snippetOffsetsOfColumn(
sl@0	3080	Query *pQuery,
sl@0	3081	Snippet *pSnippet,
sl@0	3082	int iColumn,
sl@0	3083	const char *zDoc,
sl@0	3084	int nDoc
sl@0	3085	){
sl@0	3086	const sqlite3_tokenizer_module pTModule; / The tokenizer module */
sl@0	3087	sqlite3_tokenizer pTokenizer; / The specific tokenizer */
sl@0	3088	sqlite3_tokenizer_cursor pTCursor; / Tokenizer cursor */
sl@0	3089	fulltext_vtab pVtab; / The full text index */
sl@0	3090	int nColumn; /* Number of columns in the index */
sl@0	3091	const QueryTerm aTerm; / Query string terms */
sl@0	3092	int nTerm; /* Number of query string terms */
sl@0	3093	int i, j; /* Loop counters */
sl@0	3094	int rc; /* Return code */
sl@0	3095	unsigned int match, prevMatch; /* Phrase search bitmasks */
sl@0	3096	const char zToken; / Next token from the tokenizer */
sl@0	3097	int nToken; /* Size of zToken */
sl@0	3098	int iBegin, iEnd, iPos; /* Offsets of beginning and end */
sl@0	3099
sl@0	3100	/* The following variables keep a circular buffer of the last
sl@0	3101	** few tokens */
sl@0	3102	unsigned int iRotor = 0; /* Index of current token */
sl@0	3103	int iRotorBegin[FTS2_ROTOR_SZ]; /* Beginning offset of token */
sl@0	3104	int iRotorLen[FTS2_ROTOR_SZ]; /* Length of token */
sl@0	3105
sl@0	3106	pVtab = pQuery->pFts;
sl@0	3107	nColumn = pVtab->nColumn;
sl@0	3108	pTokenizer = pVtab->pTokenizer;
sl@0	3109	pTModule = pTokenizer->pModule;
sl@0	3110	rc = pTModule->xOpen(pTokenizer, zDoc, nDoc, &pTCursor);
sl@0	3111	if( rc ) return;
sl@0	3112	pTCursor->pTokenizer = pTokenizer;
sl@0	3113	aTerm = pQuery->pTerms;
sl@0	3114	nTerm = pQuery->nTerms;
sl@0	3115	if( nTerm>=FTS2_ROTOR_SZ ){
sl@0	3116	nTerm = FTS2_ROTOR_SZ - 1;
sl@0	3117	}
sl@0	3118	prevMatch = 0;
sl@0	3119	while(1){
sl@0	3120	rc = pTModule->xNext(pTCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos);
sl@0	3121	if( rc ) break;
sl@0	3122	iRotorBegin[iRotor&FTS2_ROTOR_MASK] = iBegin;
sl@0	3123	iRotorLen[iRotor&FTS2_ROTOR_MASK] = iEnd-iBegin;
sl@0	3124	match = 0;
sl@0	3125	for(i=0; i<nTerm; i++){
sl@0	3126	int iCol;
sl@0	3127	iCol = aTerm[i].iColumn;
sl@0	3128	if( iCol>=0 && iCol<nColumn && iCol!=iColumn ) continue;
sl@0	3129	if( aTerm[i].nTerm>nToken ) continue;
sl@0	3130	if( !aTerm[i].isPrefix && aTerm[i].nTerm<nToken ) continue;
sl@0	3131	assert( aTerm[i].nTerm<=nToken );
sl@0	3132	if( memcmp(aTerm[i].pTerm, zToken, aTerm[i].nTerm) ) continue;
sl@0	3133	if( aTerm[i].iPhrase>1 && (prevMatch & (1<<i))==0 ) continue;
sl@0	3134	match \|= 1<<i;
sl@0	3135	if( i==nTerm-1 \|\| aTerm[i+1].iPhrase==1 ){
sl@0	3136	for(j=aTerm[i].iPhrase-1; j>=0; j--){
sl@0	3137	int k = (iRotor-j) & FTS2_ROTOR_MASK;
sl@0	3138	snippetAppendMatch(pSnippet, iColumn, i-j,
sl@0	3139	iRotorBegin[k], iRotorLen[k]);
sl@0	3140	}
sl@0	3141	}
sl@0	3142	}
sl@0	3143	prevMatch = match<<1;
sl@0	3144	iRotor++;
sl@0	3145	}
sl@0	3146	pTModule->xClose(pTCursor);
sl@0	3147	}
sl@0	3148
sl@0	3149
sl@0	3150	/*
sl@0	3151	** Compute all offsets for the current row of the query.
sl@0	3152	** If the offsets have already been computed, this routine is a no-op.
sl@0	3153	*/
sl@0	3154	static void snippetAllOffsets(fulltext_cursor *p){
sl@0	3155	int nColumn;
sl@0	3156	int iColumn, i;
sl@0	3157	int iFirst, iLast;
sl@0	3158	fulltext_vtab *pFts;
sl@0	3159
sl@0	3160	if( p->snippet.nMatch ) return;
sl@0	3161	if( p->q.nTerms==0 ) return;
sl@0	3162	pFts = p->q.pFts;
sl@0	3163	nColumn = pFts->nColumn;
sl@0	3164	iColumn = (p->iCursorType - QUERY_FULLTEXT);
sl@0	3165	if( iColumn<0 \|\| iColumn>=nColumn ){
sl@0	3166	iFirst = 0;
sl@0	3167	iLast = nColumn-1;
sl@0	3168	}else{
sl@0	3169	iFirst = iColumn;
sl@0	3170	iLast = iColumn;
sl@0	3171	}
sl@0	3172	for(i=iFirst; i<=iLast; i++){
sl@0	3173	const char *zDoc;
sl@0	3174	int nDoc;
sl@0	3175	zDoc = (const char*)sqlite3_column_text(p->pStmt, i+1);
sl@0	3176	nDoc = sqlite3_column_bytes(p->pStmt, i+1);
sl@0	3177	snippetOffsetsOfColumn(&p->q, &p->snippet, i, zDoc, nDoc);
sl@0	3178	}
sl@0	3179	}
sl@0	3180
sl@0	3181	/*
sl@0	3182	** Convert the information in the aMatch[] array of the snippet
sl@0	3183	** into the string zOffset[0..nOffset-1].
sl@0	3184	*/
sl@0	3185	static void snippetOffsetText(Snippet *p){
sl@0	3186	int i;
sl@0	3187	int cnt = 0;
sl@0	3188	StringBuffer sb;
sl@0	3189	char zBuf[200];
sl@0	3190	if( p->zOffset ) return;
sl@0	3191	initStringBuffer(&sb);
sl@0	3192	for(i=0; i<p->nMatch; i++){
sl@0	3193	struct snippetMatch *pMatch = &p->aMatch[i];
sl@0	3194	zBuf[0] = ' ';
sl@0	3195	sqlite3_snprintf(sizeof(zBuf)-1, &zBuf[cnt>0], "%d %d %d %d",
sl@0	3196	pMatch->iCol, pMatch->iTerm, pMatch->iStart, pMatch->nByte);
sl@0	3197	append(&sb, zBuf);
sl@0	3198	cnt++;
sl@0	3199	}
sl@0	3200	p->zOffset = stringBufferData(&sb);
sl@0	3201	p->nOffset = stringBufferLength(&sb);
sl@0	3202	}
sl@0	3203
sl@0	3204	/*
sl@0	3205	** zDoc[0..nDoc-1] is phrase of text. aMatch[0..nMatch-1] are a set
sl@0	3206	** of matching words some of which might be in zDoc. zDoc is column
sl@0	3207	** number iCol.
sl@0	3208	**
sl@0	3209	** iBreak is suggested spot in zDoc where we could begin or end an
sl@0	3210	** excerpt. Return a value similar to iBreak but possibly adjusted
sl@0	3211	** to be a little left or right so that the break point is better.
sl@0	3212	*/
sl@0	3213	static int wordBoundary(
sl@0	3214	int iBreak, /* The suggested break point */
sl@0	3215	const char zDoc, / Document text */
sl@0	3216	int nDoc, /* Number of bytes in zDoc[] */
sl@0	3217	struct snippetMatch aMatch, / Matching words */
sl@0	3218	int nMatch, /* Number of entries in aMatch[] */
sl@0	3219	int iCol /* The column number for zDoc[] */
sl@0	3220	){
sl@0	3221	int i;
sl@0	3222	if( iBreak<=10 ){
sl@0	3223	return 0;
sl@0	3224	}
sl@0	3225	if( iBreak>=nDoc-10 ){
sl@0	3226	return nDoc;
sl@0	3227	}
sl@0	3228	for(i=0; i<nMatch && aMatch[i].iCol<iCol; i++){}
sl@0	3229	while( i<nMatch && aMatch[i].iStart+aMatch[i].nByte<iBreak ){ i++; }
sl@0	3230	if( i<nMatch ){
sl@0	3231	if( aMatch[i].iStart<iBreak+10 ){
sl@0	3232	return aMatch[i].iStart;
sl@0	3233	}
sl@0	3234	if( i>0 && aMatch[i-1].iStart+aMatch[i-1].nByte>=iBreak ){
sl@0	3235	return aMatch[i-1].iStart;
sl@0	3236	}
sl@0	3237	}
sl@0	3238	for(i=1; i<=10; i++){
sl@0	3239	if( safe_isspace(zDoc[iBreak-i]) ){
sl@0	3240	return iBreak - i + 1;
sl@0	3241	}
sl@0	3242	if( safe_isspace(zDoc[iBreak+i]) ){
sl@0	3243	return iBreak + i + 1;
sl@0	3244	}
sl@0	3245	}
sl@0	3246	return iBreak;
sl@0	3247	}
sl@0	3248
sl@0	3249
sl@0	3250
sl@0	3251	/*
sl@0	3252	** Allowed values for Snippet.aMatch[].snStatus
sl@0	3253	*/
sl@0	3254	#define SNIPPET_IGNORE 0 /* It is ok to omit this match from the snippet */
sl@0	3255	#define SNIPPET_DESIRED 1 /* We want to include this match in the snippet */
sl@0	3256
sl@0	3257	/*
sl@0	3258	** Generate the text of a snippet.
sl@0	3259	*/
sl@0	3260	static void snippetText(
sl@0	3261	fulltext_cursor pCursor, / The cursor we need the snippet for */
sl@0	3262	const char zStartMark, / Markup to appear before each match */
sl@0	3263	const char zEndMark, / Markup to appear after each match */
sl@0	3264	const char zEllipsis / Ellipsis mark */
sl@0	3265	){
sl@0	3266	int i, j;
sl@0	3267	struct snippetMatch *aMatch;
sl@0	3268	int nMatch;
sl@0	3269	int nDesired;
sl@0	3270	StringBuffer sb;
sl@0	3271	int tailCol;
sl@0	3272	int tailOffset;
sl@0	3273	int iCol;
sl@0	3274	int nDoc;
sl@0	3275	const char *zDoc;
sl@0	3276	int iStart, iEnd;
sl@0	3277	int tailEllipsis = 0;
sl@0	3278	int iMatch;
sl@0	3279
sl@0	3280
sl@0	3281	sqlite3_free(pCursor->snippet.zSnippet);
sl@0	3282	pCursor->snippet.zSnippet = 0;
sl@0	3283	aMatch = pCursor->snippet.aMatch;
sl@0	3284	nMatch = pCursor->snippet.nMatch;
sl@0	3285	initStringBuffer(&sb);
sl@0	3286
sl@0	3287	for(i=0; i<nMatch; i++){
sl@0	3288	aMatch[i].snStatus = SNIPPET_IGNORE;
sl@0	3289	}
sl@0	3290	nDesired = 0;
sl@0	3291	for(i=0; i<pCursor->q.nTerms; i++){
sl@0	3292	for(j=0; j<nMatch; j++){
sl@0	3293	if( aMatch[j].iTerm==i ){
sl@0	3294	aMatch[j].snStatus = SNIPPET_DESIRED;
sl@0	3295	nDesired++;
sl@0	3296	break;
sl@0	3297	}
sl@0	3298	}
sl@0	3299	}
sl@0	3300
sl@0	3301	iMatch = 0;
sl@0	3302	tailCol = -1;
sl@0	3303	tailOffset = 0;
sl@0	3304	for(i=0; i<nMatch && nDesired>0; i++){
sl@0	3305	if( aMatch[i].snStatus!=SNIPPET_DESIRED ) continue;
sl@0	3306	nDesired--;
sl@0	3307	iCol = aMatch[i].iCol;
sl@0	3308	zDoc = (const char*)sqlite3_column_text(pCursor->pStmt, iCol+1);
sl@0	3309	nDoc = sqlite3_column_bytes(pCursor->pStmt, iCol+1);
sl@0	3310	iStart = aMatch[i].iStart - 40;
sl@0	3311	iStart = wordBoundary(iStart, zDoc, nDoc, aMatch, nMatch, iCol);
sl@0	3312	if( iStart<=10 ){
sl@0	3313	iStart = 0;
sl@0	3314	}
sl@0	3315	if( iCol==tailCol && iStart<=tailOffset+20 ){
sl@0	3316	iStart = tailOffset;
sl@0	3317	}
sl@0	3318	if( (iCol!=tailCol && tailCol>=0) \|\| iStart!=tailOffset ){
sl@0	3319	trimWhiteSpace(&sb);
sl@0	3320	appendWhiteSpace(&sb);
sl@0	3321	append(&sb, zEllipsis);
sl@0	3322	appendWhiteSpace(&sb);
sl@0	3323	}
sl@0	3324	iEnd = aMatch[i].iStart + aMatch[i].nByte + 40;
sl@0	3325	iEnd = wordBoundary(iEnd, zDoc, nDoc, aMatch, nMatch, iCol);
sl@0	3326	if( iEnd>=nDoc-10 ){
sl@0	3327	iEnd = nDoc;
sl@0	3328	tailEllipsis = 0;
sl@0	3329	}else{
sl@0	3330	tailEllipsis = 1;
sl@0	3331	}
sl@0	3332	while( iMatch<nMatch && aMatch[iMatch].iCol<iCol ){ iMatch++; }
sl@0	3333	while( iStart<iEnd ){
sl@0	3334	while( iMatch<nMatch && aMatch[iMatch].iStart<iStart
sl@0	3335	&& aMatch[iMatch].iCol<=iCol ){
sl@0	3336	iMatch++;
sl@0	3337	}
sl@0	3338	if( iMatch<nMatch && aMatch[iMatch].iStart<iEnd
sl@0	3339	&& aMatch[iMatch].iCol==iCol ){
sl@0	3340	nappend(&sb, &zDoc[iStart], aMatch[iMatch].iStart - iStart);
sl@0	3341	iStart = aMatch[iMatch].iStart;
sl@0	3342	append(&sb, zStartMark);
sl@0	3343	nappend(&sb, &zDoc[iStart], aMatch[iMatch].nByte);
sl@0	3344	append(&sb, zEndMark);
sl@0	3345	iStart += aMatch[iMatch].nByte;
sl@0	3346	for(j=iMatch+1; j<nMatch; j++){
sl@0	3347	if( aMatch[j].iTerm==aMatch[iMatch].iTerm
sl@0	3348	&& aMatch[j].snStatus==SNIPPET_DESIRED ){
sl@0	3349	nDesired--;
sl@0	3350	aMatch[j].snStatus = SNIPPET_IGNORE;
sl@0	3351	}
sl@0	3352	}
sl@0	3353	}else{
sl@0	3354	nappend(&sb, &zDoc[iStart], iEnd - iStart);
sl@0	3355	iStart = iEnd;
sl@0	3356	}
sl@0	3357	}
sl@0	3358	tailCol = iCol;
sl@0	3359	tailOffset = iEnd;
sl@0	3360	}
sl@0	3361	trimWhiteSpace(&sb);
sl@0	3362	if( tailEllipsis ){
sl@0	3363	appendWhiteSpace(&sb);
sl@0	3364	append(&sb, zEllipsis);
sl@0	3365	}
sl@0	3366	pCursor->snippet.zSnippet = stringBufferData(&sb);
sl@0	3367	pCursor->snippet.nSnippet = stringBufferLength(&sb);
sl@0	3368	}
sl@0	3369
sl@0	3370
sl@0	3371	/*
sl@0	3372	** Close the cursor. For additional information see the documentation
sl@0	3373	** on the xClose method of the virtual table interface.
sl@0	3374	*/
sl@0	3375	static int fulltextClose(sqlite3_vtab_cursor *pCursor){
sl@0	3376	fulltext_cursor c = (fulltext_cursor ) pCursor;
sl@0	3377	TRACE(("FTS2 Close %p\n", c));
sl@0	3378	sqlite3_finalize(c->pStmt);
sl@0	3379	queryClear(&c->q);
sl@0	3380	snippetClear(&c->snippet);
sl@0	3381	if( c->result.nData!=0 ) dlrDestroy(&c->reader);
sl@0	3382	dataBufferDestroy(&c->result);
sl@0	3383	sqlite3_free(c);
sl@0	3384	return SQLITE_OK;
sl@0	3385	}
sl@0	3386
sl@0	3387	static int fulltextNext(sqlite3_vtab_cursor *pCursor){
sl@0	3388	fulltext_cursor c = (fulltext_cursor ) pCursor;
sl@0	3389	int rc;
sl@0	3390
sl@0	3391	TRACE(("FTS2 Next %p\n", pCursor));
sl@0	3392	snippetClear(&c->snippet);
sl@0	3393	if( c->iCursorType < QUERY_FULLTEXT ){
sl@0	3394	/* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
sl@0	3395	rc = sqlite3_step(c->pStmt);
sl@0	3396	switch( rc ){
sl@0	3397	case SQLITE_ROW:
sl@0	3398	c->eof = 0;
sl@0	3399	return SQLITE_OK;
sl@0	3400	case SQLITE_DONE:
sl@0	3401	c->eof = 1;
sl@0	3402	return SQLITE_OK;
sl@0	3403	default:
sl@0	3404	c->eof = 1;
sl@0	3405	return rc;
sl@0	3406	}
sl@0	3407	} else { /* full-text query */
sl@0	3408	rc = sqlite3_reset(c->pStmt);
sl@0	3409	if( rc!=SQLITE_OK ) return rc;
sl@0	3410
sl@0	3411	if( c->result.nData==0 \|\| dlrAtEnd(&c->reader) ){
sl@0	3412	c->eof = 1;
sl@0	3413	return SQLITE_OK;
sl@0	3414	}
sl@0	3415	rc = sqlite3_bind_int64(c->pStmt, 1, dlrDocid(&c->reader));
sl@0	3416	dlrStep(&c->reader);
sl@0	3417	if( rc!=SQLITE_OK ) return rc;
sl@0	3418	/* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
sl@0	3419	rc = sqlite3_step(c->pStmt);
sl@0	3420	if( rc==SQLITE_ROW ){ /* the case we expect */
sl@0	3421	c->eof = 0;
sl@0	3422	return SQLITE_OK;
sl@0	3423	}
sl@0	3424	/* an error occurred; abort */
sl@0	3425	return rc==SQLITE_DONE ? SQLITE_ERROR : rc;
sl@0	3426	}
sl@0	3427	}
sl@0	3428
sl@0	3429
sl@0	3430	/* TODO(shess) If we pushed LeafReader to the top of the file, or to
sl@0	3431	** another file, term_select() could be pushed above
sl@0	3432	** docListOfTerm().
sl@0	3433	*/
sl@0	3434	static int termSelect(fulltext_vtab *v, int iColumn,
sl@0	3435	const char *pTerm, int nTerm, int isPrefix,
sl@0	3436	DocListType iType, DataBuffer *out);
sl@0	3437
sl@0	3438	/* Return a DocList corresponding to the query term pTerm. If pTerm
sl@0	3439	** is the first term of a phrase query, go ahead and evaluate the phrase
sl@0	3440	** query and return the doclist for the entire phrase query.
sl@0	3441	**
sl@0	3442	** The resulting DL_DOCIDS doclist is stored in pResult, which is
sl@0	3443	** overwritten.
sl@0	3444	*/
sl@0	3445	static int docListOfTerm(
sl@0	3446	fulltext_vtab v, / The full text index */
sl@0	3447	int iColumn, /* column to restrict to. No restriction if >=nColumn */
sl@0	3448	QueryTerm pQTerm, / Term we are looking for, or 1st term of a phrase */
sl@0	3449	DataBuffer pResult / Write the result here */
sl@0	3450	){
sl@0	3451	DataBuffer left, right, new;
sl@0	3452	int i, rc;
sl@0	3453
sl@0	3454	/* No phrase search if no position info. */
sl@0	3455	assert( pQTerm->nPhrase==0 \|\| DL_DEFAULT!=DL_DOCIDS );
sl@0	3456
sl@0	3457	/* This code should never be called with buffered updates. */
sl@0	3458	assert( v->nPendingData<0 );
sl@0	3459
sl@0	3460	dataBufferInit(&left, 0);
sl@0	3461	rc = termSelect(v, iColumn, pQTerm->pTerm, pQTerm->nTerm, pQTerm->isPrefix,
sl@0	3462	0<pQTerm->nPhrase ? DL_POSITIONS : DL_DOCIDS, &left);
sl@0	3463	if( rc ) return rc;
sl@0	3464	for(i=1; i<=pQTerm->nPhrase && left.nData>0; i++){
sl@0	3465	dataBufferInit(&right, 0);
sl@0	3466	rc = termSelect(v, iColumn, pQTerm[i].pTerm, pQTerm[i].nTerm,
sl@0	3467	pQTerm[i].isPrefix, DL_POSITIONS, &right);
sl@0	3468	if( rc ){
sl@0	3469	dataBufferDestroy(&left);
sl@0	3470	return rc;
sl@0	3471	}
sl@0	3472	dataBufferInit(&new, 0);
sl@0	3473	docListPhraseMerge(left.pData, left.nData, right.pData, right.nData,
sl@0	3474	i<pQTerm->nPhrase ? DL_POSITIONS : DL_DOCIDS, &new);
sl@0	3475	dataBufferDestroy(&left);
sl@0	3476	dataBufferDestroy(&right);
sl@0	3477	left = new;
sl@0	3478	}
sl@0	3479	*pResult = left;
sl@0	3480	return SQLITE_OK;
sl@0	3481	}
sl@0	3482
sl@0	3483	/* Add a new term pTerm[0..nTerm-1] to the query *q.
sl@0	3484	*/
sl@0	3485	static void queryAdd(Query q, const char pTerm, int nTerm){
sl@0	3486	QueryTerm *t;
sl@0	3487	++q->nTerms;
sl@0	3488	q->pTerms = sqlite3_realloc(q->pTerms, q->nTerms * sizeof(q->pTerms[0]));
sl@0	3489	if( q->pTerms==0 ){
sl@0	3490	q->nTerms = 0;
sl@0	3491	return;
sl@0	3492	}
sl@0	3493	t = &q->pTerms[q->nTerms - 1];
sl@0	3494	CLEAR(t);
sl@0	3495	t->pTerm = sqlite3_malloc(nTerm+1);
sl@0	3496	memcpy(t->pTerm, pTerm, nTerm);
sl@0	3497	t->pTerm[nTerm] = 0;
sl@0	3498	t->nTerm = nTerm;
sl@0	3499	t->isOr = q->nextIsOr;
sl@0	3500	t->isPrefix = 0;
sl@0	3501	q->nextIsOr = 0;
sl@0	3502	t->iColumn = q->nextColumn;
sl@0	3503	q->nextColumn = q->dfltColumn;
sl@0	3504	}
sl@0	3505
sl@0	3506	/*
sl@0	3507	** Check to see if the string zToken[0...nToken-1] matches any
sl@0	3508	** column name in the virtual table. If it does,
sl@0	3509	** return the zero-indexed column number. If not, return -1.
sl@0	3510	*/
sl@0	3511	static int checkColumnSpecifier(
sl@0	3512	fulltext_vtab pVtab, / The virtual table */
sl@0	3513	const char zToken, / Text of the token */
sl@0	3514	int nToken /* Number of characters in the token */
sl@0	3515	){
sl@0	3516	int i;
sl@0	3517	for(i=0; i<pVtab->nColumn; i++){
sl@0	3518	if( memcmp(pVtab->azColumn[i], zToken, nToken)==0
sl@0	3519	&& pVtab->azColumn[i][nToken]==0 ){
sl@0	3520	return i;
sl@0	3521	}
sl@0	3522	}
sl@0	3523	return -1;
sl@0	3524	}
sl@0	3525
sl@0	3526	/*
sl@0	3527	** Parse the text at pSegment[0..nSegment-1]. Add additional terms
sl@0	3528	** to the query being assemblied in pQuery.
sl@0	3529	**
sl@0	3530	** inPhrase is true if pSegment[0..nSegement-1] is contained within
sl@0	3531	** double-quotes. If inPhrase is true, then the first term
sl@0	3532	** is marked with the number of terms in the phrase less one and
sl@0	3533	** OR and "-" syntax is ignored. If inPhrase is false, then every
sl@0	3534	** term found is marked with nPhrase=0 and OR and "-" syntax is significant.
sl@0	3535	*/
sl@0	3536	static int tokenizeSegment(
sl@0	3537	sqlite3_tokenizer pTokenizer, / The tokenizer to use */
sl@0	3538	const char pSegment, int nSegment, / Query expression being parsed */
sl@0	3539	int inPhrase, /* True if within "..." */
sl@0	3540	Query pQuery / Append results here */
sl@0	3541	){
sl@0	3542	const sqlite3_tokenizer_module *pModule = pTokenizer->pModule;
sl@0	3543	sqlite3_tokenizer_cursor *pCursor;
sl@0	3544	int firstIndex = pQuery->nTerms;
sl@0	3545	int iCol;
sl@0	3546	int nTerm = 1;
sl@0	3547
sl@0	3548	int rc = pModule->xOpen(pTokenizer, pSegment, nSegment, &pCursor);
sl@0	3549	if( rc!=SQLITE_OK ) return rc;
sl@0	3550	pCursor->pTokenizer = pTokenizer;
sl@0	3551
sl@0	3552	while( 1 ){
sl@0	3553	const char *pToken;
sl@0	3554	int nToken, iBegin, iEnd, iPos;
sl@0	3555
sl@0	3556	rc = pModule->xNext(pCursor,
sl@0	3557	&pToken, &nToken,
sl@0	3558	&iBegin, &iEnd, &iPos);
sl@0	3559	if( rc!=SQLITE_OK ) break;
sl@0	3560	if( !inPhrase &&
sl@0	3561	pSegment[iEnd]==':' &&
sl@0	3562	(iCol = checkColumnSpecifier(pQuery->pFts, pToken, nToken))>=0 ){
sl@0	3563	pQuery->nextColumn = iCol;
sl@0	3564	continue;
sl@0	3565	}
sl@0	3566	if( !inPhrase && pQuery->nTerms>0 && nToken==2
sl@0	3567	&& pSegment[iBegin]=='O' && pSegment[iBegin+1]=='R' ){
sl@0	3568	pQuery->nextIsOr = 1;
sl@0	3569	continue;
sl@0	3570	}
sl@0	3571	queryAdd(pQuery, pToken, nToken);
sl@0	3572	if( !inPhrase && iBegin>0 && pSegment[iBegin-1]=='-' ){
sl@0	3573	pQuery->pTerms[pQuery->nTerms-1].isNot = 1;
sl@0	3574	}
sl@0	3575	if( iEnd<nSegment && pSegment[iEnd]=='*' ){
sl@0	3576	pQuery->pTerms[pQuery->nTerms-1].isPrefix = 1;
sl@0	3577	}
sl@0	3578	pQuery->pTerms[pQuery->nTerms-1].iPhrase = nTerm;
sl@0	3579	if( inPhrase ){
sl@0	3580	nTerm++;
sl@0	3581	}
sl@0	3582	}
sl@0	3583
sl@0	3584	if( inPhrase && pQuery->nTerms>firstIndex ){
sl@0	3585	pQuery->pTerms[firstIndex].nPhrase = pQuery->nTerms - firstIndex - 1;
sl@0	3586	}
sl@0	3587
sl@0	3588	return pModule->xClose(pCursor);
sl@0	3589	}
sl@0	3590
sl@0	3591	/* Parse a query string, yielding a Query object pQuery.
sl@0	3592	**
sl@0	3593	** The calling function will need to queryClear() to clean up
sl@0	3594	** the dynamically allocated memory held by pQuery.
sl@0	3595	*/
sl@0	3596	static int parseQuery(
sl@0	3597	fulltext_vtab v, / The fulltext index */
sl@0	3598	const char zInput, / Input text of the query string */
sl@0	3599	int nInput, /* Size of the input text */
sl@0	3600	int dfltColumn, /* Default column of the index to match against */
sl@0	3601	Query pQuery / Write the parse results here. */
sl@0	3602	){
sl@0	3603	int iInput, inPhrase = 0;
sl@0	3604
sl@0	3605	if( zInput==0 ) nInput = 0;
sl@0	3606	if( nInput<0 ) nInput = strlen(zInput);
sl@0	3607	pQuery->nTerms = 0;
sl@0	3608	pQuery->pTerms = NULL;
sl@0	3609	pQuery->nextIsOr = 0;
sl@0	3610	pQuery->nextColumn = dfltColumn;
sl@0	3611	pQuery->dfltColumn = dfltColumn;
sl@0	3612	pQuery->pFts = v;
sl@0	3613
sl@0	3614	for(iInput=0; iInput<nInput; ++iInput){
sl@0	3615	int i;
sl@0	3616	for(i=iInput; i<nInput && zInput[i]!='"'; ++i){}
sl@0	3617	if( i>iInput ){
sl@0	3618	tokenizeSegment(v->pTokenizer, zInput+iInput, i-iInput, inPhrase,
sl@0	3619	pQuery);
sl@0	3620	}
sl@0	3621	iInput = i;
sl@0	3622	if( i<nInput ){
sl@0	3623	assert( zInput[i]=='"' );
sl@0	3624	inPhrase = !inPhrase;
sl@0	3625	}
sl@0	3626	}
sl@0	3627
sl@0	3628	if( inPhrase ){
sl@0	3629	/* unmatched quote */
sl@0	3630	queryClear(pQuery);
sl@0	3631	return SQLITE_ERROR;
sl@0	3632	}
sl@0	3633	return SQLITE_OK;
sl@0	3634	}
sl@0	3635
sl@0	3636	/* TODO(shess) Refactor the code to remove this forward decl. */
sl@0	3637	static int flushPendingTerms(fulltext_vtab *v);
sl@0	3638
sl@0	3639	/* Perform a full-text query using the search expression in
sl@0	3640	** zInput[0..nInput-1]. Return a list of matching documents
sl@0	3641	** in pResult.
sl@0	3642	**
sl@0	3643	** Queries must match column iColumn. Or if iColumn>=nColumn
sl@0	3644	** they are allowed to match against any column.
sl@0	3645	*/
sl@0	3646	static int fulltextQuery(
sl@0	3647	fulltext_vtab v, / The full text index */
sl@0	3648	int iColumn, /* Match against this column by default */
sl@0	3649	const char zInput, / The query string */
sl@0	3650	int nInput, /* Number of bytes in zInput[] */
sl@0	3651	DataBuffer pResult, / Write the result doclist here */
sl@0	3652	Query pQuery / Put parsed query string here */
sl@0	3653	){
sl@0	3654	int i, iNext, rc;
sl@0	3655	DataBuffer left, right, or, new;
sl@0	3656	int nNot = 0;
sl@0	3657	QueryTerm *aTerm;
sl@0	3658
sl@0	3659	/* TODO(shess) Instead of flushing pendingTerms, we could query for
sl@0	3660	** the relevant term and merge the doclist into what we receive from
sl@0	3661	** the database. Wait and see if this is a common issue, first.
sl@0	3662	**
sl@0	3663	** A good reason not to flush is to not generate update-related
sl@0	3664	** error codes from here.
sl@0	3665	*/
sl@0	3666
sl@0	3667	/* Flush any buffered updates before executing the query. */
sl@0	3668	rc = flushPendingTerms(v);
sl@0	3669	if( rc!=SQLITE_OK ) return rc;
sl@0	3670
sl@0	3671	/* TODO(shess) I think that the queryClear() calls below are not
sl@0	3672	** necessary, because fulltextClose() already clears the query.
sl@0	3673	*/
sl@0	3674	rc = parseQuery(v, zInput, nInput, iColumn, pQuery);
sl@0	3675	if( rc!=SQLITE_OK ) return rc;
sl@0	3676
sl@0	3677	/* Empty or NULL queries return no results. */
sl@0	3678	if( pQuery->nTerms==0 ){
sl@0	3679	dataBufferInit(pResult, 0);
sl@0	3680	return SQLITE_OK;
sl@0	3681	}
sl@0	3682
sl@0	3683	/* Merge AND terms. */
sl@0	3684	/* TODO(shess) I think we can early-exit if( i>nNot && left.nData==0 ). */
sl@0	3685	aTerm = pQuery->pTerms;
sl@0	3686	for(i = 0; i<pQuery->nTerms; i=iNext){
sl@0	3687	if( aTerm[i].isNot ){
sl@0	3688	/* Handle all NOT terms in a separate pass */
sl@0	3689	nNot++;
sl@0	3690	iNext = i + aTerm[i].nPhrase+1;
sl@0	3691	continue;
sl@0	3692	}
sl@0	3693	iNext = i + aTerm[i].nPhrase + 1;
sl@0	3694	rc = docListOfTerm(v, aTerm[i].iColumn, &aTerm[i], &right);
sl@0	3695	if( rc ){
sl@0	3696	if( i!=nNot ) dataBufferDestroy(&left);
sl@0	3697	queryClear(pQuery);
sl@0	3698	return rc;
sl@0	3699	}
sl@0	3700	while( iNext<pQuery->nTerms && aTerm[iNext].isOr ){
sl@0	3701	rc = docListOfTerm(v, aTerm[iNext].iColumn, &aTerm[iNext], &or);
sl@0	3702	iNext += aTerm[iNext].nPhrase + 1;
sl@0	3703	if( rc ){
sl@0	3704	if( i!=nNot ) dataBufferDestroy(&left);
sl@0	3705	dataBufferDestroy(&right);
sl@0	3706	queryClear(pQuery);
sl@0	3707	return rc;
sl@0	3708	}
sl@0	3709	dataBufferInit(&new, 0);
sl@0	3710	docListOrMerge(right.pData, right.nData, or.pData, or.nData, &new);
sl@0	3711	dataBufferDestroy(&right);
sl@0	3712	dataBufferDestroy(&or);
sl@0	3713	right = new;
sl@0	3714	}
sl@0	3715	if( i==nNot ){ /* first term processed. */
sl@0	3716	left = right;
sl@0	3717	}else{
sl@0	3718	dataBufferInit(&new, 0);
sl@0	3719	docListAndMerge(left.pData, left.nData, right.pData, right.nData, &new);
sl@0	3720	dataBufferDestroy(&right);
sl@0	3721	dataBufferDestroy(&left);
sl@0	3722	left = new;
sl@0	3723	}
sl@0	3724	}
sl@0	3725
sl@0	3726	if( nNot==pQuery->nTerms ){
sl@0	3727	/* We do not yet know how to handle a query of only NOT terms */
sl@0	3728	return SQLITE_ERROR;
sl@0	3729	}
sl@0	3730
sl@0	3731	/* Do the EXCEPT terms */
sl@0	3732	for(i=0; i<pQuery->nTerms; i += aTerm[i].nPhrase + 1){
sl@0	3733	if( !aTerm[i].isNot ) continue;
sl@0	3734	rc = docListOfTerm(v, aTerm[i].iColumn, &aTerm[i], &right);
sl@0	3735	if( rc ){
sl@0	3736	queryClear(pQuery);
sl@0	3737	dataBufferDestroy(&left);
sl@0	3738	return rc;
sl@0	3739	}
sl@0	3740	dataBufferInit(&new, 0);
sl@0	3741	docListExceptMerge(left.pData, left.nData, right.pData, right.nData, &new);
sl@0	3742	dataBufferDestroy(&right);
sl@0	3743	dataBufferDestroy(&left);
sl@0	3744	left = new;
sl@0	3745	}
sl@0	3746
sl@0	3747	*pResult = left;
sl@0	3748	return rc;
sl@0	3749	}
sl@0	3750
sl@0	3751	/*
sl@0	3752	** This is the xFilter interface for the virtual table. See
sl@0	3753	** the virtual table xFilter method documentation for additional
sl@0	3754	** information.
sl@0	3755	**
sl@0	3756	** If idxNum==QUERY_GENERIC then do a full table scan against
sl@0	3757	** the %_content table.
sl@0	3758	**
sl@0	3759	** If idxNum==QUERY_ROWID then do a rowid lookup for a single entry
sl@0	3760	** in the %_content table.
sl@0	3761	**
sl@0	3762	** If idxNum>=QUERY_FULLTEXT then use the full text index. The
sl@0	3763	** column on the left-hand side of the MATCH operator is column
sl@0	3764	** number idxNum-QUERY_FULLTEXT, 0 indexed. argv[0] is the right-hand
sl@0	3765	** side of the MATCH operator.
sl@0	3766	*/
sl@0	3767	/* TODO(shess) Upgrade the cursor initialization and destruction to
sl@0	3768	** account for fulltextFilter() being called multiple times on the
sl@0	3769	** same cursor. The current solution is very fragile. Apply fix to
sl@0	3770	** fts2 as appropriate.
sl@0	3771	*/
sl@0	3772	static int fulltextFilter(
sl@0	3773	sqlite3_vtab_cursor pCursor, / The cursor used for this query */
sl@0	3774	int idxNum, const char idxStr, / Which indexing scheme to use */
sl@0	3775	int argc, sqlite3_value *argv / Arguments for the indexing scheme */
sl@0	3776	){
sl@0	3777	fulltext_cursor c = (fulltext_cursor ) pCursor;
sl@0	3778	fulltext_vtab *v = cursor_vtab(c);
sl@0	3779	int rc;
sl@0	3780
sl@0	3781	TRACE(("FTS2 Filter %p\n",pCursor));
sl@0	3782
sl@0	3783	/* If the cursor has a statement that was not prepared according to
sl@0	3784	** idxNum, clear it. I believe all calls to fulltextFilter with a
sl@0	3785	** given cursor will have the same idxNum , but in this case it's
sl@0	3786	** easy to be safe.
sl@0	3787	*/
sl@0	3788	if( c->pStmt && c->iCursorType!=idxNum ){
sl@0	3789	sqlite3_finalize(c->pStmt);
sl@0	3790	c->pStmt = NULL;
sl@0	3791	}
sl@0	3792
sl@0	3793	/* Get a fresh statement appropriate to idxNum. */
sl@0	3794	/* TODO(shess): Add a prepared-statement cache in the vt structure.
sl@0	3795	** The cache must handle multiple open cursors. Easier to cache the
sl@0	3796	** statement variants at the vt to reduce malloc/realloc/free here.
sl@0	3797	** Or we could have a StringBuffer variant which allowed stack
sl@0	3798	** construction for small values.
sl@0	3799	*/
sl@0	3800	if( !c->pStmt ){
sl@0	3801	char zSql = sqlite3_mprintf("select rowid, from %%_content %s",
sl@0	3802	idxNum==QUERY_GENERIC ? "" : "where rowid=?");
sl@0	3803	rc = sql_prepare(v->db, v->zDb, v->zName, &c->pStmt, zSql);
sl@0	3804	sqlite3_free(zSql);
sl@0	3805	if( rc!=SQLITE_OK ) return rc;
sl@0	3806	c->iCursorType = idxNum;
sl@0	3807	}else{
sl@0	3808	sqlite3_reset(c->pStmt);
sl@0	3809	assert( c->iCursorType==idxNum );
sl@0	3810	}
sl@0	3811
sl@0	3812	switch( idxNum ){
sl@0	3813	case QUERY_GENERIC:
sl@0	3814	break;
sl@0	3815
sl@0	3816	case QUERY_ROWID:
sl@0	3817	rc = sqlite3_bind_int64(c->pStmt, 1, sqlite3_value_int64(argv[0]));
sl@0	3818	if( rc!=SQLITE_OK ) return rc;
sl@0	3819	break;
sl@0	3820
sl@0	3821	default: /* full-text search */
sl@0	3822	{
sl@0	3823	const char zQuery = (const char )sqlite3_value_text(argv[0]);
sl@0	3824	assert( idxNum<=QUERY_FULLTEXT+v->nColumn);
sl@0	3825	assert( argc==1 );
sl@0	3826	queryClear(&c->q);
sl@0	3827	if( c->result.nData!=0 ){
sl@0	3828	/* This case happens if the same cursor is used repeatedly. */
sl@0	3829	dlrDestroy(&c->reader);
sl@0	3830	dataBufferReset(&c->result);
sl@0	3831	}else{
sl@0	3832	dataBufferInit(&c->result, 0);
sl@0	3833	}
sl@0	3834	rc = fulltextQuery(v, idxNum-QUERY_FULLTEXT, zQuery, -1, &c->result, &c->q);
sl@0	3835	if( rc!=SQLITE_OK ) return rc;
sl@0	3836	if( c->result.nData!=0 ){
sl@0	3837	dlrInit(&c->reader, DL_DOCIDS, c->result.pData, c->result.nData);
sl@0	3838	}
sl@0	3839	break;
sl@0	3840	}
sl@0	3841	}
sl@0	3842
sl@0	3843	return fulltextNext(pCursor);
sl@0	3844	}
sl@0	3845
sl@0	3846	/* This is the xEof method of the virtual table. The SQLite core
sl@0	3847	** calls this routine to find out if it has reached the end of
sl@0	3848	** a query's results set.
sl@0	3849	*/
sl@0	3850	static int fulltextEof(sqlite3_vtab_cursor *pCursor){
sl@0	3851	fulltext_cursor c = (fulltext_cursor ) pCursor;
sl@0	3852	return c->eof;
sl@0	3853	}
sl@0	3854
sl@0	3855	/* This is the xColumn method of the virtual table. The SQLite
sl@0	3856	** core calls this method during a query when it needs the value
sl@0	3857	** of a column from the virtual table. This method needs to use
sl@0	3858	** one of the sqlite3_result_*() routines to store the requested
sl@0	3859	** value back in the pContext.
sl@0	3860	*/
sl@0	3861	static int fulltextColumn(sqlite3_vtab_cursor *pCursor,
sl@0	3862	sqlite3_context *pContext, int idxCol){
sl@0	3863	fulltext_cursor c = (fulltext_cursor ) pCursor;
sl@0	3864	fulltext_vtab *v = cursor_vtab(c);
sl@0	3865
sl@0	3866	if( idxCol<v->nColumn ){
sl@0	3867	sqlite3_value *pVal = sqlite3_column_value(c->pStmt, idxCol+1);
sl@0	3868	sqlite3_result_value(pContext, pVal);
sl@0	3869	}else if( idxCol==v->nColumn ){
sl@0	3870	/* The extra column whose name is the same as the table.
sl@0	3871	** Return a blob which is a pointer to the cursor
sl@0	3872	*/
sl@0	3873	sqlite3_result_blob(pContext, &c, sizeof(c), SQLITE_TRANSIENT);
sl@0	3874	}
sl@0	3875	return SQLITE_OK;
sl@0	3876	}
sl@0	3877
sl@0	3878	/* This is the xRowid method. The SQLite core calls this routine to
sl@0	3879	** retrive the rowid for the current row of the result set. The
sl@0	3880	** rowid should be written to *pRowid.
sl@0	3881	*/
sl@0	3882	static int fulltextRowid(sqlite3_vtab_cursor pCursor, sqlite_int64 pRowid){
sl@0	3883	fulltext_cursor c = (fulltext_cursor ) pCursor;
sl@0	3884
sl@0	3885	*pRowid = sqlite3_column_int64(c->pStmt, 0);
sl@0	3886	return SQLITE_OK;
sl@0	3887	}
sl@0	3888
sl@0	3889	/* Add all terms in [zText] to pendingTerms table. If [iColumn] > 0,
sl@0	3890	** we also store positions and offsets in the hash table using that
sl@0	3891	** column number.
sl@0	3892	*/
sl@0	3893	static int buildTerms(fulltext_vtab *v, sqlite_int64 iDocid,
sl@0	3894	const char *zText, int iColumn){
sl@0	3895	sqlite3_tokenizer *pTokenizer = v->pTokenizer;
sl@0	3896	sqlite3_tokenizer_cursor *pCursor;
sl@0	3897	const char *pToken;
sl@0	3898	int nTokenBytes;
sl@0	3899	int iStartOffset, iEndOffset, iPosition;
sl@0	3900	int rc;
sl@0	3901
sl@0	3902	rc = pTokenizer->pModule->xOpen(pTokenizer, zText, -1, &pCursor);
sl@0	3903	if( rc!=SQLITE_OK ) return rc;
sl@0	3904
sl@0	3905	pCursor->pTokenizer = pTokenizer;
sl@0	3906	while( SQLITE_OK==(rc=pTokenizer->pModule->xNext(pCursor,
sl@0	3907	&pToken, &nTokenBytes,
sl@0	3908	&iStartOffset, &iEndOffset,
sl@0	3909	&iPosition)) ){
sl@0	3910	DLCollector *p;
sl@0	3911	int nData; /* Size of doclist before our update. */
sl@0	3912
sl@0	3913	/* Positions can't be negative; we use -1 as a terminator
sl@0	3914	* internally. Token can't be NULL or empty. */
sl@0	3915	if( iPosition<0 \|\| pToken == NULL \|\| nTokenBytes == 0 ){
sl@0	3916	rc = SQLITE_ERROR;
sl@0	3917	break;
sl@0	3918	}
sl@0	3919
sl@0	3920	p = fts2HashFind(&v->pendingTerms, pToken, nTokenBytes);
sl@0	3921	if( p==NULL ){
sl@0	3922	nData = 0;
sl@0	3923	p = dlcNew(iDocid, DL_DEFAULT);
sl@0	3924	fts2HashInsert(&v->pendingTerms, pToken, nTokenBytes, p);
sl@0	3925
sl@0	3926	/* Overhead for our hash table entry, the key, and the value. */
sl@0	3927	v->nPendingData += sizeof(struct fts2HashElem)+sizeof(*p)+nTokenBytes;
sl@0	3928	}else{
sl@0	3929	nData = p->b.nData;
sl@0	3930	if( p->dlw.iPrevDocid!=iDocid ) dlcNext(p, iDocid);
sl@0	3931	}
sl@0	3932	if( iColumn>=0 ){
sl@0	3933	dlcAddPos(p, iColumn, iPosition, iStartOffset, iEndOffset);
sl@0	3934	}
sl@0	3935
sl@0	3936	/* Accumulate data added by dlcNew or dlcNext, and dlcAddPos. */
sl@0	3937	v->nPendingData += p->b.nData-nData;
sl@0	3938	}
sl@0	3939
sl@0	3940	/* TODO(shess) Check return? Should this be able to cause errors at
sl@0	3941	** this point? Actually, same question about sqlite3_finalize(),
sl@0	3942	** though one could argue that failure there means that the data is
sl@0	3943	** not durable. ponder
sl@0	3944	*/
sl@0	3945	pTokenizer->pModule->xClose(pCursor);
sl@0	3946	if( SQLITE_DONE == rc ) return SQLITE_OK;
sl@0	3947	return rc;
sl@0	3948	}
sl@0	3949
sl@0	3950	/* Add doclists for all terms in [pValues] to pendingTerms table. */
sl@0	3951	static int insertTerms(fulltext_vtab *v, sqlite_int64 iRowid,
sl@0	3952	sqlite3_value **pValues){
sl@0	3953	int i;
sl@0	3954	for(i = 0; i < v->nColumn ; ++i){
sl@0	3955	char zText = (char)sqlite3_value_text(pValues[i]);
sl@0	3956	int rc = buildTerms(v, iRowid, zText, i);
sl@0	3957	if( rc!=SQLITE_OK ) return rc;
sl@0	3958	}
sl@0	3959	return SQLITE_OK;
sl@0	3960	}
sl@0	3961
sl@0	3962	/* Add empty doclists for all terms in the given row's content to
sl@0	3963	** pendingTerms.
sl@0	3964	*/
sl@0	3965	static int deleteTerms(fulltext_vtab *v, sqlite_int64 iRowid){
sl@0	3966	const char **pValues;
sl@0	3967	int i, rc;
sl@0	3968
sl@0	3969	/* TODO(shess) Should we allow such tables at all? */
sl@0	3970	if( DL_DEFAULT==DL_DOCIDS ) return SQLITE_ERROR;
sl@0	3971
sl@0	3972	rc = content_select(v, iRowid, &pValues);
sl@0	3973	if( rc!=SQLITE_OK ) return rc;
sl@0	3974
sl@0	3975	for(i = 0 ; i < v->nColumn; ++i) {
sl@0	3976	rc = buildTerms(v, iRowid, pValues[i], -1);
sl@0	3977	if( rc!=SQLITE_OK ) break;
sl@0	3978	}
sl@0	3979
sl@0	3980	freeStringArray(v->nColumn, pValues);
sl@0	3981	return SQLITE_OK;
sl@0	3982	}
sl@0	3983
sl@0	3984	/* TODO(shess) Refactor the code to remove this forward decl. */
sl@0	3985	static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid);
sl@0	3986
sl@0	3987	/* Insert a row into the %_content table; set *piRowid to be the ID of the
sl@0	3988	** new row. Add doclists for terms to pendingTerms.
sl@0	3989	*/
sl@0	3990	static int index_insert(fulltext_vtab v, sqlite3_value pRequestRowid,
sl@0	3991	sqlite3_value *pValues, sqlite_int64 piRowid){
sl@0	3992	int rc;
sl@0	3993
sl@0	3994	rc = content_insert(v, pRequestRowid, pValues); /* execute an SQL INSERT */
sl@0	3995	if( rc!=SQLITE_OK ) return rc;
sl@0	3996
sl@0	3997	*piRowid = sqlite3_last_insert_rowid(v->db);
sl@0	3998	rc = initPendingTerms(v, *piRowid);
sl@0	3999	if( rc!=SQLITE_OK ) return rc;
sl@0	4000
sl@0	4001	return insertTerms(v, *piRowid, pValues);
sl@0	4002	}
sl@0	4003
sl@0	4004	/* Delete a row from the %_content table; add empty doclists for terms
sl@0	4005	** to pendingTerms.
sl@0	4006	*/
sl@0	4007	static int index_delete(fulltext_vtab *v, sqlite_int64 iRow){
sl@0	4008	int rc = initPendingTerms(v, iRow);
sl@0	4009	if( rc!=SQLITE_OK ) return rc;
sl@0	4010
sl@0	4011	rc = deleteTerms(v, iRow);
sl@0	4012	if( rc!=SQLITE_OK ) return rc;
sl@0	4013
sl@0	4014	return content_delete(v, iRow); /* execute an SQL DELETE */
sl@0	4015	}
sl@0	4016
sl@0	4017	/* Update a row in the %_content table; add delete doclists to
sl@0	4018	** pendingTerms for old terms not in the new data, add insert doclists
sl@0	4019	** to pendingTerms for terms in the new data.
sl@0	4020	*/
sl@0	4021	static int index_update(fulltext_vtab *v, sqlite_int64 iRow,
sl@0	4022	sqlite3_value **pValues){
sl@0	4023	int rc = initPendingTerms(v, iRow);
sl@0	4024	if( rc!=SQLITE_OK ) return rc;
sl@0	4025
sl@0	4026	/* Generate an empty doclist for each term that previously appeared in this
sl@0	4027	* row. */
sl@0	4028	rc = deleteTerms(v, iRow);
sl@0	4029	if( rc!=SQLITE_OK ) return rc;
sl@0	4030
sl@0	4031	rc = content_update(v, pValues, iRow); /* execute an SQL UPDATE */
sl@0	4032	if( rc!=SQLITE_OK ) return rc;
sl@0	4033
sl@0	4034	/* Now add positions for terms which appear in the updated row. */
sl@0	4035	return insertTerms(v, iRow, pValues);
sl@0	4036	}
sl@0	4037
sl@0	4038	/*******************************************************************/
sl@0	4039	/* InteriorWriter is used to collect terms and block references into
sl@0	4040	** interior nodes in %_segments. See commentary at top of file for
sl@0	4041	** format.
sl@0	4042	*/
sl@0	4043
sl@0	4044	/* How large interior nodes can grow. */
sl@0	4045	#define INTERIOR_MAX 2048
sl@0	4046
sl@0	4047	/* Minimum number of terms per interior node (except the root). This
sl@0	4048	** prevents large terms from making the tree too skinny - must be >0
sl@0	4049	** so that the tree always makes progress. Note that the min tree
sl@0	4050	** fanout will be INTERIOR_MIN_TERMS+1.
sl@0	4051	*/
sl@0	4052	#define INTERIOR_MIN_TERMS 7
sl@0	4053	#if INTERIOR_MIN_TERMS<1
sl@0	4054	# error INTERIOR_MIN_TERMS must be greater than 0.
sl@0	4055	#endif
sl@0	4056
sl@0	4057	/* ROOT_MAX controls how much data is stored inline in the segment
sl@0	4058	** directory.
sl@0	4059	*/
sl@0	4060	/* TODO(shess) Push ROOT_MAX down to whoever is writing things. It's
sl@0	4061	** only here so that interiorWriterRootInfo() and leafWriterRootInfo()
sl@0	4062	** can both see it, but if the caller passed it in, we wouldn't even
sl@0	4063	** need a define.
sl@0	4064	*/
sl@0	4065	#define ROOT_MAX 1024
sl@0	4066	#if ROOT_MAX<VARINT_MAX*2
sl@0	4067	# error ROOT_MAX must have enough space for a header.
sl@0	4068	#endif
sl@0	4069
sl@0	4070	/* InteriorBlock stores a linked-list of interior blocks while a lower
sl@0	4071	** layer is being constructed.
sl@0	4072	*/
sl@0	4073	typedef struct InteriorBlock {
sl@0	4074	DataBuffer term; /* Leftmost term in block's subtree. */
sl@0	4075	DataBuffer data; /* Accumulated data for the block. */
sl@0	4076	struct InteriorBlock *next;
sl@0	4077	} InteriorBlock;
sl@0	4078
sl@0	4079	static InteriorBlock *interiorBlockNew(int iHeight, sqlite_int64 iChildBlock,
sl@0	4080	const char *pTerm, int nTerm){
sl@0	4081	InteriorBlock *block = sqlite3_malloc(sizeof(InteriorBlock));
sl@0	4082	char c[VARINT_MAX+VARINT_MAX];
sl@0	4083	int n;
sl@0	4084
sl@0	4085	if( block ){
sl@0	4086	memset(block, 0, sizeof(*block));
sl@0	4087	dataBufferInit(&block->term, 0);
sl@0	4088	dataBufferReplace(&block->term, pTerm, nTerm);
sl@0	4089
sl@0	4090	n = putVarint(c, iHeight);
sl@0	4091	n += putVarint(c+n, iChildBlock);
sl@0	4092	dataBufferInit(&block->data, INTERIOR_MAX);
sl@0	4093	dataBufferReplace(&block->data, c, n);
sl@0	4094	}
sl@0	4095	return block;
sl@0	4096	}
sl@0	4097
sl@0	4098	#ifndef NDEBUG
sl@0	4099	/* Verify that the data is readable as an interior node. */
sl@0	4100	static void interiorBlockValidate(InteriorBlock *pBlock){
sl@0	4101	const char *pData = pBlock->data.pData;
sl@0	4102	int nData = pBlock->data.nData;
sl@0	4103	int n, iDummy;
sl@0	4104	sqlite_int64 iBlockid;
sl@0	4105
sl@0	4106	assert( nData>0 );
sl@0	4107	assert( pData!=0 );
sl@0	4108	assert( pData+nData>pData );
sl@0	4109
sl@0	4110	/* Must lead with height of node as a varint(n), n>0 */
sl@0	4111	n = getVarint32(pData, &iDummy);
sl@0	4112	assert( n>0 );
sl@0	4113	assert( iDummy>0 );
sl@0	4114	assert( n<nData );
sl@0	4115	pData += n;
sl@0	4116	nData -= n;
sl@0	4117
sl@0	4118	/* Must contain iBlockid. */
sl@0	4119	n = getVarint(pData, &iBlockid);
sl@0	4120	assert( n>0 );
sl@0	4121	assert( n<=nData );
sl@0	4122	pData += n;
sl@0	4123	nData -= n;
sl@0	4124
sl@0	4125	/* Zero or more terms of positive length */
sl@0	4126	if( nData!=0 ){
sl@0	4127	/* First term is not delta-encoded. */
sl@0	4128	n = getVarint32(pData, &iDummy);
sl@0	4129	assert( n>0 );
sl@0	4130	assert( iDummy>0 );
sl@0	4131	assert( n+iDummy>0);
sl@0	4132	assert( n+iDummy<=nData );
sl@0	4133	pData += n+iDummy;
sl@0	4134	nData -= n+iDummy;
sl@0	4135
sl@0	4136	/* Following terms delta-encoded. */
sl@0	4137	while( nData!=0 ){
sl@0	4138	/* Length of shared prefix. */
sl@0	4139	n = getVarint32(pData, &iDummy);
sl@0	4140	assert( n>0 );
sl@0	4141	assert( iDummy>=0 );
sl@0	4142	assert( n<nData );
sl@0	4143	pData += n;
sl@0	4144	nData -= n;
sl@0	4145
sl@0	4146	/* Length and data of distinct suffix. */
sl@0	4147	n = getVarint32(pData, &iDummy);
sl@0	4148	assert( n>0 );
sl@0	4149	assert( iDummy>0 );
sl@0	4150	assert( n+iDummy>0);
sl@0	4151	assert( n+iDummy<=nData );
sl@0	4152	pData += n+iDummy;
sl@0	4153	nData -= n+iDummy;
sl@0	4154	}
sl@0	4155	}
sl@0	4156	}
sl@0	4157	#define ASSERT_VALID_INTERIOR_BLOCK(x) interiorBlockValidate(x)
sl@0	4158	#else
sl@0	4159	#define ASSERT_VALID_INTERIOR_BLOCK(x) assert( 1 )
sl@0	4160	#endif
sl@0	4161
sl@0	4162	typedef struct InteriorWriter {
sl@0	4163	int iHeight; /* from 0 at leaves. */
sl@0	4164	InteriorBlock first, last;
sl@0	4165	struct InteriorWriter *parentWriter;
sl@0	4166
sl@0	4167	DataBuffer term; /* Last term written to block "last". */
sl@0	4168	sqlite_int64 iOpeningChildBlock; /* First child block in block "last". */
sl@0	4169	#ifndef NDEBUG
sl@0	4170	sqlite_int64 iLastChildBlock; /* for consistency checks. */
sl@0	4171	#endif
sl@0	4172	} InteriorWriter;
sl@0	4173
sl@0	4174	/* Initialize an interior node where pTerm[nTerm] marks the leftmost
sl@0	4175	** term in the tree. iChildBlock is the leftmost child block at the
sl@0	4176	** next level down the tree.
sl@0	4177	*/
sl@0	4178	static void interiorWriterInit(int iHeight, const char *pTerm, int nTerm,
sl@0	4179	sqlite_int64 iChildBlock,
sl@0	4180	InteriorWriter *pWriter){
sl@0	4181	InteriorBlock *block;
sl@0	4182	assert( iHeight>0 );
sl@0	4183	CLEAR(pWriter);
sl@0	4184
sl@0	4185	pWriter->iHeight = iHeight;
sl@0	4186	pWriter->iOpeningChildBlock = iChildBlock;
sl@0	4187	#ifndef NDEBUG
sl@0	4188	pWriter->iLastChildBlock = iChildBlock;
sl@0	4189	#endif
sl@0	4190	block = interiorBlockNew(iHeight, iChildBlock, pTerm, nTerm);
sl@0	4191	pWriter->last = pWriter->first = block;
sl@0	4192	ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
sl@0	4193	dataBufferInit(&pWriter->term, 0);
sl@0	4194	}
sl@0	4195
sl@0	4196	/* Append the child node rooted at iChildBlock to the interior node,
sl@0	4197	** with pTerm[nTerm] as the leftmost term in iChildBlock's subtree.
sl@0	4198	*/
sl@0	4199	static void interiorWriterAppend(InteriorWriter *pWriter,
sl@0	4200	const char *pTerm, int nTerm,
sl@0	4201	sqlite_int64 iChildBlock){
sl@0	4202	char c[VARINT_MAX+VARINT_MAX];
sl@0	4203	int n, nPrefix = 0;
sl@0	4204
sl@0	4205	ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
sl@0	4206
sl@0	4207	/* The first term written into an interior node is actually
sl@0	4208	** associated with the second child added (the first child was added
sl@0	4209	** in interiorWriterInit, or in the if clause at the bottom of this
sl@0	4210	** function). That term gets encoded straight up, with nPrefix left
sl@0	4211	** at 0.
sl@0	4212	*/
sl@0	4213	if( pWriter->term.nData==0 ){
sl@0	4214	n = putVarint(c, nTerm);
sl@0	4215	}else{
sl@0	4216	while( nPrefix<pWriter->term.nData &&
sl@0	4217	pTerm[nPrefix]==pWriter->term.pData[nPrefix] ){
sl@0	4218	nPrefix++;
sl@0	4219	}
sl@0	4220
sl@0	4221	n = putVarint(c, nPrefix);
sl@0	4222	n += putVarint(c+n, nTerm-nPrefix);
sl@0	4223	}
sl@0	4224
sl@0	4225	#ifndef NDEBUG
sl@0	4226	pWriter->iLastChildBlock++;
sl@0	4227	#endif
sl@0	4228	assert( pWriter->iLastChildBlock==iChildBlock );
sl@0	4229
sl@0	4230	/* Overflow to a new block if the new term makes the current block
sl@0	4231	** too big, and the current block already has enough terms.
sl@0	4232	*/
sl@0	4233	if( pWriter->last->data.nData+n+nTerm-nPrefix>INTERIOR_MAX &&
sl@0	4234	iChildBlock-pWriter->iOpeningChildBlock>INTERIOR_MIN_TERMS ){
sl@0	4235	pWriter->last->next = interiorBlockNew(pWriter->iHeight, iChildBlock,
sl@0	4236	pTerm, nTerm);
sl@0	4237	pWriter->last = pWriter->last->next;
sl@0	4238	pWriter->iOpeningChildBlock = iChildBlock;
sl@0	4239	dataBufferReset(&pWriter->term);
sl@0	4240	}else{
sl@0	4241	dataBufferAppend2(&pWriter->last->data, c, n,
sl@0	4242	pTerm+nPrefix, nTerm-nPrefix);
sl@0	4243	dataBufferReplace(&pWriter->term, pTerm, nTerm);
sl@0	4244	}
sl@0	4245	ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
sl@0	4246	}
sl@0	4247
sl@0	4248	/* Free the space used by pWriter, including the linked-list of
sl@0	4249	** InteriorBlocks, and parentWriter, if present.
sl@0	4250	*/
sl@0	4251	static int interiorWriterDestroy(InteriorWriter *pWriter){
sl@0	4252	InteriorBlock *block = pWriter->first;
sl@0	4253
sl@0	4254	while( block!=NULL ){
sl@0	4255	InteriorBlock *b = block;
sl@0	4256	block = block->next;
sl@0	4257	dataBufferDestroy(&b->term);
sl@0	4258	dataBufferDestroy(&b->data);
sl@0	4259	sqlite3_free(b);
sl@0	4260	}
sl@0	4261	if( pWriter->parentWriter!=NULL ){
sl@0	4262	interiorWriterDestroy(pWriter->parentWriter);
sl@0	4263	sqlite3_free(pWriter->parentWriter);
sl@0	4264	}
sl@0	4265	dataBufferDestroy(&pWriter->term);
sl@0	4266	SCRAMBLE(pWriter);
sl@0	4267	return SQLITE_OK;
sl@0	4268	}
sl@0	4269
sl@0	4270	/* If pWriter can fit entirely in ROOT_MAX, return it as the root info
sl@0	4271	** directly, leaving *piEndBlockid unchanged. Otherwise, flush
sl@0	4272	** pWriter to %_segments, building a new layer of interior nodes, and
sl@0	4273	** recursively ask for their root into.
sl@0	4274	*/
sl@0	4275	static int interiorWriterRootInfo(fulltext_vtab v, InteriorWriter pWriter,
sl@0	4276	char *ppRootInfo, int pnRootInfo,
sl@0	4277	sqlite_int64 *piEndBlockid){
sl@0	4278	InteriorBlock *block = pWriter->first;
sl@0	4279	sqlite_int64 iBlockid = 0;
sl@0	4280	int rc;
sl@0	4281
sl@0	4282	/* If we can fit the segment inline */
sl@0	4283	if( block==pWriter->last && block->data.nData<ROOT_MAX ){
sl@0	4284	*ppRootInfo = block->data.pData;
sl@0	4285	*pnRootInfo = block->data.nData;
sl@0	4286	return SQLITE_OK;
sl@0	4287	}
sl@0	4288
sl@0	4289	/* Flush the first block to %_segments, and create a new level of
sl@0	4290	** interior node.
sl@0	4291	*/
sl@0	4292	ASSERT_VALID_INTERIOR_BLOCK(block);
sl@0	4293	rc = block_insert(v, block->data.pData, block->data.nData, &iBlockid);
sl@0	4294	if( rc!=SQLITE_OK ) return rc;
sl@0	4295	*piEndBlockid = iBlockid;
sl@0	4296
sl@0	4297	pWriter->parentWriter = sqlite3_malloc(sizeof(*pWriter->parentWriter));
sl@0	4298	interiorWriterInit(pWriter->iHeight+1,
sl@0	4299	block->term.pData, block->term.nData,
sl@0	4300	iBlockid, pWriter->parentWriter);
sl@0	4301
sl@0	4302	/* Flush additional blocks and append to the higher interior
sl@0	4303	** node.
sl@0	4304	*/
sl@0	4305	for(block=block->next; block!=NULL; block=block->next){
sl@0	4306	ASSERT_VALID_INTERIOR_BLOCK(block);
sl@0	4307	rc = block_insert(v, block->data.pData, block->data.nData, &iBlockid);
sl@0	4308	if( rc!=SQLITE_OK ) return rc;
sl@0	4309	*piEndBlockid = iBlockid;
sl@0	4310
sl@0	4311	interiorWriterAppend(pWriter->parentWriter,
sl@0	4312	block->term.pData, block->term.nData, iBlockid);
sl@0	4313	}
sl@0	4314
sl@0	4315	/* Parent node gets the chance to be the root. */
sl@0	4316	return interiorWriterRootInfo(v, pWriter->parentWriter,
sl@0	4317	ppRootInfo, pnRootInfo, piEndBlockid);
sl@0	4318	}
sl@0	4319
sl@0	4320	/****************************************************************/
sl@0	4321	/* InteriorReader is used to read off the data from an interior node
sl@0	4322	** (see comment at top of file for the format).
sl@0	4323	*/
sl@0	4324	typedef struct InteriorReader {
sl@0	4325	const char *pData;
sl@0	4326	int nData;
sl@0	4327
sl@0	4328	DataBuffer term; /* previous term, for decoding term delta. */
sl@0	4329
sl@0	4330	sqlite_int64 iBlockid;
sl@0	4331	} InteriorReader;
sl@0	4332
sl@0	4333	static void interiorReaderDestroy(InteriorReader *pReader){
sl@0	4334	dataBufferDestroy(&pReader->term);
sl@0	4335	SCRAMBLE(pReader);
sl@0	4336	}
sl@0	4337
sl@0	4338	/* TODO(shess) The assertions are great, but what if we're in NDEBUG
sl@0	4339	** and the blob is empty or otherwise contains suspect data?
sl@0	4340	*/
sl@0	4341	static void interiorReaderInit(const char *pData, int nData,
sl@0	4342	InteriorReader *pReader){
sl@0	4343	int n, nTerm;
sl@0	4344
sl@0	4345	/* Require at least the leading flag byte */
sl@0	4346	assert( nData>0 );
sl@0	4347	assert( pData[0]!='\0' );
sl@0	4348
sl@0	4349	CLEAR(pReader);
sl@0	4350
sl@0	4351	/* Decode the base blockid, and set the cursor to the first term. */
sl@0	4352	n = getVarint(pData+1, &pReader->iBlockid);
sl@0	4353	assert( 1+n<=nData );
sl@0	4354	pReader->pData = pData+1+n;
sl@0	4355	pReader->nData = nData-(1+n);
sl@0	4356
sl@0	4357	/* A single-child interior node (such as when a leaf node was too
sl@0	4358	** large for the segment directory) won't have any terms.
sl@0	4359	** Otherwise, decode the first term.
sl@0	4360	*/
sl@0	4361	if( pReader->nData==0 ){
sl@0	4362	dataBufferInit(&pReader->term, 0);
sl@0	4363	}else{
sl@0	4364	n = getVarint32(pReader->pData, &nTerm);
sl@0	4365	dataBufferInit(&pReader->term, nTerm);
sl@0	4366	dataBufferReplace(&pReader->term, pReader->pData+n, nTerm);
sl@0	4367	assert( n+nTerm<=pReader->nData );
sl@0	4368	pReader->pData += n+nTerm;
sl@0	4369	pReader->nData -= n+nTerm;
sl@0	4370	}
sl@0	4371	}
sl@0	4372
sl@0	4373	static int interiorReaderAtEnd(InteriorReader *pReader){
sl@0	4374	return pReader->term.nData==0;
sl@0	4375	}
sl@0	4376
sl@0	4377	static sqlite_int64 interiorReaderCurrentBlockid(InteriorReader *pReader){
sl@0	4378	return pReader->iBlockid;
sl@0	4379	}
sl@0	4380
sl@0	4381	static int interiorReaderTermBytes(InteriorReader *pReader){
sl@0	4382	assert( !interiorReaderAtEnd(pReader) );
sl@0	4383	return pReader->term.nData;
sl@0	4384	}
sl@0	4385	static const char interiorReaderTerm(InteriorReader pReader){
sl@0	4386	assert( !interiorReaderAtEnd(pReader) );
sl@0	4387	return pReader->term.pData;
sl@0	4388	}
sl@0	4389
sl@0	4390	/* Step forward to the next term in the node. */
sl@0	4391	static void interiorReaderStep(InteriorReader *pReader){
sl@0	4392	assert( !interiorReaderAtEnd(pReader) );
sl@0	4393
sl@0	4394	/* If the last term has been read, signal eof, else construct the
sl@0	4395	** next term.
sl@0	4396	*/
sl@0	4397	if( pReader->nData==0 ){
sl@0	4398	dataBufferReset(&pReader->term);
sl@0	4399	}else{
sl@0	4400	int n, nPrefix, nSuffix;
sl@0	4401
sl@0	4402	n = getVarint32(pReader->pData, &nPrefix);
sl@0	4403	n += getVarint32(pReader->pData+n, &nSuffix);
sl@0	4404
sl@0	4405	/* Truncate the current term and append suffix data. */
sl@0	4406	pReader->term.nData = nPrefix;
sl@0	4407	dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix);
sl@0	4408
sl@0	4409	assert( n+nSuffix<=pReader->nData );
sl@0	4410	pReader->pData += n+nSuffix;
sl@0	4411	pReader->nData -= n+nSuffix;
sl@0	4412	}
sl@0	4413	pReader->iBlockid++;
sl@0	4414	}
sl@0	4415
sl@0	4416	/* Compare the current term to pTerm[nTerm], returning strcmp-style
sl@0	4417	** results. If isPrefix, equality means equal through nTerm bytes.
sl@0	4418	*/
sl@0	4419	static int interiorReaderTermCmp(InteriorReader *pReader,
sl@0	4420	const char *pTerm, int nTerm, int isPrefix){
sl@0	4421	const char *pReaderTerm = interiorReaderTerm(pReader);
sl@0	4422	int nReaderTerm = interiorReaderTermBytes(pReader);
sl@0	4423	int c, n = nReaderTerm<nTerm ? nReaderTerm : nTerm;
sl@0	4424
sl@0	4425	if( n==0 ){
sl@0	4426	if( nReaderTerm>0 ) return -1;
sl@0	4427	if( nTerm>0 ) return 1;
sl@0	4428	return 0;
sl@0	4429	}
sl@0	4430
sl@0	4431	c = memcmp(pReaderTerm, pTerm, n);
sl@0	4432	if( c!=0 ) return c;
sl@0	4433	if( isPrefix && n==nTerm ) return 0;
sl@0	4434	return nReaderTerm - nTerm;
sl@0	4435	}
sl@0	4436
sl@0	4437	/****************************************************************/
sl@0	4438	/* LeafWriter is used to collect terms and associated doclist data
sl@0	4439	** into leaf blocks in %_segments (see top of file for format info).
sl@0	4440	** Expected usage is:
sl@0	4441	**
sl@0	4442	** LeafWriter writer;
sl@0	4443	** leafWriterInit(0, 0, &writer);
sl@0	4444	** while( sorted_terms_left_to_process ){
sl@0	4445	** // data is doclist data for that term.
sl@0	4446	** rc = leafWriterStep(v, &writer, pTerm, nTerm, pData, nData);
sl@0	4447	** if( rc!=SQLITE_OK ) goto err;
sl@0	4448	** }
sl@0	4449	** rc = leafWriterFinalize(v, &writer);
sl@0	4450	**err:
sl@0	4451	** leafWriterDestroy(&writer);
sl@0	4452	** return rc;
sl@0	4453	**
sl@0	4454	** leafWriterStep() may write a collected leaf out to %_segments.
sl@0	4455	** leafWriterFinalize() finishes writing any buffered data and stores
sl@0	4456	** a root node in %_segdir. leafWriterDestroy() frees all buffers and
sl@0	4457	** InteriorWriters allocated as part of writing this segment.
sl@0	4458	**
sl@0	4459	** TODO(shess) Document leafWriterStepMerge().
sl@0	4460	*/
sl@0	4461
sl@0	4462	/* Put terms with data this big in their own block. */
sl@0	4463	#define STANDALONE_MIN 1024
sl@0	4464
sl@0	4465	/* Keep leaf blocks below this size. */
sl@0	4466	#define LEAF_MAX 2048
sl@0	4467
sl@0	4468	typedef struct LeafWriter {
sl@0	4469	int iLevel;
sl@0	4470	int idx;
sl@0	4471	sqlite_int64 iStartBlockid; /* needed to create the root info */
sl@0	4472	sqlite_int64 iEndBlockid; /* when we're done writing. */
sl@0	4473
sl@0	4474	DataBuffer term; /* previous encoded term */
sl@0	4475	DataBuffer data; /* encoding buffer */
sl@0	4476
sl@0	4477	/* bytes of first term in the current node which distinguishes that
sl@0	4478	** term from the last term of the previous node.
sl@0	4479	*/
sl@0	4480	int nTermDistinct;
sl@0	4481
sl@0	4482	InteriorWriter parentWriter; /* if we overflow */
sl@0	4483	int has_parent;
sl@0	4484	} LeafWriter;
sl@0	4485
sl@0	4486	static void leafWriterInit(int iLevel, int idx, LeafWriter *pWriter){
sl@0	4487	CLEAR(pWriter);
sl@0	4488	pWriter->iLevel = iLevel;
sl@0	4489	pWriter->idx = idx;
sl@0	4490
sl@0	4491	dataBufferInit(&pWriter->term, 32);
sl@0	4492
sl@0	4493	/* Start out with a reasonably sized block, though it can grow. */
sl@0	4494	dataBufferInit(&pWriter->data, LEAF_MAX);
sl@0	4495	}
sl@0	4496
sl@0	4497	#ifndef NDEBUG
sl@0	4498	/* Verify that the data is readable as a leaf node. */
sl@0	4499	static void leafNodeValidate(const char *pData, int nData){
sl@0	4500	int n, iDummy;
sl@0	4501
sl@0	4502	if( nData==0 ) return;
sl@0	4503	assert( nData>0 );
sl@0	4504	assert( pData!=0 );
sl@0	4505	assert( pData+nData>pData );
sl@0	4506
sl@0	4507	/* Must lead with a varint(0) */
sl@0	4508	n = getVarint32(pData, &iDummy);
sl@0	4509	assert( iDummy==0 );
sl@0	4510	assert( n>0 );
sl@0	4511	assert( n<nData );
sl@0	4512	pData += n;
sl@0	4513	nData -= n;
sl@0	4514
sl@0	4515	/* Leading term length and data must fit in buffer. */
sl@0	4516	n = getVarint32(pData, &iDummy);
sl@0	4517	assert( n>0 );
sl@0	4518	assert( iDummy>0 );
sl@0	4519	assert( n+iDummy>0 );
sl@0	4520	assert( n+iDummy<nData );
sl@0	4521	pData += n+iDummy;
sl@0	4522	nData -= n+iDummy;
sl@0	4523
sl@0	4524	/* Leading term's doclist length and data must fit. */
sl@0	4525	n = getVarint32(pData, &iDummy);
sl@0	4526	assert( n>0 );
sl@0	4527	assert( iDummy>0 );
sl@0	4528	assert( n+iDummy>0 );
sl@0	4529	assert( n+iDummy<=nData );
sl@0	4530	ASSERT_VALID_DOCLIST(DL_DEFAULT, pData+n, iDummy, NULL);
sl@0	4531	pData += n+iDummy;
sl@0	4532	nData -= n+iDummy;
sl@0	4533
sl@0	4534	/* Verify that trailing terms and doclists also are readable. */
sl@0	4535	while( nData!=0 ){
sl@0	4536	n = getVarint32(pData, &iDummy);
sl@0	4537	assert( n>0 );
sl@0	4538	assert( iDummy>=0 );
sl@0	4539	assert( n<nData );
sl@0	4540	pData += n;
sl@0	4541	nData -= n;
sl@0	4542	n = getVarint32(pData, &iDummy);
sl@0	4543	assert( n>0 );
sl@0	4544	assert( iDummy>0 );
sl@0	4545	assert( n+iDummy>0 );
sl@0	4546	assert( n+iDummy<nData );
sl@0	4547	pData += n+iDummy;
sl@0	4548	nData -= n+iDummy;
sl@0	4549
sl@0	4550	n = getVarint32(pData, &iDummy);
sl@0	4551	assert( n>0 );
sl@0	4552	assert( iDummy>0 );
sl@0	4553	assert( n+iDummy>0 );
sl@0	4554	assert( n+iDummy<=nData );
sl@0	4555	ASSERT_VALID_DOCLIST(DL_DEFAULT, pData+n, iDummy, NULL);
sl@0	4556	pData += n+iDummy;
sl@0	4557	nData -= n+iDummy;
sl@0	4558	}
sl@0	4559	}
sl@0	4560	#define ASSERT_VALID_LEAF_NODE(p, n) leafNodeValidate(p, n)
sl@0	4561	#else
sl@0	4562	#define ASSERT_VALID_LEAF_NODE(p, n) assert( 1 )
sl@0	4563	#endif
sl@0	4564
sl@0	4565	/* Flush the current leaf node to %_segments, and adding the resulting
sl@0	4566	** blockid and the starting term to the interior node which will
sl@0	4567	** contain it.
sl@0	4568	*/
sl@0	4569	static int leafWriterInternalFlush(fulltext_vtab v, LeafWriter pWriter,
sl@0	4570	int iData, int nData){
sl@0	4571	sqlite_int64 iBlockid = 0;
sl@0	4572	const char *pStartingTerm;
sl@0	4573	int nStartingTerm, rc, n;
sl@0	4574
sl@0	4575	/* Must have the leading varint(0) flag, plus at least some
sl@0	4576	** valid-looking data.
sl@0	4577	*/
sl@0	4578	assert( nData>2 );
sl@0	4579	assert( iData>=0 );
sl@0	4580	assert( iData+nData<=pWriter->data.nData );
sl@0	4581	ASSERT_VALID_LEAF_NODE(pWriter->data.pData+iData, nData);
sl@0	4582
sl@0	4583	rc = block_insert(v, pWriter->data.pData+iData, nData, &iBlockid);
sl@0	4584	if( rc!=SQLITE_OK ) return rc;
sl@0	4585	assert( iBlockid!=0 );
sl@0	4586
sl@0	4587	/* Reconstruct the first term in the leaf for purposes of building
sl@0	4588	** the interior node.
sl@0	4589	*/
sl@0	4590	n = getVarint32(pWriter->data.pData+iData+1, &nStartingTerm);
sl@0	4591	pStartingTerm = pWriter->data.pData+iData+1+n;
sl@0	4592	assert( pWriter->data.nData>iData+1+n+nStartingTerm );
sl@0	4593	assert( pWriter->nTermDistinct>0 );
sl@0	4594	assert( pWriter->nTermDistinct<=nStartingTerm );
sl@0	4595	nStartingTerm = pWriter->nTermDistinct;
sl@0	4596
sl@0	4597	if( pWriter->has_parent ){
sl@0	4598	interiorWriterAppend(&pWriter->parentWriter,
sl@0	4599	pStartingTerm, nStartingTerm, iBlockid);
sl@0	4600	}else{
sl@0	4601	interiorWriterInit(1, pStartingTerm, nStartingTerm, iBlockid,
sl@0	4602	&pWriter->parentWriter);
sl@0	4603	pWriter->has_parent = 1;
sl@0	4604	}
sl@0	4605
sl@0	4606	/* Track the span of this segment's leaf nodes. */
sl@0	4607	if( pWriter->iEndBlockid==0 ){
sl@0	4608	pWriter->iEndBlockid = pWriter->iStartBlockid = iBlockid;
sl@0	4609	}else{
sl@0	4610	pWriter->iEndBlockid++;
sl@0	4611	assert( iBlockid==pWriter->iEndBlockid );
sl@0	4612	}
sl@0	4613
sl@0	4614	return SQLITE_OK;
sl@0	4615	}
sl@0	4616	static int leafWriterFlush(fulltext_vtab v, LeafWriter pWriter){
sl@0	4617	int rc = leafWriterInternalFlush(v, pWriter, 0, pWriter->data.nData);
sl@0	4618	if( rc!=SQLITE_OK ) return rc;
sl@0	4619
sl@0	4620	/* Re-initialize the output buffer. */
sl@0	4621	dataBufferReset(&pWriter->data);
sl@0	4622
sl@0	4623	return SQLITE_OK;
sl@0	4624	}
sl@0	4625
sl@0	4626	/* Fetch the root info for the segment. If the entire leaf fits
sl@0	4627	** within ROOT_MAX, then it will be returned directly, otherwise it
sl@0	4628	** will be flushed and the root info will be returned from the
sl@0	4629	** interior node. *piEndBlockid is set to the blockid of the last
sl@0	4630	** interior or leaf node written to disk (0 if none are written at
sl@0	4631	** all).
sl@0	4632	*/
sl@0	4633	static int leafWriterRootInfo(fulltext_vtab v, LeafWriter pWriter,
sl@0	4634	char *ppRootInfo, int pnRootInfo,
sl@0	4635	sqlite_int64 *piEndBlockid){
sl@0	4636	/* we can fit the segment entirely inline */
sl@0	4637	if( !pWriter->has_parent && pWriter->data.nData<ROOT_MAX ){
sl@0	4638	*ppRootInfo = pWriter->data.pData;
sl@0	4639	*pnRootInfo = pWriter->data.nData;
sl@0	4640	*piEndBlockid = 0;
sl@0	4641	return SQLITE_OK;
sl@0	4642	}
sl@0	4643
sl@0	4644	/* Flush remaining leaf data. */
sl@0	4645	if( pWriter->data.nData>0 ){
sl@0	4646	int rc = leafWriterFlush(v, pWriter);
sl@0	4647	if( rc!=SQLITE_OK ) return rc;
sl@0	4648	}
sl@0	4649
sl@0	4650	/* We must have flushed a leaf at some point. */
sl@0	4651	assert( pWriter->has_parent );
sl@0	4652
sl@0	4653	/* Tenatively set the end leaf blockid as the end blockid. If the
sl@0	4654	** interior node can be returned inline, this will be the final
sl@0	4655	** blockid, otherwise it will be overwritten by
sl@0	4656	** interiorWriterRootInfo().
sl@0	4657	*/
sl@0	4658	*piEndBlockid = pWriter->iEndBlockid;
sl@0	4659
sl@0	4660	return interiorWriterRootInfo(v, &pWriter->parentWriter,
sl@0	4661	ppRootInfo, pnRootInfo, piEndBlockid);
sl@0	4662	}
sl@0	4663
sl@0	4664	/* Collect the rootInfo data and store it into the segment directory.
sl@0	4665	** This has the effect of flushing the segment's leaf data to
sl@0	4666	** %_segments, and also flushing any interior nodes to %_segments.
sl@0	4667	*/
sl@0	4668	static int leafWriterFinalize(fulltext_vtab v, LeafWriter pWriter){
sl@0	4669	sqlite_int64 iEndBlockid;
sl@0	4670	char *pRootInfo;
sl@0	4671	int rc, nRootInfo;
sl@0	4672
sl@0	4673	rc = leafWriterRootInfo(v, pWriter, &pRootInfo, &nRootInfo, &iEndBlockid);
sl@0	4674	if( rc!=SQLITE_OK ) return rc;
sl@0	4675
sl@0	4676	/* Don't bother storing an entirely empty segment. */
sl@0	4677	if( iEndBlockid==0 && nRootInfo==0 ) return SQLITE_OK;
sl@0	4678
sl@0	4679	return segdir_set(v, pWriter->iLevel, pWriter->idx,
sl@0	4680	pWriter->iStartBlockid, pWriter->iEndBlockid,
sl@0	4681	iEndBlockid, pRootInfo, nRootInfo);
sl@0	4682	}
sl@0	4683
sl@0	4684	static void leafWriterDestroy(LeafWriter *pWriter){
sl@0	4685	if( pWriter->has_parent ) interiorWriterDestroy(&pWriter->parentWriter);
sl@0	4686	dataBufferDestroy(&pWriter->term);
sl@0	4687	dataBufferDestroy(&pWriter->data);
sl@0	4688	}
sl@0	4689
sl@0	4690	/* Encode a term into the leafWriter, delta-encoding as appropriate.
sl@0	4691	** Returns the length of the new term which distinguishes it from the
sl@0	4692	** previous term, which can be used to set nTermDistinct when a node
sl@0	4693	** boundary is crossed.
sl@0	4694	*/
sl@0	4695	static int leafWriterEncodeTerm(LeafWriter *pWriter,
sl@0	4696	const char *pTerm, int nTerm){
sl@0	4697	char c[VARINT_MAX+VARINT_MAX];
sl@0	4698	int n, nPrefix = 0;
sl@0	4699
sl@0	4700	assert( nTerm>0 );
sl@0	4701	while( nPrefix<pWriter->term.nData &&
sl@0	4702	pTerm[nPrefix]==pWriter->term.pData[nPrefix] ){
sl@0	4703	nPrefix++;
sl@0	4704	/* Failing this implies that the terms weren't in order. */
sl@0	4705	assert( nPrefix<nTerm );
sl@0	4706	}
sl@0	4707
sl@0	4708	if( pWriter->data.nData==0 ){
sl@0	4709	/* Encode the node header and leading term as:
sl@0	4710	** varint(0)
sl@0	4711	** varint(nTerm)
sl@0	4712	** char pTerm[nTerm]
sl@0	4713	*/
sl@0	4714	n = putVarint(c, '\0');
sl@0	4715	n += putVarint(c+n, nTerm);
sl@0	4716	dataBufferAppend2(&pWriter->data, c, n, pTerm, nTerm);
sl@0	4717	}else{
sl@0	4718	/* Delta-encode the term as:
sl@0	4719	** varint(nPrefix)
sl@0	4720	** varint(nSuffix)
sl@0	4721	** char pTermSuffix[nSuffix]
sl@0	4722	*/
sl@0	4723	n = putVarint(c, nPrefix);
sl@0	4724	n += putVarint(c+n, nTerm-nPrefix);
sl@0	4725	dataBufferAppend2(&pWriter->data, c, n, pTerm+nPrefix, nTerm-nPrefix);
sl@0	4726	}
sl@0	4727	dataBufferReplace(&pWriter->term, pTerm, nTerm);
sl@0	4728
sl@0	4729	return nPrefix+1;
sl@0	4730	}
sl@0	4731
sl@0	4732	/* Used to avoid a memmove when a large amount of doclist data is in
sl@0	4733	** the buffer. This constructs a node and term header before
sl@0	4734	** iDoclistData and flushes the resulting complete node using
sl@0	4735	** leafWriterInternalFlush().
sl@0	4736	*/
sl@0	4737	static int leafWriterInlineFlush(fulltext_vtab v, LeafWriter pWriter,
sl@0	4738	const char *pTerm, int nTerm,
sl@0	4739	int iDoclistData){
sl@0	4740	char c[VARINT_MAX+VARINT_MAX];
sl@0	4741	int iData, n = putVarint(c, 0);
sl@0	4742	n += putVarint(c+n, nTerm);
sl@0	4743
sl@0	4744	/* There should always be room for the header. Even if pTerm shared
sl@0	4745	** a substantial prefix with the previous term, the entire prefix
sl@0	4746	** could be constructed from earlier data in the doclist, so there
sl@0	4747	** should be room.
sl@0	4748	*/
sl@0	4749	assert( iDoclistData>=n+nTerm );
sl@0	4750
sl@0	4751	iData = iDoclistData-(n+nTerm);
sl@0	4752	memcpy(pWriter->data.pData+iData, c, n);
sl@0	4753	memcpy(pWriter->data.pData+iData+n, pTerm, nTerm);
sl@0	4754
sl@0	4755	return leafWriterInternalFlush(v, pWriter, iData, pWriter->data.nData-iData);
sl@0	4756	}
sl@0	4757
sl@0	4758	/* Push pTerm[nTerm] along with the doclist data to the leaf layer of
sl@0	4759	** %_segments.
sl@0	4760	*/
sl@0	4761	static int leafWriterStepMerge(fulltext_vtab v, LeafWriter pWriter,
sl@0	4762	const char *pTerm, int nTerm,
sl@0	4763	DLReader *pReaders, int nReaders){
sl@0	4764	char c[VARINT_MAX+VARINT_MAX];
sl@0	4765	int iTermData = pWriter->data.nData, iDoclistData;
sl@0	4766	int i, nData, n, nActualData, nActual, rc, nTermDistinct;
sl@0	4767
sl@0	4768	ASSERT_VALID_LEAF_NODE(pWriter->data.pData, pWriter->data.nData);
sl@0	4769	nTermDistinct = leafWriterEncodeTerm(pWriter, pTerm, nTerm);
sl@0	4770
sl@0	4771	/* Remember nTermDistinct if opening a new node. */
sl@0	4772	if( iTermData==0 ) pWriter->nTermDistinct = nTermDistinct;
sl@0	4773
sl@0	4774	iDoclistData = pWriter->data.nData;
sl@0	4775
sl@0	4776	/* Estimate the length of the merged doclist so we can leave space
sl@0	4777	** to encode it.
sl@0	4778	*/
sl@0	4779	for(i=0, nData=0; i<nReaders; i++){
sl@0	4780	nData += dlrAllDataBytes(&pReaders[i]);
sl@0	4781	}
sl@0	4782	n = putVarint(c, nData);
sl@0	4783	dataBufferAppend(&pWriter->data, c, n);
sl@0	4784
sl@0	4785	docListMerge(&pWriter->data, pReaders, nReaders);
sl@0	4786	ASSERT_VALID_DOCLIST(DL_DEFAULT,
sl@0	4787	pWriter->data.pData+iDoclistData+n,
sl@0	4788	pWriter->data.nData-iDoclistData-n, NULL);
sl@0	4789
sl@0	4790	/* The actual amount of doclist data at this point could be smaller
sl@0	4791	** than the length we encoded. Additionally, the space required to
sl@0	4792	** encode this length could be smaller. For small doclists, this is
sl@0	4793	** not a big deal, we can just use memmove() to adjust things.
sl@0	4794	*/
sl@0	4795	nActualData = pWriter->data.nData-(iDoclistData+n);
sl@0	4796	nActual = putVarint(c, nActualData);
sl@0	4797	assert( nActualData<=nData );
sl@0	4798	assert( nActual<=n );
sl@0	4799
sl@0	4800	/* If the new doclist is big enough for force a standalone leaf
sl@0	4801	** node, we can immediately flush it inline without doing the
sl@0	4802	** memmove().
sl@0	4803	*/
sl@0	4804	/* TODO(shess) This test matches leafWriterStep(), which does this
sl@0	4805	** test before it knows the cost to varint-encode the term and
sl@0	4806	** doclist lengths. At some point, change to
sl@0	4807	** pWriter->data.nData-iTermData>STANDALONE_MIN.
sl@0	4808	*/
sl@0	4809	if( nTerm+nActualData>STANDALONE_MIN ){
sl@0	4810	/* Push leaf node from before this term. */
sl@0	4811	if( iTermData>0 ){
sl@0	4812	rc = leafWriterInternalFlush(v, pWriter, 0, iTermData);
sl@0	4813	if( rc!=SQLITE_OK ) return rc;
sl@0	4814
sl@0	4815	pWriter->nTermDistinct = nTermDistinct;
sl@0	4816	}
sl@0	4817
sl@0	4818	/* Fix the encoded doclist length. */
sl@0	4819	iDoclistData += n - nActual;
sl@0	4820	memcpy(pWriter->data.pData+iDoclistData, c, nActual);
sl@0	4821
sl@0	4822	/* Push the standalone leaf node. */
sl@0	4823	rc = leafWriterInlineFlush(v, pWriter, pTerm, nTerm, iDoclistData);
sl@0	4824	if( rc!=SQLITE_OK ) return rc;
sl@0	4825
sl@0	4826	/* Leave the node empty. */
sl@0	4827	dataBufferReset(&pWriter->data);
sl@0	4828
sl@0	4829	return rc;
sl@0	4830	}
sl@0	4831
sl@0	4832	/* At this point, we know that the doclist was small, so do the
sl@0	4833	** memmove if indicated.
sl@0	4834	*/
sl@0	4835	if( nActual<n ){
sl@0	4836	memmove(pWriter->data.pData+iDoclistData+nActual,
sl@0	4837	pWriter->data.pData+iDoclistData+n,
sl@0	4838	pWriter->data.nData-(iDoclistData+n));
sl@0	4839	pWriter->data.nData -= n-nActual;
sl@0	4840	}
sl@0	4841
sl@0	4842	/* Replace written length with actual length. */
sl@0	4843	memcpy(pWriter->data.pData+iDoclistData, c, nActual);
sl@0	4844
sl@0	4845	/* If the node is too large, break things up. */
sl@0	4846	/* TODO(shess) This test matches leafWriterStep(), which does this
sl@0	4847	** test before it knows the cost to varint-encode the term and
sl@0	4848	** doclist lengths. At some point, change to
sl@0	4849	** pWriter->data.nData>LEAF_MAX.
sl@0	4850	*/
sl@0	4851	if( iTermData+nTerm+nActualData>LEAF_MAX ){
sl@0	4852	/* Flush out the leading data as a node */
sl@0	4853	rc = leafWriterInternalFlush(v, pWriter, 0, iTermData);
sl@0	4854	if( rc!=SQLITE_OK ) return rc;
sl@0	4855
sl@0	4856	pWriter->nTermDistinct = nTermDistinct;
sl@0	4857
sl@0	4858	/* Rebuild header using the current term */
sl@0	4859	n = putVarint(pWriter->data.pData, 0);
sl@0	4860	n += putVarint(pWriter->data.pData+n, nTerm);
sl@0	4861	memcpy(pWriter->data.pData+n, pTerm, nTerm);
sl@0	4862	n += nTerm;
sl@0	4863
sl@0	4864	/* There should always be room, because the previous encoding
sl@0	4865	** included all data necessary to construct the term.
sl@0	4866	*/
sl@0	4867	assert( n<iDoclistData );
sl@0	4868	/* So long as STANDALONE_MIN is half or less of LEAF_MAX, the
sl@0	4869	** following memcpy() is safe (as opposed to needing a memmove).
sl@0	4870	*/
sl@0	4871	assert( 2*STANDALONE_MIN<=LEAF_MAX );
sl@0	4872	assert( n+pWriter->data.nData-iDoclistData<iDoclistData );
sl@0	4873	memcpy(pWriter->data.pData+n,
sl@0	4874	pWriter->data.pData+iDoclistData,
sl@0	4875	pWriter->data.nData-iDoclistData);
sl@0	4876	pWriter->data.nData -= iDoclistData-n;
sl@0	4877	}
sl@0	4878	ASSERT_VALID_LEAF_NODE(pWriter->data.pData, pWriter->data.nData);
sl@0	4879
sl@0	4880	return SQLITE_OK;
sl@0	4881	}
sl@0	4882
sl@0	4883	/* Push pTerm[nTerm] along with the doclist data to the leaf layer of
sl@0	4884	** %_segments.
sl@0	4885	*/
sl@0	4886	/* TODO(shess) Revise writeZeroSegment() so that doclists are
sl@0	4887	** constructed directly in pWriter->data.
sl@0	4888	*/
sl@0	4889	static int leafWriterStep(fulltext_vtab v, LeafWriter pWriter,
sl@0	4890	const char *pTerm, int nTerm,
sl@0	4891	const char *pData, int nData){
sl@0	4892	int rc;
sl@0	4893	DLReader reader;
sl@0	4894
sl@0	4895	dlrInit(&reader, DL_DEFAULT, pData, nData);
sl@0	4896	rc = leafWriterStepMerge(v, pWriter, pTerm, nTerm, &reader, 1);
sl@0	4897	dlrDestroy(&reader);
sl@0	4898
sl@0	4899	return rc;
sl@0	4900	}
sl@0	4901
sl@0	4902
sl@0	4903	/****************************************************************/
sl@0	4904	/* LeafReader is used to iterate over an individual leaf node. */
sl@0	4905	typedef struct LeafReader {
sl@0	4906	DataBuffer term; /* copy of current term. */
sl@0	4907
sl@0	4908	const char pData; / data for current term. */
sl@0	4909	int nData;
sl@0	4910	} LeafReader;
sl@0	4911
sl@0	4912	static void leafReaderDestroy(LeafReader *pReader){
sl@0	4913	dataBufferDestroy(&pReader->term);
sl@0	4914	SCRAMBLE(pReader);
sl@0	4915	}
sl@0	4916
sl@0	4917	static int leafReaderAtEnd(LeafReader *pReader){
sl@0	4918	return pReader->nData<=0;
sl@0	4919	}
sl@0	4920
sl@0	4921	/* Access the current term. */
sl@0	4922	static int leafReaderTermBytes(LeafReader *pReader){
sl@0	4923	return pReader->term.nData;
sl@0	4924	}
sl@0	4925	static const char leafReaderTerm(LeafReader pReader){
sl@0	4926	assert( pReader->term.nData>0 );
sl@0	4927	return pReader->term.pData;
sl@0	4928	}
sl@0	4929
sl@0	4930	/* Access the doclist data for the current term. */
sl@0	4931	static int leafReaderDataBytes(LeafReader *pReader){
sl@0	4932	int nData;
sl@0	4933	assert( pReader->term.nData>0 );
sl@0	4934	getVarint32(pReader->pData, &nData);
sl@0	4935	return nData;
sl@0	4936	}
sl@0	4937	static const char leafReaderData(LeafReader pReader){
sl@0	4938	int n, nData;
sl@0	4939	assert( pReader->term.nData>0 );
sl@0	4940	n = getVarint32(pReader->pData, &nData);
sl@0	4941	return pReader->pData+n;
sl@0	4942	}
sl@0	4943
sl@0	4944	static void leafReaderInit(const char *pData, int nData,
sl@0	4945	LeafReader *pReader){
sl@0	4946	int nTerm, n;
sl@0	4947
sl@0	4948	assert( nData>0 );
sl@0	4949	assert( pData[0]=='\0' );
sl@0	4950
sl@0	4951	CLEAR(pReader);
sl@0	4952
sl@0	4953	/* Read the first term, skipping the header byte. */
sl@0	4954	n = getVarint32(pData+1, &nTerm);
sl@0	4955	dataBufferInit(&pReader->term, nTerm);
sl@0	4956	dataBufferReplace(&pReader->term, pData+1+n, nTerm);
sl@0	4957
sl@0	4958	/* Position after the first term. */
sl@0	4959	assert( 1+n+nTerm<nData );
sl@0	4960	pReader->pData = pData+1+n+nTerm;
sl@0	4961	pReader->nData = nData-1-n-nTerm;
sl@0	4962	}
sl@0	4963
sl@0	4964	/* Step the reader forward to the next term. */
sl@0	4965	static void leafReaderStep(LeafReader *pReader){
sl@0	4966	int n, nData, nPrefix, nSuffix;
sl@0	4967	assert( !leafReaderAtEnd(pReader) );
sl@0	4968
sl@0	4969	/* Skip previous entry's data block. */
sl@0	4970	n = getVarint32(pReader->pData, &nData);
sl@0	4971	assert( n+nData<=pReader->nData );
sl@0	4972	pReader->pData += n+nData;
sl@0	4973	pReader->nData -= n+nData;
sl@0	4974
sl@0	4975	if( !leafReaderAtEnd(pReader) ){
sl@0	4976	/* Construct the new term using a prefix from the old term plus a
sl@0	4977	** suffix from the leaf data.
sl@0	4978	*/
sl@0	4979	n = getVarint32(pReader->pData, &nPrefix);
sl@0	4980	n += getVarint32(pReader->pData+n, &nSuffix);
sl@0	4981	assert( n+nSuffix<pReader->nData );
sl@0	4982	pReader->term.nData = nPrefix;
sl@0	4983	dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix);
sl@0	4984
sl@0	4985	pReader->pData += n+nSuffix;
sl@0	4986	pReader->nData -= n+nSuffix;
sl@0	4987	}
sl@0	4988	}
sl@0	4989
sl@0	4990	/* strcmp-style comparison of pReader's current term against pTerm.
sl@0	4991	** If isPrefix, equality means equal through nTerm bytes.
sl@0	4992	*/
sl@0	4993	static int leafReaderTermCmp(LeafReader *pReader,
sl@0	4994	const char *pTerm, int nTerm, int isPrefix){
sl@0	4995	int c, n = pReader->term.nData<nTerm ? pReader->term.nData : nTerm;
sl@0	4996	if( n==0 ){
sl@0	4997	if( pReader->term.nData>0 ) return -1;
sl@0	4998	if(nTerm>0 ) return 1;
sl@0	4999	return 0;
sl@0	5000	}
sl@0	5001
sl@0	5002	c = memcmp(pReader->term.pData, pTerm, n);
sl@0	5003	if( c!=0 ) return c;
sl@0	5004	if( isPrefix && n==nTerm ) return 0;
sl@0	5005	return pReader->term.nData - nTerm;
sl@0	5006	}
sl@0	5007
sl@0	5008
sl@0	5009	/****************************************************************/
sl@0	5010	/* LeavesReader wraps LeafReader to allow iterating over the entire
sl@0	5011	** leaf layer of the tree.
sl@0	5012	*/
sl@0	5013	typedef struct LeavesReader {
sl@0	5014	int idx; /* Index within the segment. */
sl@0	5015
sl@0	5016	sqlite3_stmt pStmt; / Statement we're streaming leaves from. */
sl@0	5017	int eof; /* we've seen SQLITE_DONE from pStmt. */
sl@0	5018
sl@0	5019	LeafReader leafReader; /* reader for the current leaf. */
sl@0	5020	DataBuffer rootData; /* root data for inline. */
sl@0	5021	} LeavesReader;
sl@0	5022
sl@0	5023	/* Access the current term. */
sl@0	5024	static int leavesReaderTermBytes(LeavesReader *pReader){
sl@0	5025	assert( !pReader->eof );
sl@0	5026	return leafReaderTermBytes(&pReader->leafReader);
sl@0	5027	}
sl@0	5028	static const char leavesReaderTerm(LeavesReader pReader){
sl@0	5029	assert( !pReader->eof );
sl@0	5030	return leafReaderTerm(&pReader->leafReader);
sl@0	5031	}
sl@0	5032
sl@0	5033	/* Access the doclist data for the current term. */
sl@0	5034	static int leavesReaderDataBytes(LeavesReader *pReader){
sl@0	5035	assert( !pReader->eof );
sl@0	5036	return leafReaderDataBytes(&pReader->leafReader);
sl@0	5037	}
sl@0	5038	static const char leavesReaderData(LeavesReader pReader){
sl@0	5039	assert( !pReader->eof );
sl@0	5040	return leafReaderData(&pReader->leafReader);
sl@0	5041	}
sl@0	5042
sl@0	5043	static int leavesReaderAtEnd(LeavesReader *pReader){
sl@0	5044	return pReader->eof;
sl@0	5045	}
sl@0	5046
sl@0	5047	/* loadSegmentLeaves() may not read all the way to SQLITE_DONE, thus
sl@0	5048	** leaving the statement handle open, which locks the table.
sl@0	5049	*/
sl@0	5050	/* TODO(shess) This "solution" is not satisfactory. Really, there
sl@0	5051	** should be check-in function for all statement handles which
sl@0	5052	** arranges to call sqlite3_reset(). This most likely will require
sl@0	5053	** modification to control flow all over the place, though, so for now
sl@0	5054	** just punt.
sl@0	5055	**
sl@0	5056	** Note the the current system assumes that segment merges will run to
sl@0	5057	** completion, which is why this particular probably hasn't arisen in
sl@0	5058	** this case. Probably a brittle assumption.
sl@0	5059	*/
sl@0	5060	static int leavesReaderReset(LeavesReader *pReader){
sl@0	5061	return sqlite3_reset(pReader->pStmt);
sl@0	5062	}
sl@0	5063
sl@0	5064	static void leavesReaderDestroy(LeavesReader *pReader){
sl@0	5065	/* If idx is -1, that means we're using a non-cached statement
sl@0	5066	** handle in the optimize() case, so we need to release it.
sl@0	5067	*/
sl@0	5068	if( pReader->pStmt!=NULL && pReader->idx==-1 ){
sl@0	5069	sqlite3_finalize(pReader->pStmt);
sl@0	5070	}
sl@0	5071	leafReaderDestroy(&pReader->leafReader);
sl@0	5072	dataBufferDestroy(&pReader->rootData);
sl@0	5073	SCRAMBLE(pReader);
sl@0	5074	}
sl@0	5075
sl@0	5076	/* Initialize pReader with the given root data (if iStartBlockid==0
sl@0	5077	** the leaf data was entirely contained in the root), or from the
sl@0	5078	** stream of blocks between iStartBlockid and iEndBlockid, inclusive.
sl@0	5079	*/
sl@0	5080	static int leavesReaderInit(fulltext_vtab *v,
sl@0	5081	int idx,
sl@0	5082	sqlite_int64 iStartBlockid,
sl@0	5083	sqlite_int64 iEndBlockid,
sl@0	5084	const char *pRootData, int nRootData,
sl@0	5085	LeavesReader *pReader){
sl@0	5086	CLEAR(pReader);
sl@0	5087	pReader->idx = idx;
sl@0	5088
sl@0	5089	dataBufferInit(&pReader->rootData, 0);
sl@0	5090	if( iStartBlockid==0 ){
sl@0	5091	/* Entire leaf level fit in root data. */
sl@0	5092	dataBufferReplace(&pReader->rootData, pRootData, nRootData);
sl@0	5093	leafReaderInit(pReader->rootData.pData, pReader->rootData.nData,
sl@0	5094	&pReader->leafReader);
sl@0	5095	}else{
sl@0	5096	sqlite3_stmt *s;
sl@0	5097	int rc = sql_get_leaf_statement(v, idx, &s);
sl@0	5098	if( rc!=SQLITE_OK ) return rc;
sl@0	5099
sl@0	5100	rc = sqlite3_bind_int64(s, 1, iStartBlockid);
sl@0	5101	if( rc!=SQLITE_OK ) return rc;
sl@0	5102
sl@0	5103	rc = sqlite3_bind_int64(s, 2, iEndBlockid);
sl@0	5104	if( rc!=SQLITE_OK ) return rc;
sl@0	5105
sl@0	5106	rc = sqlite3_step(s);
sl@0	5107	if( rc==SQLITE_DONE ){
sl@0	5108	pReader->eof = 1;
sl@0	5109	return SQLITE_OK;
sl@0	5110	}
sl@0	5111	if( rc!=SQLITE_ROW ) return rc;
sl@0	5112
sl@0	5113	pReader->pStmt = s;
sl@0	5114	leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
sl@0	5115	sqlite3_column_bytes(pReader->pStmt, 0),
sl@0	5116	&pReader->leafReader);
sl@0	5117	}
sl@0	5118	return SQLITE_OK;
sl@0	5119	}
sl@0	5120
sl@0	5121	/* Step the current leaf forward to the next term. If we reach the
sl@0	5122	** end of the current leaf, step forward to the next leaf block.
sl@0	5123	*/
sl@0	5124	static int leavesReaderStep(fulltext_vtab v, LeavesReader pReader){
sl@0	5125	assert( !leavesReaderAtEnd(pReader) );
sl@0	5126	leafReaderStep(&pReader->leafReader);
sl@0	5127
sl@0	5128	if( leafReaderAtEnd(&pReader->leafReader) ){
sl@0	5129	int rc;
sl@0	5130	if( pReader->rootData.pData ){
sl@0	5131	pReader->eof = 1;
sl@0	5132	return SQLITE_OK;
sl@0	5133	}
sl@0	5134	rc = sqlite3_step(pReader->pStmt);
sl@0	5135	if( rc!=SQLITE_ROW ){
sl@0	5136	pReader->eof = 1;
sl@0	5137	return rc==SQLITE_DONE ? SQLITE_OK : rc;
sl@0	5138	}
sl@0	5139	leafReaderDestroy(&pReader->leafReader);
sl@0	5140	leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
sl@0	5141	sqlite3_column_bytes(pReader->pStmt, 0),
sl@0	5142	&pReader->leafReader);
sl@0	5143	}
sl@0	5144	return SQLITE_OK;
sl@0	5145	}
sl@0	5146
sl@0	5147	/* Order LeavesReaders by their term, ignoring idx. Readers at eof
sl@0	5148	** always sort to the end.
sl@0	5149	*/
sl@0	5150	static int leavesReaderTermCmp(LeavesReader lr1, LeavesReader lr2){
sl@0	5151	if( leavesReaderAtEnd(lr1) ){
sl@0	5152	if( leavesReaderAtEnd(lr2) ) return 0;
sl@0	5153	return 1;
sl@0	5154	}
sl@0	5155	if( leavesReaderAtEnd(lr2) ) return -1;
sl@0	5156
sl@0	5157	return leafReaderTermCmp(&lr1->leafReader,
sl@0	5158	leavesReaderTerm(lr2), leavesReaderTermBytes(lr2),
sl@0	5159	0);
sl@0	5160	}
sl@0	5161
sl@0	5162	/* Similar to leavesReaderTermCmp(), with additional ordering by idx
sl@0	5163	** so that older segments sort before newer segments.
sl@0	5164	*/
sl@0	5165	static int leavesReaderCmp(LeavesReader lr1, LeavesReader lr2){
sl@0	5166	int c = leavesReaderTermCmp(lr1, lr2);
sl@0	5167	if( c!=0 ) return c;
sl@0	5168	return lr1->idx-lr2->idx;
sl@0	5169	}
sl@0	5170
sl@0	5171	/* Assume that pLr[1]..pLr[nLr] are sorted. Bubble pLr[0] into its
sl@0	5172	** sorted position.
sl@0	5173	*/
sl@0	5174	static void leavesReaderReorder(LeavesReader *pLr, int nLr){
sl@0	5175	while( nLr>1 && leavesReaderCmp(pLr, pLr+1)>0 ){
sl@0	5176	LeavesReader tmp = pLr[0];
sl@0	5177	pLr[0] = pLr[1];
sl@0	5178	pLr[1] = tmp;
sl@0	5179	nLr--;
sl@0	5180	pLr++;
sl@0	5181	}
sl@0	5182	}
sl@0	5183
sl@0	5184	/* Initializes pReaders with the segments from level iLevel, returning
sl@0	5185	** the number of segments in *piReaders. Leaves pReaders in sorted
sl@0	5186	** order.
sl@0	5187	*/
sl@0	5188	static int leavesReadersInit(fulltext_vtab *v, int iLevel,
sl@0	5189	LeavesReader pReaders, int piReaders){
sl@0	5190	sqlite3_stmt *s;
sl@0	5191	int i, rc = sql_get_statement(v, SEGDIR_SELECT_LEVEL_STMT, &s);
sl@0	5192	if( rc!=SQLITE_OK ) return rc;
sl@0	5193
sl@0	5194	rc = sqlite3_bind_int(s, 1, iLevel);
sl@0	5195	if( rc!=SQLITE_OK ) return rc;
sl@0	5196
sl@0	5197	i = 0;
sl@0	5198	while( (rc = sqlite3_step(s))==SQLITE_ROW ){
sl@0	5199	sqlite_int64 iStart = sqlite3_column_int64(s, 0);
sl@0	5200	sqlite_int64 iEnd = sqlite3_column_int64(s, 1);
sl@0	5201	const char *pRootData = sqlite3_column_blob(s, 2);
sl@0	5202	int nRootData = sqlite3_column_bytes(s, 2);
sl@0	5203
sl@0	5204	assert( i<MERGE_COUNT );
sl@0	5205	rc = leavesReaderInit(v, i, iStart, iEnd, pRootData, nRootData,
sl@0	5206	&pReaders[i]);
sl@0	5207	if( rc!=SQLITE_OK ) break;
sl@0	5208
sl@0	5209	i++;
sl@0	5210	}
sl@0	5211	if( rc!=SQLITE_DONE ){
sl@0	5212	while( i-->0 ){
sl@0	5213	leavesReaderDestroy(&pReaders[i]);
sl@0	5214	}
sl@0	5215	return rc;
sl@0	5216	}
sl@0	5217
sl@0	5218	*piReaders = i;
sl@0	5219
sl@0	5220	/* Leave our results sorted by term, then age. */
sl@0	5221	while( i-- ){
sl@0	5222	leavesReaderReorder(pReaders+i, *piReaders-i);
sl@0	5223	}
sl@0	5224	return SQLITE_OK;
sl@0	5225	}
sl@0	5226
sl@0	5227	/* Merge doclists from pReaders[nReaders] into a single doclist, which
sl@0	5228	** is written to pWriter. Assumes pReaders is ordered oldest to
sl@0	5229	** newest.
sl@0	5230	*/
sl@0	5231	/* TODO(shess) Consider putting this inline in segmentMerge(). */
sl@0	5232	static int leavesReadersMerge(fulltext_vtab *v,
sl@0	5233	LeavesReader *pReaders, int nReaders,
sl@0	5234	LeafWriter *pWriter){
sl@0	5235	DLReader dlReaders[MERGE_COUNT];
sl@0	5236	const char *pTerm = leavesReaderTerm(pReaders);
sl@0	5237	int i, nTerm = leavesReaderTermBytes(pReaders);
sl@0	5238
sl@0	5239	assert( nReaders<=MERGE_COUNT );
sl@0	5240
sl@0	5241	for(i=0; i<nReaders; i++){
sl@0	5242	dlrInit(&dlReaders[i], DL_DEFAULT,
sl@0	5243	leavesReaderData(pReaders+i),
sl@0	5244	leavesReaderDataBytes(pReaders+i));
sl@0	5245	}
sl@0	5246
sl@0	5247	return leafWriterStepMerge(v, pWriter, pTerm, nTerm, dlReaders, nReaders);
sl@0	5248	}
sl@0	5249
sl@0	5250	/* Forward ref due to mutual recursion with segdirNextIndex(). */
sl@0	5251	static int segmentMerge(fulltext_vtab *v, int iLevel);
sl@0	5252
sl@0	5253	/* Put the next available index at iLevel into *pidx. If iLevel
sl@0	5254	** already has MERGE_COUNT segments, they are merged to a higher
sl@0	5255	** level to make room.
sl@0	5256	*/
sl@0	5257	static int segdirNextIndex(fulltext_vtab v, int iLevel, int pidx){
sl@0	5258	int rc = segdir_max_index(v, iLevel, pidx);
sl@0	5259	if( rc==SQLITE_DONE ){ /* No segments at iLevel. */
sl@0	5260	*pidx = 0;
sl@0	5261	}else if( rc==SQLITE_ROW ){
sl@0	5262	if( *pidx==(MERGE_COUNT-1) ){
sl@0	5263	rc = segmentMerge(v, iLevel);
sl@0	5264	if( rc!=SQLITE_OK ) return rc;
sl@0	5265	*pidx = 0;
sl@0	5266	}else{
sl@0	5267	(*pidx)++;
sl@0	5268	}
sl@0	5269	}else{
sl@0	5270	return rc;
sl@0	5271	}
sl@0	5272	return SQLITE_OK;
sl@0	5273	}
sl@0	5274
sl@0	5275	/* Merge MERGE_COUNT segments at iLevel into a new segment at
sl@0	5276	** iLevel+1. If iLevel+1 is already full of segments, those will be
sl@0	5277	** merged to make room.
sl@0	5278	*/
sl@0	5279	static int segmentMerge(fulltext_vtab *v, int iLevel){
sl@0	5280	LeafWriter writer;
sl@0	5281	LeavesReader lrs[MERGE_COUNT];
sl@0	5282	int i, rc, idx = 0;
sl@0	5283
sl@0	5284	/* Determine the next available segment index at the next level,
sl@0	5285	** merging as necessary.
sl@0	5286	*/
sl@0	5287	rc = segdirNextIndex(v, iLevel+1, &idx);
sl@0	5288	if( rc!=SQLITE_OK ) return rc;
sl@0	5289
sl@0	5290	/* TODO(shess) This assumes that we'll always see exactly
sl@0	5291	** MERGE_COUNT segments to merge at a given level. That will be
sl@0	5292	** broken if we allow the developer to request preemptive or
sl@0	5293	** deferred merging.
sl@0	5294	*/
sl@0	5295	memset(&lrs, '\0', sizeof(lrs));
sl@0	5296	rc = leavesReadersInit(v, iLevel, lrs, &i);
sl@0	5297	if( rc!=SQLITE_OK ) return rc;
sl@0	5298	assert( i==MERGE_COUNT );
sl@0	5299
sl@0	5300	leafWriterInit(iLevel+1, idx, &writer);
sl@0	5301
sl@0	5302	/* Since leavesReaderReorder() pushes readers at eof to the end,
sl@0	5303	** when the first reader is empty, all will be empty.
sl@0	5304	*/
sl@0	5305	while( !leavesReaderAtEnd(lrs) ){
sl@0	5306	/* Figure out how many readers share their next term. */
sl@0	5307	for(i=1; i<MERGE_COUNT && !leavesReaderAtEnd(lrs+i); i++){
sl@0	5308	if( 0!=leavesReaderTermCmp(lrs, lrs+i) ) break;
sl@0	5309	}
sl@0	5310
sl@0	5311	rc = leavesReadersMerge(v, lrs, i, &writer);
sl@0	5312	if( rc!=SQLITE_OK ) goto err;
sl@0	5313
sl@0	5314	/* Step forward those that were merged. */
sl@0	5315	while( i-->0 ){
sl@0	5316	rc = leavesReaderStep(v, lrs+i);
sl@0	5317	if( rc!=SQLITE_OK ) goto err;
sl@0	5318
sl@0	5319	/* Reorder by term, then by age. */
sl@0	5320	leavesReaderReorder(lrs+i, MERGE_COUNT-i);
sl@0	5321	}
sl@0	5322	}
sl@0	5323
sl@0	5324	for(i=0; i<MERGE_COUNT; i++){
sl@0	5325	leavesReaderDestroy(&lrs[i]);
sl@0	5326	}
sl@0	5327
sl@0	5328	rc = leafWriterFinalize(v, &writer);
sl@0	5329	leafWriterDestroy(&writer);
sl@0	5330	if( rc!=SQLITE_OK ) return rc;
sl@0	5331
sl@0	5332	/* Delete the merged segment data. */
sl@0	5333	return segdir_delete(v, iLevel);
sl@0	5334
sl@0	5335	err:
sl@0	5336	for(i=0; i<MERGE_COUNT; i++){
sl@0	5337	leavesReaderDestroy(&lrs[i]);
sl@0	5338	}
sl@0	5339	leafWriterDestroy(&writer);
sl@0	5340	return rc;
sl@0	5341	}
sl@0	5342
sl@0	5343	/* Accumulate the union of acc and pData into acc. /
sl@0	5344	static void docListAccumulateUnion(DataBuffer *acc,
sl@0	5345	const char *pData, int nData) {
sl@0	5346	DataBuffer tmp = *acc;
sl@0	5347	dataBufferInit(acc, tmp.nData+nData);
sl@0	5348	docListUnion(tmp.pData, tmp.nData, pData, nData, acc);
sl@0	5349	dataBufferDestroy(&tmp);
sl@0	5350	}
sl@0	5351
sl@0	5352	/* TODO(shess) It might be interesting to explore different merge
sl@0	5353	** strategies, here. For instance, since this is a sorted merge, we
sl@0	5354	** could easily merge many doclists in parallel. With some
sl@0	5355	** comprehension of the storage format, we could merge all of the
sl@0	5356	** doclists within a leaf node directly from the leaf node's storage.
sl@0	5357	** It may be worthwhile to merge smaller doclists before larger
sl@0	5358	** doclists, since they can be traversed more quickly - but the
sl@0	5359	** results may have less overlap, making them more expensive in a
sl@0	5360	** different way.
sl@0	5361	*/
sl@0	5362
sl@0	5363	/* Scan pReader for pTerm/nTerm, and merge the term's doclist over
sl@0	5364	** out (any doclists with duplicate docids overwrite those in out).
sl@0	5365	** Internal function for loadSegmentLeaf().
sl@0	5366	*/
sl@0	5367	static int loadSegmentLeavesInt(fulltext_vtab v, LeavesReader pReader,
sl@0	5368	const char *pTerm, int nTerm, int isPrefix,
sl@0	5369	DataBuffer *out){
sl@0	5370	/* doclist data is accumulated into pBuffers similar to how one does
sl@0	5371	** increment in binary arithmetic. If index 0 is empty, the data is
sl@0	5372	** stored there. If there is data there, it is merged and the
sl@0	5373	** results carried into position 1, with further merge-and-carry
sl@0	5374	** until an empty position is found.
sl@0	5375	*/
sl@0	5376	DataBuffer *pBuffers = NULL;
sl@0	5377	int nBuffers = 0, nMaxBuffers = 0, rc;
sl@0	5378
sl@0	5379	assert( nTerm>0 );
sl@0	5380
sl@0	5381	for(rc=SQLITE_OK; rc==SQLITE_OK && !leavesReaderAtEnd(pReader);
sl@0	5382	rc=leavesReaderStep(v, pReader)){
sl@0	5383	/* TODO(shess) Really want leavesReaderTermCmp(), but that name is
sl@0	5384	** already taken to compare the terms of two LeavesReaders. Think
sl@0	5385	** on a better name. [Meanwhile, break encapsulation rather than
sl@0	5386	** use a confusing name.]
sl@0	5387	*/
sl@0	5388	int c = leafReaderTermCmp(&pReader->leafReader, pTerm, nTerm, isPrefix);
sl@0	5389	if( c>0 ) break; /* Past any possible matches. */
sl@0	5390	if( c==0 ){
sl@0	5391	const char *pData = leavesReaderData(pReader);
sl@0	5392	int iBuffer, nData = leavesReaderDataBytes(pReader);
sl@0	5393
sl@0	5394	/* Find the first empty buffer. */
sl@0	5395	for(iBuffer=0; iBuffer<nBuffers; ++iBuffer){
sl@0	5396	if( 0==pBuffers[iBuffer].nData ) break;
sl@0	5397	}
sl@0	5398
sl@0	5399	/* Out of buffers, add an empty one. */
sl@0	5400	if( iBuffer==nBuffers ){
sl@0	5401	if( nBuffers==nMaxBuffers ){
sl@0	5402	DataBuffer *p;
sl@0	5403	nMaxBuffers += 20;
sl@0	5404
sl@0	5405	/* Manual realloc so we can handle NULL appropriately. */
sl@0	5406	p = sqlite3_malloc(nMaxBufferssizeof(pBuffers));
sl@0	5407	if( p==NULL ){
sl@0	5408	rc = SQLITE_NOMEM;
sl@0	5409	break;
sl@0	5410	}
sl@0	5411
sl@0	5412	if( nBuffers>0 ){
sl@0	5413	assert(pBuffers!=NULL);
sl@0	5414	memcpy(p, pBuffers, nBufferssizeof(pBuffers));
sl@0	5415	sqlite3_free(pBuffers);
sl@0	5416	}
sl@0	5417	pBuffers = p;
sl@0	5418	}
sl@0	5419	dataBufferInit(&(pBuffers[nBuffers]), 0);
sl@0	5420	nBuffers++;
sl@0	5421	}
sl@0	5422
sl@0	5423	/* At this point, must have an empty at iBuffer. */
sl@0	5424	assert(iBuffer<nBuffers && pBuffers[iBuffer].nData==0);
sl@0	5425
sl@0	5426	/* If empty was first buffer, no need for merge logic. */
sl@0	5427	if( iBuffer==0 ){
sl@0	5428	dataBufferReplace(&(pBuffers[0]), pData, nData);
sl@0	5429	}else{
sl@0	5430	/* pAcc is the empty buffer the merged data will end up in. */
sl@0	5431	DataBuffer *pAcc = &(pBuffers[iBuffer]);
sl@0	5432	DataBuffer *p = &(pBuffers[0]);
sl@0	5433
sl@0	5434	/* Handle position 0 specially to avoid need to prime pAcc
sl@0	5435	** with pData/nData.
sl@0	5436	*/
sl@0	5437	dataBufferSwap(p, pAcc);
sl@0	5438	docListAccumulateUnion(pAcc, pData, nData);
sl@0	5439
sl@0	5440	/* Accumulate remaining doclists into pAcc. */
sl@0	5441	for(++p; p<pAcc; ++p){
sl@0	5442	docListAccumulateUnion(pAcc, p->pData, p->nData);
sl@0	5443
sl@0	5444	/* dataBufferReset() could allow a large doclist to blow up
sl@0	5445	** our memory requirements.
sl@0	5446	*/
sl@0	5447	if( p->nCapacity<1024 ){
sl@0	5448	dataBufferReset(p);
sl@0	5449	}else{
sl@0	5450	dataBufferDestroy(p);
sl@0	5451	dataBufferInit(p, 0);
sl@0	5452	}
sl@0	5453	}
sl@0	5454	}
sl@0	5455	}
sl@0	5456	}
sl@0	5457
sl@0	5458	/* Union all the doclists together into out. /
sl@0	5459	/* TODO(shess) What if out is big? Sigh. /
sl@0	5460	if( rc==SQLITE_OK && nBuffers>0 ){
sl@0	5461	int iBuffer;
sl@0	5462	for(iBuffer=0; iBuffer<nBuffers; ++iBuffer){
sl@0	5463	if( pBuffers[iBuffer].nData>0 ){
sl@0	5464	if( out->nData==0 ){
sl@0	5465	dataBufferSwap(out, &(pBuffers[iBuffer]));
sl@0	5466	}else{
sl@0	5467	docListAccumulateUnion(out, pBuffers[iBuffer].pData,
sl@0	5468	pBuffers[iBuffer].nData);
sl@0	5469	}
sl@0	5470	}
sl@0	5471	}
sl@0	5472	}
sl@0	5473
sl@0	5474	while( nBuffers-- ){
sl@0	5475	dataBufferDestroy(&(pBuffers[nBuffers]));
sl@0	5476	}
sl@0	5477	if( pBuffers!=NULL ) sqlite3_free(pBuffers);
sl@0	5478
sl@0	5479	return rc;
sl@0	5480	}
sl@0	5481
sl@0	5482	/* Call loadSegmentLeavesInt() with pData/nData as input. */
sl@0	5483	static int loadSegmentLeaf(fulltext_vtab v, const char pData, int nData,
sl@0	5484	const char *pTerm, int nTerm, int isPrefix,
sl@0	5485	DataBuffer *out){
sl@0	5486	LeavesReader reader;
sl@0	5487	int rc;
sl@0	5488
sl@0	5489	assert( nData>1 );
sl@0	5490	assert( *pData=='\0' );
sl@0	5491	rc = leavesReaderInit(v, 0, 0, 0, pData, nData, &reader);
sl@0	5492	if( rc!=SQLITE_OK ) return rc;
sl@0	5493
sl@0	5494	rc = loadSegmentLeavesInt(v, &reader, pTerm, nTerm, isPrefix, out);
sl@0	5495	leavesReaderReset(&reader);
sl@0	5496	leavesReaderDestroy(&reader);
sl@0	5497	return rc;
sl@0	5498	}
sl@0	5499
sl@0	5500	/* Call loadSegmentLeavesInt() with the leaf nodes from iStartLeaf to
sl@0	5501	** iEndLeaf (inclusive) as input, and merge the resulting doclist into
sl@0	5502	** out.
sl@0	5503	*/
sl@0	5504	static int loadSegmentLeaves(fulltext_vtab *v,
sl@0	5505	sqlite_int64 iStartLeaf, sqlite_int64 iEndLeaf,
sl@0	5506	const char *pTerm, int nTerm, int isPrefix,
sl@0	5507	DataBuffer *out){
sl@0	5508	int rc;
sl@0	5509	LeavesReader reader;
sl@0	5510
sl@0	5511	assert( iStartLeaf<=iEndLeaf );
sl@0	5512	rc = leavesReaderInit(v, 0, iStartLeaf, iEndLeaf, NULL, 0, &reader);
sl@0	5513	if( rc!=SQLITE_OK ) return rc;
sl@0	5514
sl@0	5515	rc = loadSegmentLeavesInt(v, &reader, pTerm, nTerm, isPrefix, out);
sl@0	5516	leavesReaderReset(&reader);
sl@0	5517	leavesReaderDestroy(&reader);
sl@0	5518	return rc;
sl@0	5519	}
sl@0	5520
sl@0	5521	/* Taking pData/nData as an interior node, find the sequence of child
sl@0	5522	** nodes which could include pTerm/nTerm/isPrefix. Note that the
sl@0	5523	** interior node terms logically come between the blocks, so there is
sl@0	5524	** one more blockid than there are terms (that block contains terms >=
sl@0	5525	** the last interior-node term).
sl@0	5526	*/
sl@0	5527	/* TODO(shess) The calling code may already know that the end child is
sl@0	5528	** not worth calculating, because the end may be in a later sibling
sl@0	5529	** node. Consider whether breaking symmetry is worthwhile. I suspect
sl@0	5530	** it is not worthwhile.
sl@0	5531	*/
sl@0	5532	static void getChildrenContaining(const char *pData, int nData,
sl@0	5533	const char *pTerm, int nTerm, int isPrefix,
sl@0	5534	sqlite_int64 *piStartChild,
sl@0	5535	sqlite_int64 *piEndChild){
sl@0	5536	InteriorReader reader;
sl@0	5537
sl@0	5538	assert( nData>1 );
sl@0	5539	assert( *pData!='\0' );
sl@0	5540	interiorReaderInit(pData, nData, &reader);
sl@0	5541
sl@0	5542	/* Scan for the first child which could contain pTerm/nTerm. */
sl@0	5543	while( !interiorReaderAtEnd(&reader) ){
sl@0	5544	if( interiorReaderTermCmp(&reader, pTerm, nTerm, 0)>0 ) break;
sl@0	5545	interiorReaderStep(&reader);
sl@0	5546	}
sl@0	5547	*piStartChild = interiorReaderCurrentBlockid(&reader);
sl@0	5548
sl@0	5549	/* Keep scanning to find a term greater than our term, using prefix
sl@0	5550	** comparison if indicated. If isPrefix is false, this will be the
sl@0	5551	** same blockid as the starting block.
sl@0	5552	*/
sl@0	5553	while( !interiorReaderAtEnd(&reader) ){
sl@0	5554	if( interiorReaderTermCmp(&reader, pTerm, nTerm, isPrefix)>0 ) break;
sl@0	5555	interiorReaderStep(&reader);
sl@0	5556	}
sl@0	5557	*piEndChild = interiorReaderCurrentBlockid(&reader);
sl@0	5558
sl@0	5559	interiorReaderDestroy(&reader);
sl@0	5560
sl@0	5561	/* Children must ascend, and if !prefix, both must be the same. */
sl@0	5562	assert( piEndChild>=piStartChild );
sl@0	5563	assert( isPrefix \|\| piStartChild==piEndChild );
sl@0	5564	}
sl@0	5565
sl@0	5566	/* Read block at iBlockid and pass it with other params to
sl@0	5567	** getChildrenContaining().
sl@0	5568	*/
sl@0	5569	static int loadAndGetChildrenContaining(
sl@0	5570	fulltext_vtab *v,
sl@0	5571	sqlite_int64 iBlockid,
sl@0	5572	const char *pTerm, int nTerm, int isPrefix,
sl@0	5573	sqlite_int64 piStartChild, sqlite_int64 piEndChild
sl@0	5574	){
sl@0	5575	sqlite3_stmt *s = NULL;
sl@0	5576	int rc;
sl@0	5577
sl@0	5578	assert( iBlockid!=0 );
sl@0	5579	assert( pTerm!=NULL );
sl@0	5580	assert( nTerm!=0 ); /* TODO(shess) Why not allow this? */
sl@0	5581	assert( piStartChild!=NULL );
sl@0	5582	assert( piEndChild!=NULL );
sl@0	5583
sl@0	5584	rc = sql_get_statement(v, BLOCK_SELECT_STMT, &s);
sl@0	5585	if( rc!=SQLITE_OK ) return rc;
sl@0	5586
sl@0	5587	rc = sqlite3_bind_int64(s, 1, iBlockid);
sl@0	5588	if( rc!=SQLITE_OK ) return rc;
sl@0	5589
sl@0	5590	rc = sqlite3_step(s);
sl@0	5591	if( rc==SQLITE_DONE ) return SQLITE_ERROR;
sl@0	5592	if( rc!=SQLITE_ROW ) return rc;
sl@0	5593
sl@0	5594	getChildrenContaining(sqlite3_column_blob(s, 0), sqlite3_column_bytes(s, 0),
sl@0	5595	pTerm, nTerm, isPrefix, piStartChild, piEndChild);
sl@0	5596
sl@0	5597	/* We expect only one row. We must execute another sqlite3_step()
sl@0	5598	* to complete the iteration; otherwise the table will remain
sl@0	5599	* locked. */
sl@0	5600	rc = sqlite3_step(s);
sl@0	5601	if( rc==SQLITE_ROW ) return SQLITE_ERROR;
sl@0	5602	if( rc!=SQLITE_DONE ) return rc;
sl@0	5603
sl@0	5604	return SQLITE_OK;
sl@0	5605	}
sl@0	5606
sl@0	5607	/* Traverse the tree represented by pData[nData] looking for
sl@0	5608	** pTerm[nTerm], placing its doclist into *out. This is internal to
sl@0	5609	** loadSegment() to make error-handling cleaner.
sl@0	5610	*/
sl@0	5611	static int loadSegmentInt(fulltext_vtab v, const char pData, int nData,
sl@0	5612	sqlite_int64 iLeavesEnd,
sl@0	5613	const char *pTerm, int nTerm, int isPrefix,
sl@0	5614	DataBuffer *out){
sl@0	5615	/* Special case where root is a leaf. */
sl@0	5616	if( *pData=='\0' ){
sl@0	5617	return loadSegmentLeaf(v, pData, nData, pTerm, nTerm, isPrefix, out);
sl@0	5618	}else{
sl@0	5619	int rc;
sl@0	5620	sqlite_int64 iStartChild, iEndChild;
sl@0	5621
sl@0	5622	/* Process pData as an interior node, then loop down the tree
sl@0	5623	** until we find the set of leaf nodes to scan for the term.
sl@0	5624	*/
sl@0	5625	getChildrenContaining(pData, nData, pTerm, nTerm, isPrefix,
sl@0	5626	&iStartChild, &iEndChild);
sl@0	5627	while( iStartChild>iLeavesEnd ){
sl@0	5628	sqlite_int64 iNextStart, iNextEnd;
sl@0	5629	rc = loadAndGetChildrenContaining(v, iStartChild, pTerm, nTerm, isPrefix,
sl@0	5630	&iNextStart, &iNextEnd);
sl@0	5631	if( rc!=SQLITE_OK ) return rc;
sl@0	5632
sl@0	5633	/* If we've branched, follow the end branch, too. */
sl@0	5634	if( iStartChild!=iEndChild ){
sl@0	5635	sqlite_int64 iDummy;
sl@0	5636	rc = loadAndGetChildrenContaining(v, iEndChild, pTerm, nTerm, isPrefix,
sl@0	5637	&iDummy, &iNextEnd);
sl@0	5638	if( rc!=SQLITE_OK ) return rc;
sl@0	5639	}
sl@0	5640
sl@0	5641	assert( iNextStart<=iNextEnd );
sl@0	5642	iStartChild = iNextStart;
sl@0	5643	iEndChild = iNextEnd;
sl@0	5644	}
sl@0	5645	assert( iStartChild<=iLeavesEnd );
sl@0	5646	assert( iEndChild<=iLeavesEnd );
sl@0	5647
sl@0	5648	/* Scan through the leaf segments for doclists. */
sl@0	5649	return loadSegmentLeaves(v, iStartChild, iEndChild,
sl@0	5650	pTerm, nTerm, isPrefix, out);
sl@0	5651	}
sl@0	5652	}
sl@0	5653
sl@0	5654	/* Call loadSegmentInt() to collect the doclist for pTerm/nTerm, then
sl@0	5655	** merge its doclist over *out (any duplicate doclists read from the
sl@0	5656	** segment rooted at pData will overwrite those in *out).
sl@0	5657	*/
sl@0	5658	/* TODO(shess) Consider changing this to determine the depth of the
sl@0	5659	** leaves using either the first characters of interior nodes (when
sl@0	5660	** ==1, we're one level above the leaves), or the first character of
sl@0	5661	** the root (which will describe the height of the tree directly).
sl@0	5662	** Either feels somewhat tricky to me.
sl@0	5663	*/
sl@0	5664	/* TODO(shess) The current merge is likely to be slow for large
sl@0	5665	** doclists (though it should process from newest/smallest to
sl@0	5666	** oldest/largest, so it may not be that bad). It might be useful to
sl@0	5667	** modify things to allow for N-way merging. This could either be
sl@0	5668	** within a segment, with pairwise merges across segments, or across
sl@0	5669	** all segments at once.
sl@0	5670	*/
sl@0	5671	static int loadSegment(fulltext_vtab v, const char pData, int nData,
sl@0	5672	sqlite_int64 iLeavesEnd,
sl@0	5673	const char *pTerm, int nTerm, int isPrefix,
sl@0	5674	DataBuffer *out){
sl@0	5675	DataBuffer result;
sl@0	5676	int rc;
sl@0	5677
sl@0	5678	assert( nData>1 );
sl@0	5679
sl@0	5680	/* This code should never be called with buffered updates. */
sl@0	5681	assert( v->nPendingData<0 );
sl@0	5682
sl@0	5683	dataBufferInit(&result, 0);
sl@0	5684	rc = loadSegmentInt(v, pData, nData, iLeavesEnd,
sl@0	5685	pTerm, nTerm, isPrefix, &result);
sl@0	5686	if( rc==SQLITE_OK && result.nData>0 ){
sl@0	5687	if( out->nData==0 ){
sl@0	5688	DataBuffer tmp = *out;
sl@0	5689	*out = result;
sl@0	5690	result = tmp;
sl@0	5691	}else{
sl@0	5692	DataBuffer merged;
sl@0	5693	DLReader readers[2];
sl@0	5694
sl@0	5695	dlrInit(&readers[0], DL_DEFAULT, out->pData, out->nData);
sl@0	5696	dlrInit(&readers[1], DL_DEFAULT, result.pData, result.nData);
sl@0	5697	dataBufferInit(&merged, out->nData+result.nData);
sl@0	5698	docListMerge(&merged, readers, 2);
sl@0	5699	dataBufferDestroy(out);
sl@0	5700	*out = merged;
sl@0	5701	dlrDestroy(&readers[0]);
sl@0	5702	dlrDestroy(&readers[1]);
sl@0	5703	}
sl@0	5704	}
sl@0	5705	dataBufferDestroy(&result);
sl@0	5706	return rc;
sl@0	5707	}
sl@0	5708
sl@0	5709	/* Scan the database and merge together the posting lists for the term
sl@0	5710	** into *out.
sl@0	5711	*/
sl@0	5712	static int termSelect(fulltext_vtab *v, int iColumn,
sl@0	5713	const char *pTerm, int nTerm, int isPrefix,
sl@0	5714	DocListType iType, DataBuffer *out){
sl@0	5715	DataBuffer doclist;
sl@0	5716	sqlite3_stmt *s;
sl@0	5717	int rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
sl@0	5718	if( rc!=SQLITE_OK ) return rc;
sl@0	5719
sl@0	5720	/* This code should never be called with buffered updates. */
sl@0	5721	assert( v->nPendingData<0 );
sl@0	5722
sl@0	5723	dataBufferInit(&doclist, 0);
sl@0	5724
sl@0	5725	/* Traverse the segments from oldest to newest so that newer doclist
sl@0	5726	** elements for given docids overwrite older elements.
sl@0	5727	*/
sl@0	5728	while( (rc = sqlite3_step(s))==SQLITE_ROW ){
sl@0	5729	const char *pData = sqlite3_column_blob(s, 2);
sl@0	5730	const int nData = sqlite3_column_bytes(s, 2);
sl@0	5731	const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1);
sl@0	5732	rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, isPrefix,
sl@0	5733	&doclist);
sl@0	5734	if( rc!=SQLITE_OK ) goto err;
sl@0	5735	}
sl@0	5736	if( rc==SQLITE_DONE ){
sl@0	5737	if( doclist.nData!=0 ){
sl@0	5738	/* TODO(shess) The old term_select_all() code applied the column
sl@0	5739	** restrict as we merged segments, leading to smaller buffers.
sl@0	5740	** This is probably worthwhile to bring back, once the new storage
sl@0	5741	** system is checked in.
sl@0	5742	*/
sl@0	5743	if( iColumn==v->nColumn) iColumn = -1;
sl@0	5744	docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
sl@0	5745	iColumn, iType, out);
sl@0	5746	}
sl@0	5747	rc = SQLITE_OK;
sl@0	5748	}
sl@0	5749
sl@0	5750	err:
sl@0	5751	dataBufferDestroy(&doclist);
sl@0	5752	return rc;
sl@0	5753	}
sl@0	5754
sl@0	5755	/****************************************************************/
sl@0	5756	/* Used to hold hashtable data for sorting. */
sl@0	5757	typedef struct TermData {
sl@0	5758	const char *pTerm;
sl@0	5759	int nTerm;
sl@0	5760	DLCollector *pCollector;
sl@0	5761	} TermData;
sl@0	5762
sl@0	5763	/* Orders TermData elements in strcmp fashion ( <0 for less-than, 0
sl@0	5764	** for equal, >0 for greater-than).
sl@0	5765	*/
sl@0	5766	static int termDataCmp(const void av, const void bv){
sl@0	5767	const TermData a = (const TermData )av;
sl@0	5768	const TermData b = (const TermData )bv;
sl@0	5769	int n = a->nTerm<b->nTerm ? a->nTerm : b->nTerm;
sl@0	5770	int c = memcmp(a->pTerm, b->pTerm, n);
sl@0	5771	if( c!=0 ) return c;
sl@0	5772	return a->nTerm-b->nTerm;
sl@0	5773	}
sl@0	5774
sl@0	5775	/* Order pTerms data by term, then write a new level 0 segment using
sl@0	5776	** LeafWriter.
sl@0	5777	*/
sl@0	5778	static int writeZeroSegment(fulltext_vtab v, fts2Hash pTerms){
sl@0	5779	fts2HashElem *e;
sl@0	5780	int idx, rc, i, n;
sl@0	5781	TermData *pData;
sl@0	5782	LeafWriter writer;
sl@0	5783	DataBuffer dl;
sl@0	5784
sl@0	5785	/* Determine the next index at level 0, merging as necessary. */
sl@0	5786	rc = segdirNextIndex(v, 0, &idx);
sl@0	5787	if( rc!=SQLITE_OK ) return rc;
sl@0	5788
sl@0	5789	n = fts2HashCount(pTerms);
sl@0	5790	pData = sqlite3_malloc(n*sizeof(TermData));
sl@0	5791
sl@0	5792	for(i = 0, e = fts2HashFirst(pTerms); e; i++, e = fts2HashNext(e)){
sl@0	5793	assert( i<n );
sl@0	5794	pData[i].pTerm = fts2HashKey(e);
sl@0	5795	pData[i].nTerm = fts2HashKeysize(e);
sl@0	5796	pData[i].pCollector = fts2HashData(e);
sl@0	5797	}
sl@0	5798	assert( i==n );
sl@0	5799
sl@0	5800	/* TODO(shess) Should we allow user-defined collation sequences,
sl@0	5801	** here? I think we only need that once we support prefix searches.
sl@0	5802	*/
sl@0	5803	if( n>1 ) qsort(pData, n, sizeof(*pData), termDataCmp);
sl@0	5804
sl@0	5805	/* TODO(shess) Refactor so that we can write directly to the segment
sl@0	5806	** DataBuffer, as happens for segment merges.
sl@0	5807	*/
sl@0	5808	leafWriterInit(0, idx, &writer);
sl@0	5809	dataBufferInit(&dl, 0);
sl@0	5810	for(i=0; i<n; i++){
sl@0	5811	dataBufferReset(&dl);
sl@0	5812	dlcAddDoclist(pData[i].pCollector, &dl);
sl@0	5813	rc = leafWriterStep(v, &writer,
sl@0	5814	pData[i].pTerm, pData[i].nTerm, dl.pData, dl.nData);
sl@0	5815	if( rc!=SQLITE_OK ) goto err;
sl@0	5816	}
sl@0	5817	rc = leafWriterFinalize(v, &writer);
sl@0	5818
sl@0	5819	err:
sl@0	5820	dataBufferDestroy(&dl);
sl@0	5821	sqlite3_free(pData);
sl@0	5822	leafWriterDestroy(&writer);
sl@0	5823	return rc;
sl@0	5824	}
sl@0	5825
sl@0	5826	/* If pendingTerms has data, free it. */
sl@0	5827	static int clearPendingTerms(fulltext_vtab *v){
sl@0	5828	if( v->nPendingData>=0 ){
sl@0	5829	fts2HashElem *e;
sl@0	5830	for(e=fts2HashFirst(&v->pendingTerms); e; e=fts2HashNext(e)){
sl@0	5831	dlcDelete(fts2HashData(e));
sl@0	5832	}
sl@0	5833	fts2HashClear(&v->pendingTerms);
sl@0	5834	v->nPendingData = -1;
sl@0	5835	}
sl@0	5836	return SQLITE_OK;
sl@0	5837	}
sl@0	5838
sl@0	5839	/* If pendingTerms has data, flush it to a level-zero segment, and
sl@0	5840	** free it.
sl@0	5841	*/
sl@0	5842	static int flushPendingTerms(fulltext_vtab *v){
sl@0	5843	if( v->nPendingData>=0 ){
sl@0	5844	int rc = writeZeroSegment(v, &v->pendingTerms);
sl@0	5845	if( rc==SQLITE_OK ) clearPendingTerms(v);
sl@0	5846	return rc;
sl@0	5847	}
sl@0	5848	return SQLITE_OK;
sl@0	5849	}
sl@0	5850
sl@0	5851	/* If pendingTerms is "too big", or docid is out of order, flush it.
sl@0	5852	** Regardless, be certain that pendingTerms is initialized for use.
sl@0	5853	*/
sl@0	5854	static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid){
sl@0	5855	/* TODO(shess) Explore whether partially flushing the buffer on
sl@0	5856	** forced-flush would provide better performance. I suspect that if
sl@0	5857	** we ordered the doclists by size and flushed the largest until the
sl@0	5858	** buffer was half empty, that would let the less frequent terms
sl@0	5859	** generate longer doclists.
sl@0	5860	*/
sl@0	5861	if( iDocid<=v->iPrevDocid \|\| v->nPendingData>kPendingThreshold ){
sl@0	5862	int rc = flushPendingTerms(v);
sl@0	5863	if( rc!=SQLITE_OK ) return rc;
sl@0	5864	}
sl@0	5865	if( v->nPendingData<0 ){
sl@0	5866	fts2HashInit(&v->pendingTerms, FTS2_HASH_STRING, 1);
sl@0	5867	v->nPendingData = 0;
sl@0	5868	}
sl@0	5869	v->iPrevDocid = iDocid;
sl@0	5870	return SQLITE_OK;
sl@0	5871	}
sl@0	5872
sl@0	5873	/* This function implements the xUpdate callback; it is the top-level entry
sl@0	5874	* point for inserting, deleting or updating a row in a full-text table. */
sl@0	5875	static int fulltextUpdate(sqlite3_vtab pVtab, int nArg, sqlite3_value *ppArg,
sl@0	5876	sqlite_int64 *pRowid){
sl@0	5877	fulltext_vtab v = (fulltext_vtab ) pVtab;
sl@0	5878	int rc;
sl@0	5879
sl@0	5880	TRACE(("FTS2 Update %p\n", pVtab));
sl@0	5881
sl@0	5882	if( nArg<2 ){
sl@0	5883	rc = index_delete(v, sqlite3_value_int64(ppArg[0]));
sl@0	5884	if( rc==SQLITE_OK ){
sl@0	5885	/* If we just deleted the last row in the table, clear out the
sl@0	5886	** index data.
sl@0	5887	*/
sl@0	5888	rc = content_exists(v);
sl@0	5889	if( rc==SQLITE_ROW ){
sl@0	5890	rc = SQLITE_OK;
sl@0	5891	}else if( rc==SQLITE_DONE ){
sl@0	5892	/* Clear the pending terms so we don't flush a useless level-0
sl@0	5893	** segment when the transaction closes.
sl@0	5894	*/
sl@0	5895	rc = clearPendingTerms(v);
sl@0	5896	if( rc==SQLITE_OK ){
sl@0	5897	rc = segdir_delete_all(v);
sl@0	5898	}
sl@0	5899	}
sl@0	5900	}
sl@0	5901	} else if( sqlite3_value_type(ppArg[0]) != SQLITE_NULL ){
sl@0	5902	/* An update:
sl@0	5903	* ppArg[0] = old rowid
sl@0	5904	* ppArg[1] = new rowid
sl@0	5905	* ppArg[2..2+v->nColumn-1] = values
sl@0	5906	* ppArg[2+v->nColumn] = value for magic column (we ignore this)
sl@0	5907	*/
sl@0	5908	sqlite_int64 rowid = sqlite3_value_int64(ppArg[0]);
sl@0	5909	if( sqlite3_value_type(ppArg[1]) != SQLITE_INTEGER \|\|
sl@0	5910	sqlite3_value_int64(ppArg[1]) != rowid ){
sl@0	5911	rc = SQLITE_ERROR; /* we don't allow changing the rowid */
sl@0	5912	} else {
sl@0	5913	assert( nArg==2+v->nColumn+1);
sl@0	5914	rc = index_update(v, rowid, &ppArg[2]);
sl@0	5915	}
sl@0	5916	} else {
sl@0	5917	/* An insert:
sl@0	5918	* ppArg[1] = requested rowid
sl@0	5919	* ppArg[2..2+v->nColumn-1] = values
sl@0	5920	* ppArg[2+v->nColumn] = value for magic column (we ignore this)
sl@0	5921	*/
sl@0	5922	assert( nArg==2+v->nColumn+1);
sl@0	5923	rc = index_insert(v, ppArg[1], &ppArg[2], pRowid);
sl@0	5924	}
sl@0	5925
sl@0	5926	return rc;
sl@0	5927	}
sl@0	5928
sl@0	5929	static int fulltextSync(sqlite3_vtab *pVtab){
sl@0	5930	TRACE(("FTS2 xSync()\n"));
sl@0	5931	return flushPendingTerms((fulltext_vtab *)pVtab);
sl@0	5932	}
sl@0	5933
sl@0	5934	static int fulltextBegin(sqlite3_vtab *pVtab){
sl@0	5935	fulltext_vtab v = (fulltext_vtab ) pVtab;
sl@0	5936	TRACE(("FTS2 xBegin()\n"));
sl@0	5937
sl@0	5938	/* Any buffered updates should have been cleared by the previous
sl@0	5939	** transaction.
sl@0	5940	*/
sl@0	5941	assert( v->nPendingData<0 );
sl@0	5942	return clearPendingTerms(v);
sl@0	5943	}
sl@0	5944
sl@0	5945	static int fulltextCommit(sqlite3_vtab *pVtab){
sl@0	5946	fulltext_vtab v = (fulltext_vtab ) pVtab;
sl@0	5947	TRACE(("FTS2 xCommit()\n"));
sl@0	5948
sl@0	5949	/* Buffered updates should have been cleared by fulltextSync(). */
sl@0	5950	assert( v->nPendingData<0 );
sl@0	5951	return clearPendingTerms(v);
sl@0	5952	}
sl@0	5953
sl@0	5954	static int fulltextRollback(sqlite3_vtab *pVtab){
sl@0	5955	TRACE(("FTS2 xRollback()\n"));
sl@0	5956	return clearPendingTerms((fulltext_vtab *)pVtab);
sl@0	5957	}
sl@0	5958
sl@0	5959	/*
sl@0	5960	** Implementation of the snippet() function for FTS2
sl@0	5961	*/
sl@0	5962	static void snippetFunc(
sl@0	5963	sqlite3_context *pContext,
sl@0	5964	int argc,
sl@0	5965	sqlite3_value **argv
sl@0	5966	){
sl@0	5967	fulltext_cursor *pCursor;
sl@0	5968	if( argc<1 ) return;
sl@0	5969	if( sqlite3_value_type(argv[0])!=SQLITE_BLOB \|\|
sl@0	5970	sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
sl@0	5971	sqlite3_result_error(pContext, "illegal first argument to html_snippet",-1);
sl@0	5972	}else{
sl@0	5973	const char *zStart = "<b>";
sl@0	5974	const char *zEnd = "</b>";
sl@0	5975	const char *zEllipsis = "<b>...</b>";
sl@0	5976	memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
sl@0	5977	if( argc>=2 ){
sl@0	5978	zStart = (const char*)sqlite3_value_text(argv[1]);
sl@0	5979	if( argc>=3 ){
sl@0	5980	zEnd = (const char*)sqlite3_value_text(argv[2]);
sl@0	5981	if( argc>=4 ){
sl@0	5982	zEllipsis = (const char*)sqlite3_value_text(argv[3]);
sl@0	5983	}
sl@0	5984	}
sl@0	5985	}
sl@0	5986	snippetAllOffsets(pCursor);
sl@0	5987	snippetText(pCursor, zStart, zEnd, zEllipsis);
sl@0	5988	sqlite3_result_text(pContext, pCursor->snippet.zSnippet,
sl@0	5989	pCursor->snippet.nSnippet, SQLITE_STATIC);
sl@0	5990	}
sl@0	5991	}
sl@0	5992
sl@0	5993	/*
sl@0	5994	** Implementation of the offsets() function for FTS2
sl@0	5995	*/
sl@0	5996	static void snippetOffsetsFunc(
sl@0	5997	sqlite3_context *pContext,
sl@0	5998	int argc,
sl@0	5999	sqlite3_value **argv
sl@0	6000	){
sl@0	6001	fulltext_cursor *pCursor;
sl@0	6002	if( argc<1 ) return;
sl@0	6003	if( sqlite3_value_type(argv[0])!=SQLITE_BLOB \|\|
sl@0	6004	sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
sl@0	6005	sqlite3_result_error(pContext, "illegal first argument to offsets",-1);
sl@0	6006	}else{
sl@0	6007	memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
sl@0	6008	snippetAllOffsets(pCursor);
sl@0	6009	snippetOffsetText(&pCursor->snippet);
sl@0	6010	sqlite3_result_text(pContext,
sl@0	6011	pCursor->snippet.zOffset, pCursor->snippet.nOffset,
sl@0	6012	SQLITE_STATIC);
sl@0	6013	}
sl@0	6014	}
sl@0	6015
sl@0	6016	/* OptLeavesReader is nearly identical to LeavesReader, except that
sl@0	6017	** where LeavesReader is geared towards the merging of complete
sl@0	6018	** segment levels (with exactly MERGE_COUNT segments), OptLeavesReader
sl@0	6019	** is geared towards implementation of the optimize() function, and
sl@0	6020	** can merge all segments simultaneously. This version may be
sl@0	6021	** somewhat less efficient than LeavesReader because it merges into an
sl@0	6022	** accumulator rather than doing an N-way merge, but since segment
sl@0	6023	** size grows exponentially (so segment count logrithmically) this is
sl@0	6024	** probably not an immediate problem.
sl@0	6025	*/
sl@0	6026	/* TODO(shess): Prove that assertion, or extend the merge code to
sl@0	6027	** merge tree fashion (like the prefix-searching code does).
sl@0	6028	*/
sl@0	6029	/* TODO(shess): OptLeavesReader and LeavesReader could probably be
sl@0	6030	** merged with little or no loss of performance for LeavesReader. The
sl@0	6031	** merged code would need to handle >MERGE_COUNT segments, and would
sl@0	6032	** also need to be able to optionally optimize away deletes.
sl@0	6033	*/
sl@0	6034	typedef struct OptLeavesReader {
sl@0	6035	/* Segment number, to order readers by age. */
sl@0	6036	int segment;
sl@0	6037	LeavesReader reader;
sl@0	6038	} OptLeavesReader;
sl@0	6039
sl@0	6040	static int optLeavesReaderAtEnd(OptLeavesReader *pReader){
sl@0	6041	return leavesReaderAtEnd(&pReader->reader);
sl@0	6042	}
sl@0	6043	static int optLeavesReaderTermBytes(OptLeavesReader *pReader){
sl@0	6044	return leavesReaderTermBytes(&pReader->reader);
sl@0	6045	}
sl@0	6046	static const char optLeavesReaderData(OptLeavesReader pReader){
sl@0	6047	return leavesReaderData(&pReader->reader);
sl@0	6048	}
sl@0	6049	static int optLeavesReaderDataBytes(OptLeavesReader *pReader){
sl@0	6050	return leavesReaderDataBytes(&pReader->reader);
sl@0	6051	}
sl@0	6052	static const char optLeavesReaderTerm(OptLeavesReader pReader){
sl@0	6053	return leavesReaderTerm(&pReader->reader);
sl@0	6054	}
sl@0	6055	static int optLeavesReaderStep(fulltext_vtab v, OptLeavesReader pReader){
sl@0	6056	return leavesReaderStep(v, &pReader->reader);
sl@0	6057	}
sl@0	6058	static int optLeavesReaderTermCmp(OptLeavesReader lr1, OptLeavesReader lr2){
sl@0	6059	return leavesReaderTermCmp(&lr1->reader, &lr2->reader);
sl@0	6060	}
sl@0	6061	/* Order by term ascending, segment ascending (oldest to newest), with
sl@0	6062	** exhausted readers to the end.
sl@0	6063	*/
sl@0	6064	static int optLeavesReaderCmp(OptLeavesReader lr1, OptLeavesReader lr2){
sl@0	6065	int c = optLeavesReaderTermCmp(lr1, lr2);
sl@0	6066	if( c!=0 ) return c;
sl@0	6067	return lr1->segment-lr2->segment;
sl@0	6068	}
sl@0	6069	/* Bubble pLr[0] to appropriate place in pLr[1..nLr-1]. Assumes that
sl@0	6070	** pLr[1..nLr-1] is already sorted.
sl@0	6071	*/
sl@0	6072	static void optLeavesReaderReorder(OptLeavesReader *pLr, int nLr){
sl@0	6073	while( nLr>1 && optLeavesReaderCmp(pLr, pLr+1)>0 ){
sl@0	6074	OptLeavesReader tmp = pLr[0];
sl@0	6075	pLr[0] = pLr[1];
sl@0	6076	pLr[1] = tmp;
sl@0	6077	nLr--;
sl@0	6078	pLr++;
sl@0	6079	}
sl@0	6080	}
sl@0	6081
sl@0	6082	/* optimize() helper function. Put the readers in order and iterate
sl@0	6083	** through them, merging doclists for matching terms into pWriter.
sl@0	6084	** Returns SQLITE_OK on success, or the SQLite error code which
sl@0	6085	** prevented success.
sl@0	6086	*/
sl@0	6087	static int optimizeInternal(fulltext_vtab *v,
sl@0	6088	OptLeavesReader *readers, int nReaders,
sl@0	6089	LeafWriter *pWriter){
sl@0	6090	int i, rc = SQLITE_OK;
sl@0	6091	DataBuffer doclist, merged, tmp;
sl@0	6092
sl@0	6093	/* Order the readers. */
sl@0	6094	i = nReaders;
sl@0	6095	while( i-- > 0 ){
sl@0	6096	optLeavesReaderReorder(&readers[i], nReaders-i);
sl@0	6097	}
sl@0	6098
sl@0	6099	dataBufferInit(&doclist, LEAF_MAX);
sl@0	6100	dataBufferInit(&merged, LEAF_MAX);
sl@0	6101
sl@0	6102	/* Exhausted readers bubble to the end, so when the first reader is
sl@0	6103	** at eof, all are at eof.
sl@0	6104	*/
sl@0	6105	while( !optLeavesReaderAtEnd(&readers[0]) ){
sl@0	6106
sl@0	6107	/* Figure out how many readers share the next term. */
sl@0	6108	for(i=1; i<nReaders && !optLeavesReaderAtEnd(&readers[i]); i++){
sl@0	6109	if( 0!=optLeavesReaderTermCmp(&readers[0], &readers[i]) ) break;
sl@0	6110	}
sl@0	6111
sl@0	6112	/* Special-case for no merge. */
sl@0	6113	if( i==1 ){
sl@0	6114	/* Trim deletions from the doclist. */
sl@0	6115	dataBufferReset(&merged);
sl@0	6116	docListTrim(DL_DEFAULT,
sl@0	6117	optLeavesReaderData(&readers[0]),
sl@0	6118	optLeavesReaderDataBytes(&readers[0]),
sl@0	6119	-1, DL_DEFAULT, &merged);
sl@0	6120	}else{
sl@0	6121	DLReader dlReaders[MERGE_COUNT];
sl@0	6122	int iReader, nReaders;
sl@0	6123
sl@0	6124	/* Prime the pipeline with the first reader's doclist. After
sl@0	6125	** one pass index 0 will reference the accumulated doclist.
sl@0	6126	*/
sl@0	6127	dlrInit(&dlReaders[0], DL_DEFAULT,
sl@0	6128	optLeavesReaderData(&readers[0]),
sl@0	6129	optLeavesReaderDataBytes(&readers[0]));
sl@0	6130	iReader = 1;
sl@0	6131
sl@0	6132	assert( iReader<i ); /* Must execute the loop at least once. */
sl@0	6133	while( iReader<i ){
sl@0	6134	/* Merge 16 inputs per pass. */
sl@0	6135	for( nReaders=1; iReader<i && nReaders<MERGE_COUNT;
sl@0	6136	iReader++, nReaders++ ){
sl@0	6137	dlrInit(&dlReaders[nReaders], DL_DEFAULT,
sl@0	6138	optLeavesReaderData(&readers[iReader]),
sl@0	6139	optLeavesReaderDataBytes(&readers[iReader]));
sl@0	6140	}
sl@0	6141
sl@0	6142	/* Merge doclists and swap result into accumulator. */
sl@0	6143	dataBufferReset(&merged);
sl@0	6144	docListMerge(&merged, dlReaders, nReaders);
sl@0	6145	tmp = merged;
sl@0	6146	merged = doclist;
sl@0	6147	doclist = tmp;
sl@0	6148
sl@0	6149	while( nReaders-- > 0 ){
sl@0	6150	dlrDestroy(&dlReaders[nReaders]);
sl@0	6151	}
sl@0	6152
sl@0	6153	/* Accumulated doclist to reader 0 for next pass. */
sl@0	6154	dlrInit(&dlReaders[0], DL_DEFAULT, doclist.pData, doclist.nData);
sl@0	6155	}
sl@0	6156
sl@0	6157	/* Destroy reader that was left in the pipeline. */
sl@0	6158	dlrDestroy(&dlReaders[0]);
sl@0	6159
sl@0	6160	/* Trim deletions from the doclist. */
sl@0	6161	dataBufferReset(&merged);
sl@0	6162	docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
sl@0	6163	-1, DL_DEFAULT, &merged);
sl@0	6164	}
sl@0	6165
sl@0	6166	/* Only pass doclists with hits (skip if all hits deleted). */
sl@0	6167	if( merged.nData>0 ){
sl@0	6168	rc = leafWriterStep(v, pWriter,
sl@0	6169	optLeavesReaderTerm(&readers[0]),
sl@0	6170	optLeavesReaderTermBytes(&readers[0]),
sl@0	6171	merged.pData, merged.nData);
sl@0	6172	if( rc!=SQLITE_OK ) goto err;
sl@0	6173	}
sl@0	6174
sl@0	6175	/* Step merged readers to next term and reorder. */
sl@0	6176	while( i-- > 0 ){
sl@0	6177	rc = optLeavesReaderStep(v, &readers[i]);
sl@0	6178	if( rc!=SQLITE_OK ) goto err;
sl@0	6179
sl@0	6180	optLeavesReaderReorder(&readers[i], nReaders-i);
sl@0	6181	}
sl@0	6182	}
sl@0	6183
sl@0	6184	err:
sl@0	6185	dataBufferDestroy(&doclist);
sl@0	6186	dataBufferDestroy(&merged);
sl@0	6187	return rc;
sl@0	6188	}
sl@0	6189
sl@0	6190	/* Implement optimize() function for FTS3. optimize(t) merges all
sl@0	6191	** segments in the fts index into a single segment. 't' is the magic
sl@0	6192	** table-named column.
sl@0	6193	*/
sl@0	6194	static void optimizeFunc(sqlite3_context *pContext,
sl@0	6195	int argc, sqlite3_value **argv){
sl@0	6196	fulltext_cursor *pCursor;
sl@0	6197	if( argc>1 ){
sl@0	6198	sqlite3_result_error(pContext, "excess arguments to optimize()",-1);
sl@0	6199	}else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB \|\|
sl@0	6200	sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
sl@0	6201	sqlite3_result_error(pContext, "illegal first argument to optimize",-1);
sl@0	6202	}else{
sl@0	6203	fulltext_vtab *v;
sl@0	6204	int i, rc, iMaxLevel;
sl@0	6205	OptLeavesReader *readers;
sl@0	6206	int nReaders;
sl@0	6207	LeafWriter writer;
sl@0	6208	sqlite3_stmt *s;
sl@0	6209
sl@0	6210	memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
sl@0	6211	v = cursor_vtab(pCursor);
sl@0	6212
sl@0	6213	/* Flush any buffered updates before optimizing. */
sl@0	6214	rc = flushPendingTerms(v);
sl@0	6215	if( rc!=SQLITE_OK ) goto err;
sl@0	6216
sl@0	6217	rc = segdir_count(v, &nReaders, &iMaxLevel);
sl@0	6218	if( rc!=SQLITE_OK ) goto err;
sl@0	6219	if( nReaders==0 \|\| nReaders==1 ){
sl@0	6220	sqlite3_result_text(pContext, "Index already optimal", -1,
sl@0	6221	SQLITE_STATIC);
sl@0	6222	return;
sl@0	6223	}
sl@0	6224
sl@0	6225	rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
sl@0	6226	if( rc!=SQLITE_OK ) goto err;
sl@0	6227
sl@0	6228	readers = sqlite3_malloc(nReaders*sizeof(readers[0]));
sl@0	6229	if( readers==NULL ) goto err;
sl@0	6230
sl@0	6231	/* Note that there will already be a segment at this position
sl@0	6232	** until we call segdir_delete() on iMaxLevel.
sl@0	6233	*/
sl@0	6234	leafWriterInit(iMaxLevel, 0, &writer);
sl@0	6235
sl@0	6236	i = 0;
sl@0	6237	while( (rc = sqlite3_step(s))==SQLITE_ROW ){
sl@0	6238	sqlite_int64 iStart = sqlite3_column_int64(s, 0);
sl@0	6239	sqlite_int64 iEnd = sqlite3_column_int64(s, 1);
sl@0	6240	const char *pRootData = sqlite3_column_blob(s, 2);
sl@0	6241	int nRootData = sqlite3_column_bytes(s, 2);
sl@0	6242
sl@0	6243	assert( i<nReaders );
sl@0	6244	rc = leavesReaderInit(v, -1, iStart, iEnd, pRootData, nRootData,
sl@0	6245	&readers[i].reader);
sl@0	6246	if( rc!=SQLITE_OK ) break;
sl@0	6247
sl@0	6248	readers[i].segment = i;
sl@0	6249	i++;
sl@0	6250	}
sl@0	6251
sl@0	6252	/* If we managed to succesfully read them all, optimize them. */
sl@0	6253	if( rc==SQLITE_DONE ){
sl@0	6254	assert( i==nReaders );
sl@0	6255	rc = optimizeInternal(v, readers, nReaders, &writer);
sl@0	6256	}
sl@0	6257
sl@0	6258	while( i-- > 0 ){
sl@0	6259	leavesReaderDestroy(&readers[i].reader);
sl@0	6260	}
sl@0	6261	sqlite3_free(readers);
sl@0	6262
sl@0	6263	/* If we've successfully gotten to here, delete the old segments
sl@0	6264	** and flush the interior structure of the new segment.
sl@0	6265	*/
sl@0	6266	if( rc==SQLITE_OK ){
sl@0	6267	for( i=0; i<=iMaxLevel; i++ ){
sl@0	6268	rc = segdir_delete(v, i);
sl@0	6269	if( rc!=SQLITE_OK ) break;
sl@0	6270	}
sl@0	6271
sl@0	6272	if( rc==SQLITE_OK ) rc = leafWriterFinalize(v, &writer);
sl@0	6273	}
sl@0	6274
sl@0	6275	leafWriterDestroy(&writer);
sl@0	6276
sl@0	6277	if( rc!=SQLITE_OK ) goto err;
sl@0	6278
sl@0	6279	sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC);
sl@0	6280	return;
sl@0	6281
sl@0	6282	/* TODO(shess): Error-handling needs to be improved along the
sl@0	6283	** lines of the dump_ functions.
sl@0	6284	*/
sl@0	6285	err:
sl@0	6286	{
sl@0	6287	char buf[512];
sl@0	6288	sqlite3_snprintf(sizeof(buf), buf, "Error in optimize: %s",
sl@0	6289	sqlite3_errmsg(sqlite3_context_db_handle(pContext)));
sl@0	6290	sqlite3_result_error(pContext, buf, -1);
sl@0	6291	}
sl@0	6292	}
sl@0	6293	}
sl@0	6294
sl@0	6295	#ifdef SQLITE_TEST
sl@0	6296	/* Generate an error of the form "<prefix>: <msg>". If msg is NULL,
sl@0	6297	** pull the error from the context's db handle.
sl@0	6298	*/
sl@0	6299	static void generateError(sqlite3_context *pContext,
sl@0	6300	const char prefix, const char msg){
sl@0	6301	char buf[512];
sl@0	6302	if( msg==NULL ) msg = sqlite3_errmsg(sqlite3_context_db_handle(pContext));
sl@0	6303	sqlite3_snprintf(sizeof(buf), buf, "%s: %s", prefix, msg);
sl@0	6304	sqlite3_result_error(pContext, buf, -1);
sl@0	6305	}
sl@0	6306
sl@0	6307	/* Helper function to collect the set of terms in the segment into
sl@0	6308	** pTerms. The segment is defined by the leaf nodes between
sl@0	6309	** iStartBlockid and iEndBlockid, inclusive, or by the contents of
sl@0	6310	** pRootData if iStartBlockid is 0 (in which case the entire segment
sl@0	6311	** fit in a leaf).
sl@0	6312	*/
sl@0	6313	static int collectSegmentTerms(fulltext_vtab v, sqlite3_stmt s,
sl@0	6314	fts2Hash *pTerms){
sl@0	6315	const sqlite_int64 iStartBlockid = sqlite3_column_int64(s, 0);
sl@0	6316	const sqlite_int64 iEndBlockid = sqlite3_column_int64(s, 1);
sl@0	6317	const char *pRootData = sqlite3_column_blob(s, 2);
sl@0	6318	const int nRootData = sqlite3_column_bytes(s, 2);
sl@0	6319	LeavesReader reader;
sl@0	6320	int rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid,
sl@0	6321	pRootData, nRootData, &reader);
sl@0	6322	if( rc!=SQLITE_OK ) return rc;
sl@0	6323
sl@0	6324	while( rc==SQLITE_OK && !leavesReaderAtEnd(&reader) ){
sl@0	6325	const char *pTerm = leavesReaderTerm(&reader);
sl@0	6326	const int nTerm = leavesReaderTermBytes(&reader);
sl@0	6327	void *oldValue = sqlite3Fts2HashFind(pTerms, pTerm, nTerm);
sl@0	6328	void newValue = (void )((char *)oldValue+1);
sl@0	6329
sl@0	6330	/* From the comment before sqlite3Fts2HashInsert in fts2_hash.c,
sl@0	6331	** the data value passed is returned in case of malloc failure.
sl@0	6332	*/
sl@0	6333	if( newValue==sqlite3Fts2HashInsert(pTerms, pTerm, nTerm, newValue) ){
sl@0	6334	rc = SQLITE_NOMEM;
sl@0	6335	}else{
sl@0	6336	rc = leavesReaderStep(v, &reader);
sl@0	6337	}
sl@0	6338	}
sl@0	6339
sl@0	6340	leavesReaderDestroy(&reader);
sl@0	6341	return rc;
sl@0	6342	}
sl@0	6343
sl@0	6344	/* Helper function to build the result string for dump_terms(). */
sl@0	6345	static int generateTermsResult(sqlite3_context pContext, fts2Hash pTerms){
sl@0	6346	int iTerm, nTerms, nResultBytes, iByte;
sl@0	6347	char *result;
sl@0	6348	TermData *pData;
sl@0	6349	fts2HashElem *e;
sl@0	6350
sl@0	6351	/* Iterate pTerms to generate an array of terms in pData for
sl@0	6352	** sorting.
sl@0	6353	*/
sl@0	6354	nTerms = fts2HashCount(pTerms);
sl@0	6355	assert( nTerms>0 );
sl@0	6356	pData = sqlite3_malloc(nTerms*sizeof(TermData));
sl@0	6357	if( pData==NULL ) return SQLITE_NOMEM;
sl@0	6358
sl@0	6359	nResultBytes = 0;
sl@0	6360	for(iTerm = 0, e = fts2HashFirst(pTerms); e; iTerm++, e = fts2HashNext(e)){
sl@0	6361	nResultBytes += fts2HashKeysize(e)+1; /* Term plus trailing space */
sl@0	6362	assert( iTerm<nTerms );
sl@0	6363	pData[iTerm].pTerm = fts2HashKey(e);
sl@0	6364	pData[iTerm].nTerm = fts2HashKeysize(e);
sl@0	6365	pData[iTerm].pCollector = fts2HashData(e); /* unused */
sl@0	6366	}
sl@0	6367	assert( iTerm==nTerms );
sl@0	6368
sl@0	6369	assert( nResultBytes>0 ); /* nTerms>0, nResultsBytes must be, too. */
sl@0	6370	result = sqlite3_malloc(nResultBytes);
sl@0	6371	if( result==NULL ){
sl@0	6372	sqlite3_free(pData);
sl@0	6373	return SQLITE_NOMEM;
sl@0	6374	}
sl@0	6375
sl@0	6376	if( nTerms>1 ) qsort(pData, nTerms, sizeof(*pData), termDataCmp);
sl@0	6377
sl@0	6378	/* Read the terms in order to build the result. */
sl@0	6379	iByte = 0;
sl@0	6380	for(iTerm=0; iTerm<nTerms; ++iTerm){
sl@0	6381	memcpy(result+iByte, pData[iTerm].pTerm, pData[iTerm].nTerm);
sl@0	6382	iByte += pData[iTerm].nTerm;
sl@0	6383	result[iByte++] = ' ';
sl@0	6384	}
sl@0	6385	assert( iByte==nResultBytes );
sl@0	6386	assert( result[nResultBytes-1]==' ' );
sl@0	6387	result[nResultBytes-1] = '\0';
sl@0	6388
sl@0	6389	/* Passes away ownership of result. */
sl@0	6390	sqlite3_result_text(pContext, result, nResultBytes-1, sqlite3_free);
sl@0	6391	sqlite3_free(pData);
sl@0	6392	return SQLITE_OK;
sl@0	6393	}
sl@0	6394
sl@0	6395	/* Implements dump_terms() for use in inspecting the fts2 index from
sl@0	6396	** tests. TEXT result containing the ordered list of terms joined by
sl@0	6397	** spaces. dump_terms(t, level, idx) dumps the terms for the segment
sl@0	6398	** specified by level, idx (in %_segdir), while dump_terms(t) dumps
sl@0	6399	** all terms in the index. In both cases t is the fts table's magic
sl@0	6400	** table-named column.
sl@0	6401	*/
sl@0	6402	static void dumpTermsFunc(
sl@0	6403	sqlite3_context *pContext,
sl@0	6404	int argc, sqlite3_value **argv
sl@0	6405	){
sl@0	6406	fulltext_cursor *pCursor;
sl@0	6407	if( argc!=3 && argc!=1 ){
sl@0	6408	generateError(pContext, "dump_terms", "incorrect arguments");
sl@0	6409	}else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB \|\|
sl@0	6410	sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
sl@0	6411	generateError(pContext, "dump_terms", "illegal first argument");
sl@0	6412	}else{
sl@0	6413	fulltext_vtab *v;
sl@0	6414	fts2Hash terms;
sl@0	6415	sqlite3_stmt *s = NULL;
sl@0	6416	int rc;
sl@0	6417
sl@0	6418	memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
sl@0	6419	v = cursor_vtab(pCursor);
sl@0	6420
sl@0	6421	/* If passed only the cursor column, get all segments. Otherwise
sl@0	6422	** get the segment described by the following two arguments.
sl@0	6423	*/
sl@0	6424	if( argc==1 ){
sl@0	6425	rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
sl@0	6426	}else{
sl@0	6427	rc = sql_get_statement(v, SEGDIR_SELECT_SEGMENT_STMT, &s);
sl@0	6428	if( rc==SQLITE_OK ){
sl@0	6429	rc = sqlite3_bind_int(s, 1, sqlite3_value_int(argv[1]));
sl@0	6430	if( rc==SQLITE_OK ){
sl@0	6431	rc = sqlite3_bind_int(s, 2, sqlite3_value_int(argv[2]));
sl@0	6432	}
sl@0	6433	}
sl@0	6434	}
sl@0	6435
sl@0	6436	if( rc!=SQLITE_OK ){
sl@0	6437	generateError(pContext, "dump_terms", NULL);
sl@0	6438	return;
sl@0	6439	}
sl@0	6440
sl@0	6441	/* Collect the terms for each segment. */
sl@0	6442	sqlite3Fts2HashInit(&terms, FTS2_HASH_STRING, 1);
sl@0	6443	while( (rc = sqlite3_step(s))==SQLITE_ROW ){
sl@0	6444	rc = collectSegmentTerms(v, s, &terms);
sl@0	6445	if( rc!=SQLITE_OK ) break;
sl@0	6446	}
sl@0	6447
sl@0	6448	if( rc!=SQLITE_DONE ){
sl@0	6449	sqlite3_reset(s);
sl@0	6450	generateError(pContext, "dump_terms", NULL);
sl@0	6451	}else{
sl@0	6452	const int nTerms = fts2HashCount(&terms);
sl@0	6453	if( nTerms>0 ){
sl@0	6454	rc = generateTermsResult(pContext, &terms);
sl@0	6455	if( rc==SQLITE_NOMEM ){
sl@0	6456	generateError(pContext, "dump_terms", "out of memory");
sl@0	6457	}else{
sl@0	6458	assert( rc==SQLITE_OK );
sl@0	6459	}
sl@0	6460	}else if( argc==3 ){
sl@0	6461	/* The specific segment asked for could not be found. */
sl@0	6462	generateError(pContext, "dump_terms", "segment not found");
sl@0	6463	}else{
sl@0	6464	/* No segments found. */
sl@0	6465	/* TODO(shess): It should be impossible to reach this. This
sl@0	6466	** case can only happen for an empty table, in which case
sl@0	6467	** SQLite has no rows to call this function on.
sl@0	6468	*/
sl@0	6469	sqlite3_result_null(pContext);
sl@0	6470	}
sl@0	6471	}
sl@0	6472	sqlite3Fts2HashClear(&terms);
sl@0	6473	}
sl@0	6474	}
sl@0	6475
sl@0	6476	/* Expand the DL_DEFAULT doclist in pData into a text result in
sl@0	6477	** pContext.
sl@0	6478	*/
sl@0	6479	static void createDoclistResult(sqlite3_context *pContext,
sl@0	6480	const char *pData, int nData){
sl@0	6481	DataBuffer dump;
sl@0	6482	DLReader dlReader;
sl@0	6483
sl@0	6484	assert( pData!=NULL && nData>0 );
sl@0	6485
sl@0	6486	dataBufferInit(&dump, 0);
sl@0	6487	dlrInit(&dlReader, DL_DEFAULT, pData, nData);
sl@0	6488	for( ; !dlrAtEnd(&dlReader); dlrStep(&dlReader) ){
sl@0	6489	char buf[256];
sl@0	6490	PLReader plReader;
sl@0	6491
sl@0	6492	plrInit(&plReader, &dlReader);
sl@0	6493	if( DL_DEFAULT==DL_DOCIDS \|\| plrAtEnd(&plReader) ){
sl@0	6494	sqlite3_snprintf(sizeof(buf), buf, "[%lld] ", dlrDocid(&dlReader));
sl@0	6495	dataBufferAppend(&dump, buf, strlen(buf));
sl@0	6496	}else{
sl@0	6497	int iColumn = plrColumn(&plReader);
sl@0	6498
sl@0	6499	sqlite3_snprintf(sizeof(buf), buf, "[%lld %d[",
sl@0	6500	dlrDocid(&dlReader), iColumn);
sl@0	6501	dataBufferAppend(&dump, buf, strlen(buf));
sl@0	6502
sl@0	6503	for( ; !plrAtEnd(&plReader); plrStep(&plReader) ){
sl@0	6504	if( plrColumn(&plReader)!=iColumn ){
sl@0	6505	iColumn = plrColumn(&plReader);
sl@0	6506	sqlite3_snprintf(sizeof(buf), buf, "] %d[", iColumn);
sl@0	6507	assert( dump.nData>0 );
sl@0	6508	dump.nData--; /* Overwrite trailing space. */
sl@0	6509	assert( dump.pData[dump.nData]==' ');
sl@0	6510	dataBufferAppend(&dump, buf, strlen(buf));
sl@0	6511	}
sl@0	6512	if( DL_DEFAULT==DL_POSITIONS_OFFSETS ){
sl@0	6513	sqlite3_snprintf(sizeof(buf), buf, "%d,%d,%d ",
sl@0	6514	plrPosition(&plReader),
sl@0	6515	plrStartOffset(&plReader), plrEndOffset(&plReader));
sl@0	6516	}else if( DL_DEFAULT==DL_POSITIONS ){
sl@0	6517	sqlite3_snprintf(sizeof(buf), buf, "%d ", plrPosition(&plReader));
sl@0	6518	}else{
sl@0	6519	assert( NULL=="Unhandled DL_DEFAULT value");
sl@0	6520	}
sl@0	6521	dataBufferAppend(&dump, buf, strlen(buf));
sl@0	6522	}
sl@0	6523	plrDestroy(&plReader);
sl@0	6524
sl@0	6525	assert( dump.nData>0 );
sl@0	6526	dump.nData--; /* Overwrite trailing space. */
sl@0	6527	assert( dump.pData[dump.nData]==' ');
sl@0	6528	dataBufferAppend(&dump, "]] ", 3);
sl@0	6529	}
sl@0	6530	}
sl@0	6531	dlrDestroy(&dlReader);
sl@0	6532
sl@0	6533	assert( dump.nData>0 );
sl@0	6534	dump.nData--; /* Overwrite trailing space. */
sl@0	6535	assert( dump.pData[dump.nData]==' ');
sl@0	6536	dump.pData[dump.nData] = '\0';
sl@0	6537	assert( dump.nData>0 );
sl@0	6538
sl@0	6539	/* Passes ownership of dump's buffer to pContext. */
sl@0	6540	sqlite3_result_text(pContext, dump.pData, dump.nData, sqlite3_free);
sl@0	6541	dump.pData = NULL;
sl@0	6542	dump.nData = dump.nCapacity = 0;
sl@0	6543	}
sl@0	6544
sl@0	6545	/* Implements dump_doclist() for use in inspecting the fts2 index from
sl@0	6546	** tests. TEXT result containing a string representation of the
sl@0	6547	** doclist for the indicated term. dump_doclist(t, term, level, idx)
sl@0	6548	** dumps the doclist for term from the segment specified by level, idx
sl@0	6549	** (in %_segdir), while dump_doclist(t, term) dumps the logical
sl@0	6550	** doclist for the term across all segments. The per-segment doclist
sl@0	6551	** can contain deletions, while the full-index doclist will not
sl@0	6552	** (deletions are omitted).
sl@0	6553	**
sl@0	6554	** Result formats differ with the setting of DL_DEFAULTS. Examples:
sl@0	6555	**
sl@0	6556	** DL_DOCIDS: [1] [3] [7]
sl@0	6557	** DL_POSITIONS: [1 0[0 4] 1[17]] [3 1[5]]
sl@0	6558	** DL_POSITIONS_OFFSETS: [1 0[0,0,3 4,23,26] 1[17,102,105]] [3 1[5,20,23]]
sl@0	6559	**
sl@0	6560	** In each case the number after the outer '[' is the docid. In the
sl@0	6561	** latter two cases, the number before the inner '[' is the column
sl@0	6562	** associated with the values within. For DL_POSITIONS the numbers
sl@0	6563	** within are the positions, for DL_POSITIONS_OFFSETS they are the
sl@0	6564	** position, the start offset, and the end offset.
sl@0	6565	*/
sl@0	6566	static void dumpDoclistFunc(
sl@0	6567	sqlite3_context *pContext,
sl@0	6568	int argc, sqlite3_value **argv
sl@0	6569	){
sl@0	6570	fulltext_cursor *pCursor;
sl@0	6571	if( argc!=2 && argc!=4 ){
sl@0	6572	generateError(pContext, "dump_doclist", "incorrect arguments");
sl@0	6573	}else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB \|\|
sl@0	6574	sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
sl@0	6575	generateError(pContext, "dump_doclist", "illegal first argument");
sl@0	6576	}else if( sqlite3_value_text(argv[1])==NULL \|\|
sl@0	6577	sqlite3_value_text(argv[1])[0]=='\0' ){
sl@0	6578	generateError(pContext, "dump_doclist", "empty second argument");
sl@0	6579	}else{
sl@0	6580	const char pTerm = (const char )sqlite3_value_text(argv[1]);
sl@0	6581	const int nTerm = strlen(pTerm);
sl@0	6582	fulltext_vtab *v;
sl@0	6583	int rc;
sl@0	6584	DataBuffer doclist;
sl@0	6585
sl@0	6586	memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
sl@0	6587	v = cursor_vtab(pCursor);
sl@0	6588
sl@0	6589	dataBufferInit(&doclist, 0);
sl@0	6590
sl@0	6591	/* termSelect() yields the same logical doclist that queries are
sl@0	6592	** run against.
sl@0	6593	*/
sl@0	6594	if( argc==2 ){
sl@0	6595	rc = termSelect(v, v->nColumn, pTerm, nTerm, 0, DL_DEFAULT, &doclist);
sl@0	6596	}else{
sl@0	6597	sqlite3_stmt *s = NULL;
sl@0	6598
sl@0	6599	/* Get our specific segment's information. */
sl@0	6600	rc = sql_get_statement(v, SEGDIR_SELECT_SEGMENT_STMT, &s);
sl@0	6601	if( rc==SQLITE_OK ){
sl@0	6602	rc = sqlite3_bind_int(s, 1, sqlite3_value_int(argv[2]));
sl@0	6603	if( rc==SQLITE_OK ){
sl@0	6604	rc = sqlite3_bind_int(s, 2, sqlite3_value_int(argv[3]));
sl@0	6605	}
sl@0	6606	}
sl@0	6607
sl@0	6608	if( rc==SQLITE_OK ){
sl@0	6609	rc = sqlite3_step(s);
sl@0	6610
sl@0	6611	if( rc==SQLITE_DONE ){
sl@0	6612	dataBufferDestroy(&doclist);
sl@0	6613	generateError(pContext, "dump_doclist", "segment not found");
sl@0	6614	return;
sl@0	6615	}
sl@0	6616
sl@0	6617	/* Found a segment, load it into doclist. */
sl@0	6618	if( rc==SQLITE_ROW ){
sl@0	6619	const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1);
sl@0	6620	const char *pData = sqlite3_column_blob(s, 2);
sl@0	6621	const int nData = sqlite3_column_bytes(s, 2);
sl@0	6622
sl@0	6623	/* loadSegment() is used by termSelect() to load each
sl@0	6624	** segment's data.
sl@0	6625	*/
sl@0	6626	rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, 0,
sl@0	6627	&doclist);
sl@0	6628	if( rc==SQLITE_OK ){
sl@0	6629	rc = sqlite3_step(s);
sl@0	6630
sl@0	6631	/* Should not have more than one matching segment. */
sl@0	6632	if( rc!=SQLITE_DONE ){
sl@0	6633	sqlite3_reset(s);
sl@0	6634	dataBufferDestroy(&doclist);
sl@0	6635	generateError(pContext, "dump_doclist", "invalid segdir");
sl@0	6636	return;
sl@0	6637	}
sl@0	6638	rc = SQLITE_OK;
sl@0	6639	}
sl@0	6640	}
sl@0	6641	}
sl@0	6642
sl@0	6643	sqlite3_reset(s);
sl@0	6644	}
sl@0	6645
sl@0	6646	if( rc==SQLITE_OK ){
sl@0	6647	if( doclist.nData>0 ){
sl@0	6648	createDoclistResult(pContext, doclist.pData, doclist.nData);
sl@0	6649	}else{
sl@0	6650	/* TODO(shess): This can happen if the term is not present, or
sl@0	6651	** if all instances of the term have been deleted and this is
sl@0	6652	** an all-index dump. It may be interesting to distinguish
sl@0	6653	** these cases.
sl@0	6654	*/
sl@0	6655	sqlite3_result_text(pContext, "", 0, SQLITE_STATIC);
sl@0	6656	}
sl@0	6657	}else if( rc==SQLITE_NOMEM ){
sl@0	6658	/* Handle out-of-memory cases specially because if they are
sl@0	6659	** generated in fts2 code they may not be reflected in the db
sl@0	6660	** handle.
sl@0	6661	*/
sl@0	6662	/* TODO(shess): Handle this more comprehensively.
sl@0	6663	** sqlite3ErrStr() has what I need, but is internal.
sl@0	6664	*/
sl@0	6665	generateError(pContext, "dump_doclist", "out of memory");
sl@0	6666	}else{
sl@0	6667	generateError(pContext, "dump_doclist", NULL);
sl@0	6668	}
sl@0	6669
sl@0	6670	dataBufferDestroy(&doclist);
sl@0	6671	}
sl@0	6672	}
sl@0	6673	#endif
sl@0	6674
sl@0	6675	/*
sl@0	6676	** This routine implements the xFindFunction method for the FTS2
sl@0	6677	** virtual table.
sl@0	6678	*/
sl@0	6679	static int fulltextFindFunction(
sl@0	6680	sqlite3_vtab *pVtab,
sl@0	6681	int nArg,
sl@0	6682	const char *zName,
sl@0	6683	void (*pxFunc)(sqlite3_context,int,sqlite3_value**),
sl@0	6684	void **ppArg
sl@0	6685	){
sl@0	6686	if( strcmp(zName,"snippet")==0 ){
sl@0	6687	*pxFunc = snippetFunc;
sl@0	6688	return 1;
sl@0	6689	}else if( strcmp(zName,"offsets")==0 ){
sl@0	6690	*pxFunc = snippetOffsetsFunc;
sl@0	6691	return 1;
sl@0	6692	}else if( strcmp(zName,"optimize")==0 ){
sl@0	6693	*pxFunc = optimizeFunc;
sl@0	6694	return 1;
sl@0	6695	#ifdef SQLITE_TEST
sl@0	6696	/* NOTE(shess): These functions are present only for testing
sl@0	6697	** purposes. No particular effort is made to optimize their
sl@0	6698	** execution or how they build their results.
sl@0	6699	*/
sl@0	6700	}else if( strcmp(zName,"dump_terms")==0 ){
sl@0	6701	/* fprintf(stderr, "Found dump_terms\n"); */
sl@0	6702	*pxFunc = dumpTermsFunc;
sl@0	6703	return 1;
sl@0	6704	}else if( strcmp(zName,"dump_doclist")==0 ){
sl@0	6705	/* fprintf(stderr, "Found dump_doclist\n"); */
sl@0	6706	*pxFunc = dumpDoclistFunc;
sl@0	6707	return 1;
sl@0	6708	#endif
sl@0	6709	}
sl@0	6710	return 0;
sl@0	6711	}
sl@0	6712
sl@0	6713	/*
sl@0	6714	** Rename an fts2 table.
sl@0	6715	*/
sl@0	6716	static int fulltextRename(
sl@0	6717	sqlite3_vtab *pVtab,
sl@0	6718	const char *zName
sl@0	6719	){
sl@0	6720	fulltext_vtab p = (fulltext_vtab )pVtab;
sl@0	6721	int rc = SQLITE_NOMEM;
sl@0	6722	char *zSql = sqlite3_mprintf(
sl@0	6723	"ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';"
sl@0	6724	"ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';"
sl@0	6725	"ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';"
sl@0	6726	, p->zDb, p->zName, zName
sl@0	6727	, p->zDb, p->zName, zName
sl@0	6728	, p->zDb, p->zName, zName
sl@0	6729	);
sl@0	6730	if( zSql ){
sl@0	6731	rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
sl@0	6732	sqlite3_free(zSql);
sl@0	6733	}
sl@0	6734	return rc;
sl@0	6735	}
sl@0	6736
sl@0	6737	static const sqlite3_module fts2Module = {
sl@0	6738	/* iVersion */ 0,
sl@0	6739	/* xCreate */ fulltextCreate,
sl@0	6740	/* xConnect */ fulltextConnect,
sl@0	6741	/* xBestIndex */ fulltextBestIndex,
sl@0	6742	/* xDisconnect */ fulltextDisconnect,
sl@0	6743	/* xDestroy */ fulltextDestroy,
sl@0	6744	/* xOpen */ fulltextOpen,
sl@0	6745	/* xClose */ fulltextClose,
sl@0	6746	/* xFilter */ fulltextFilter,
sl@0	6747	/* xNext */ fulltextNext,
sl@0	6748	/* xEof */ fulltextEof,
sl@0	6749	/* xColumn */ fulltextColumn,
sl@0	6750	/* xRowid */ fulltextRowid,
sl@0	6751	/* xUpdate */ fulltextUpdate,
sl@0	6752	/* xBegin */ fulltextBegin,
sl@0	6753	/* xSync */ fulltextSync,
sl@0	6754	/* xCommit */ fulltextCommit,
sl@0	6755	/* xRollback */ fulltextRollback,
sl@0	6756	/* xFindFunction */ fulltextFindFunction,
sl@0	6757	/* xRename */ fulltextRename,
sl@0	6758	};
sl@0	6759
sl@0	6760	static void hashDestroy(void *p){
sl@0	6761	fts2Hash pHash = (fts2Hash )p;
sl@0	6762	sqlite3Fts2HashClear(pHash);
sl@0	6763	sqlite3_free(pHash);
sl@0	6764	}
sl@0	6765
sl@0	6766	/*
sl@0	6767	** The fts2 built-in tokenizers - "simple" and "porter" - are implemented
sl@0	6768	** in files fts2_tokenizer1.c and fts2_porter.c respectively. The following
sl@0	6769	** two forward declarations are for functions declared in these files
sl@0	6770	** used to retrieve the respective implementations.
sl@0	6771	**
sl@0	6772	** Calling sqlite3Fts2SimpleTokenizerModule() sets the value pointed
sl@0	6773	** to by the argument to point a the "simple" tokenizer implementation.
sl@0	6774	** Function ...PorterTokenizerModule() sets *pModule to point to the
sl@0	6775	** porter tokenizer/stemmer implementation.
sl@0	6776	*/
sl@0	6777	void sqlite3Fts2SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
sl@0	6778	void sqlite3Fts2PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
sl@0	6779	void sqlite3Fts2IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
sl@0	6780
sl@0	6781	int sqlite3Fts2InitHashTable(sqlite3 , fts2Hash , const char *);
sl@0	6782
sl@0	6783	/*
sl@0	6784	** Initialise the fts2 extension. If this extension is built as part
sl@0	6785	** of the sqlite library, then this function is called directly by
sl@0	6786	** SQLite. If fts2 is built as a dynamically loadable extension, this
sl@0	6787	** function is called by the sqlite3_extension_init() entry point.
sl@0	6788	*/
sl@0	6789	int sqlite3Fts2Init(sqlite3 *db){
sl@0	6790	int rc = SQLITE_OK;
sl@0	6791	fts2Hash *pHash = 0;
sl@0	6792	const sqlite3_tokenizer_module *pSimple = 0;
sl@0	6793	const sqlite3_tokenizer_module *pPorter = 0;
sl@0	6794	const sqlite3_tokenizer_module *pIcu = 0;
sl@0	6795
sl@0	6796	sqlite3Fts2SimpleTokenizerModule(&pSimple);
sl@0	6797	sqlite3Fts2PorterTokenizerModule(&pPorter);
sl@0	6798	#ifdef SQLITE_ENABLE_ICU
sl@0	6799	sqlite3Fts2IcuTokenizerModule(&pIcu);
sl@0	6800	#endif
sl@0	6801
sl@0	6802	/* Allocate and initialise the hash-table used to store tokenizers. */
sl@0	6803	pHash = sqlite3_malloc(sizeof(fts2Hash));
sl@0	6804	if( !pHash ){
sl@0	6805	rc = SQLITE_NOMEM;
sl@0	6806	}else{
sl@0	6807	sqlite3Fts2HashInit(pHash, FTS2_HASH_STRING, 1);
sl@0	6808	}
sl@0	6809
sl@0	6810	/* Load the built-in tokenizers into the hash table */
sl@0	6811	if( rc==SQLITE_OK ){
sl@0	6812	if( sqlite3Fts2HashInsert(pHash, "simple", 7, (void *)pSimple)
sl@0	6813	\|\| sqlite3Fts2HashInsert(pHash, "porter", 7, (void *)pPorter)
sl@0	6814	\|\| (pIcu && sqlite3Fts2HashInsert(pHash, "icu", 4, (void *)pIcu))
sl@0	6815	){
sl@0	6816	rc = SQLITE_NOMEM;
sl@0	6817	}
sl@0	6818	}
sl@0	6819
sl@0	6820	/* Create the virtual table wrapper around the hash-table and overload
sl@0	6821	** the two scalar functions. If this is successful, register the
sl@0	6822	** module with sqlite.
sl@0	6823	*/
sl@0	6824	if( SQLITE_OK==rc
sl@0	6825	&& SQLITE_OK==(rc = sqlite3Fts2InitHashTable(db, pHash, "fts2_tokenizer"))
sl@0	6826	&& SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
sl@0	6827	&& SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", -1))
sl@0	6828	&& SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", -1))
sl@0	6829	#ifdef SQLITE_TEST
sl@0	6830	&& SQLITE_OK==(rc = sqlite3_overload_function(db, "dump_terms", -1))
sl@0	6831	&& SQLITE_OK==(rc = sqlite3_overload_function(db, "dump_doclist", -1))
sl@0	6832	#endif
sl@0	6833	){
sl@0	6834	return sqlite3_create_module_v2(
sl@0	6835	db, "fts2", &fts2Module, (void *)pHash, hashDestroy
sl@0	6836	);
sl@0	6837	}
sl@0	6838
sl@0	6839	/* An error has occured. Delete the hash table and return the error code. */
sl@0	6840	assert( rc!=SQLITE_OK );
sl@0	6841	if( pHash ){
sl@0	6842	sqlite3Fts2HashClear(pHash);
sl@0	6843	sqlite3_free(pHash);
sl@0	6844	}
sl@0	6845	return rc;
sl@0	6846	}
sl@0	6847
sl@0	6848	#if !SQLITE_CORE
sl@0	6849	int sqlite3_extension_init(
sl@0	6850	sqlite3 *db,
sl@0	6851	char **pzErrMsg,
sl@0	6852	const sqlite3_api_routines *pApi
sl@0	6853	){
sl@0	6854	SQLITE_EXTENSION_INIT2(pApi)
sl@0	6855	return sqlite3Fts2Init(db);
sl@0	6856	}
sl@0	6857	#endif
sl@0	6858
sl@0	6859	#endif /* !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS2) */

author	sl@SLION-WIN7.fritz.box
	Fri, 15 Jun 2012 03:10:57 +0200
changeset 0	bde4ae8d615e
permissions	-rw-r--r--