1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/os/persistentdata/persistentstorage/sqlite3api/SQLite/fts3_tokenizer1.c Fri Jun 15 03:10:57 2012 +0200
1.3 @@ -0,0 +1,230 @@
1.4 +/*
1.5 +** 2006 Oct 10
1.6 +**
1.7 +** The author disclaims copyright to this source code. In place of
1.8 +** a legal notice, here is a blessing:
1.9 +**
1.10 +** May you do good and not evil.
1.11 +** May you find forgiveness for yourself and forgive others.
1.12 +** May you share freely, never taking more than you give.
1.13 +**
1.14 +******************************************************************************
1.15 +**
1.16 +** Implementation of the "simple" full-text-search tokenizer.
1.17 +*/
1.18 +
1.19 +/*
1.20 +** The code in this file is only compiled if:
1.21 +**
1.22 +** * The FTS3 module is being built as an extension
1.23 +** (in which case SQLITE_CORE is not defined), or
1.24 +**
1.25 +** * The FTS3 module is being built into the core of
1.26 +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
1.27 +*/
1.28 +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
1.29 +
1.30 +
1.31 +#include <assert.h>
1.32 +#include <stdlib.h>
1.33 +#include <stdio.h>
1.34 +#include <string.h>
1.35 +#include <ctype.h>
1.36 +
1.37 +#include "fts3_tokenizer.h"
1.38 +
1.39 +typedef struct simple_tokenizer {
1.40 + sqlite3_tokenizer base;
1.41 + char delim[128]; /* flag ASCII delimiters */
1.42 +} simple_tokenizer;
1.43 +
1.44 +typedef struct simple_tokenizer_cursor {
1.45 + sqlite3_tokenizer_cursor base;
1.46 + const char *pInput; /* input we are tokenizing */
1.47 + int nBytes; /* size of the input */
1.48 + int iOffset; /* current position in pInput */
1.49 + int iToken; /* index of next token to be returned */
1.50 + char *pToken; /* storage for current token */
1.51 + int nTokenAllocated; /* space allocated to zToken buffer */
1.52 +} simple_tokenizer_cursor;
1.53 +
1.54 +
1.55 +/* Forward declaration */
1.56 +static const sqlite3_tokenizer_module simpleTokenizerModule;
1.57 +
1.58 +static int simpleDelim(simple_tokenizer *t, unsigned char c){
1.59 + return c<0x80 && t->delim[c];
1.60 +}
1.61 +
1.62 +/*
1.63 +** Create a new tokenizer instance.
1.64 +*/
1.65 +static int simpleCreate(
1.66 + int argc, const char * const *argv,
1.67 + sqlite3_tokenizer **ppTokenizer
1.68 +){
1.69 + simple_tokenizer *t;
1.70 +
1.71 + t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t));
1.72 + if( t==NULL ) return SQLITE_NOMEM;
1.73 + memset(t, 0, sizeof(*t));
1.74 +
1.75 + /* TODO(shess) Delimiters need to remain the same from run to run,
1.76 + ** else we need to reindex. One solution would be a meta-table to
1.77 + ** track such information in the database, then we'd only want this
1.78 + ** information on the initial create.
1.79 + */
1.80 + if( argc>1 ){
1.81 + int i, n = strlen(argv[1]);
1.82 + for(i=0; i<n; i++){
1.83 + unsigned char ch = argv[1][i];
1.84 + /* We explicitly don't support UTF-8 delimiters for now. */
1.85 + if( ch>=0x80 ){
1.86 + sqlite3_free(t);
1.87 + return SQLITE_ERROR;
1.88 + }
1.89 + t->delim[ch] = 1;
1.90 + }
1.91 + } else {
1.92 + /* Mark non-alphanumeric ASCII characters as delimiters */
1.93 + int i;
1.94 + for(i=1; i<0x80; i++){
1.95 + t->delim[i] = !isalnum(i);
1.96 + }
1.97 + }
1.98 +
1.99 + *ppTokenizer = &t->base;
1.100 + return SQLITE_OK;
1.101 +}
1.102 +
1.103 +/*
1.104 +** Destroy a tokenizer
1.105 +*/
1.106 +static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
1.107 + sqlite3_free(pTokenizer);
1.108 + return SQLITE_OK;
1.109 +}
1.110 +
1.111 +/*
1.112 +** Prepare to begin tokenizing a particular string. The input
1.113 +** string to be tokenized is pInput[0..nBytes-1]. A cursor
1.114 +** used to incrementally tokenize this string is returned in
1.115 +** *ppCursor.
1.116 +*/
1.117 +static int simpleOpen(
1.118 + sqlite3_tokenizer *pTokenizer, /* The tokenizer */
1.119 + const char *pInput, int nBytes, /* String to be tokenized */
1.120 + sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
1.121 +){
1.122 + simple_tokenizer_cursor *c;
1.123 +
1.124 + c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
1.125 + if( c==NULL ) return SQLITE_NOMEM;
1.126 +
1.127 + c->pInput = pInput;
1.128 + if( pInput==0 ){
1.129 + c->nBytes = 0;
1.130 + }else if( nBytes<0 ){
1.131 + c->nBytes = (int)strlen(pInput);
1.132 + }else{
1.133 + c->nBytes = nBytes;
1.134 + }
1.135 + c->iOffset = 0; /* start tokenizing at the beginning */
1.136 + c->iToken = 0;
1.137 + c->pToken = NULL; /* no space allocated, yet. */
1.138 + c->nTokenAllocated = 0;
1.139 +
1.140 + *ppCursor = &c->base;
1.141 + return SQLITE_OK;
1.142 +}
1.143 +
1.144 +/*
1.145 +** Close a tokenization cursor previously opened by a call to
1.146 +** simpleOpen() above.
1.147 +*/
1.148 +static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
1.149 + simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
1.150 + sqlite3_free(c->pToken);
1.151 + sqlite3_free(c);
1.152 + return SQLITE_OK;
1.153 +}
1.154 +
1.155 +/*
1.156 +** Extract the next token from a tokenization cursor. The cursor must
1.157 +** have been opened by a prior call to simpleOpen().
1.158 +*/
1.159 +static int simpleNext(
1.160 + sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */
1.161 + const char **ppToken, /* OUT: *ppToken is the token text */
1.162 + int *pnBytes, /* OUT: Number of bytes in token */
1.163 + int *piStartOffset, /* OUT: Starting offset of token */
1.164 + int *piEndOffset, /* OUT: Ending offset of token */
1.165 + int *piPosition /* OUT: Position integer of token */
1.166 +){
1.167 + simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
1.168 + simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
1.169 + unsigned char *p = (unsigned char *)c->pInput;
1.170 +
1.171 + while( c->iOffset<c->nBytes ){
1.172 + int iStartOffset;
1.173 +
1.174 + /* Scan past delimiter characters */
1.175 + while( c->iOffset<c->nBytes && simpleDelim(t, p[c->iOffset]) ){
1.176 + c->iOffset++;
1.177 + }
1.178 +
1.179 + /* Count non-delimiter characters. */
1.180 + iStartOffset = c->iOffset;
1.181 + while( c->iOffset<c->nBytes && !simpleDelim(t, p[c->iOffset]) ){
1.182 + c->iOffset++;
1.183 + }
1.184 +
1.185 + if( c->iOffset>iStartOffset ){
1.186 + int i, n = c->iOffset-iStartOffset;
1.187 + if( n>c->nTokenAllocated ){
1.188 + c->nTokenAllocated = n+20;
1.189 + c->pToken = sqlite3_realloc(c->pToken, c->nTokenAllocated);
1.190 + if( c->pToken==NULL ) return SQLITE_NOMEM;
1.191 + }
1.192 + for(i=0; i<n; i++){
1.193 + /* TODO(shess) This needs expansion to handle UTF-8
1.194 + ** case-insensitivity.
1.195 + */
1.196 + unsigned char ch = p[iStartOffset+i];
1.197 + c->pToken[i] = ch<0x80 ? tolower(ch) : ch;
1.198 + }
1.199 + *ppToken = c->pToken;
1.200 + *pnBytes = n;
1.201 + *piStartOffset = iStartOffset;
1.202 + *piEndOffset = c->iOffset;
1.203 + *piPosition = c->iToken++;
1.204 +
1.205 + return SQLITE_OK;
1.206 + }
1.207 + }
1.208 + return SQLITE_DONE;
1.209 +}
1.210 +
1.211 +/*
1.212 +** The set of routines that implement the simple tokenizer
1.213 +*/
1.214 +static const sqlite3_tokenizer_module simpleTokenizerModule = {
1.215 + 0,
1.216 + simpleCreate,
1.217 + simpleDestroy,
1.218 + simpleOpen,
1.219 + simpleClose,
1.220 + simpleNext,
1.221 +};
1.222 +
1.223 +/*
1.224 +** Allocate a new simple tokenizer. Return a pointer to the new
1.225 +** tokenizer in *ppModule
1.226 +*/
1.227 +void sqlite3Fts3SimpleTokenizerModule(
1.228 + sqlite3_tokenizer_module const**ppModule
1.229 +){
1.230 + *ppModule = &simpleTokenizerModule;
1.231 +}
1.232 +
1.233 +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */