1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/os/persistentdata/persistentstorage/sqlite3api/SQLite/fts3_tokenizer.c Fri Jun 15 03:10:57 2012 +0200
1.3 @@ -0,0 +1,371 @@
1.4 +/*
1.5 +** 2007 June 22
1.6 +**
1.7 +** The author disclaims copyright to this source code. In place of
1.8 +** a legal notice, here is a blessing:
1.9 +**
1.10 +** May you do good and not evil.
1.11 +** May you find forgiveness for yourself and forgive others.
1.12 +** May you share freely, never taking more than you give.
1.13 +**
1.14 +******************************************************************************
1.15 +**
1.16 +** This is part of an SQLite module implementing full-text search.
1.17 +** This particular file implements the generic tokenizer interface.
1.18 +*/
1.19 +
1.20 +/*
1.21 +** The code in this file is only compiled if:
1.22 +**
1.23 +** * The FTS3 module is being built as an extension
1.24 +** (in which case SQLITE_CORE is not defined), or
1.25 +**
1.26 +** * The FTS3 module is being built into the core of
1.27 +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
1.28 +*/
1.29 +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
1.30 +
1.31 +#include "sqlite3ext.h"
1.32 +#ifndef SQLITE_CORE
1.33 + SQLITE_EXTENSION_INIT1
1.34 +#endif
1.35 +
1.36 +#include "fts3_hash.h"
1.37 +#include "fts3_tokenizer.h"
1.38 +#include <assert.h>
1.39 +
1.40 +/*
1.41 +** Implementation of the SQL scalar function for accessing the underlying
1.42 +** hash table. This function may be called as follows:
1.43 +**
1.44 +** SELECT <function-name>(<key-name>);
1.45 +** SELECT <function-name>(<key-name>, <pointer>);
1.46 +**
1.47 +** where <function-name> is the name passed as the second argument
1.48 +** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer').
1.49 +**
1.50 +** If the <pointer> argument is specified, it must be a blob value
1.51 +** containing a pointer to be stored as the hash data corresponding
1.52 +** to the string <key-name>. If <pointer> is not specified, then
1.53 +** the string <key-name> must already exist in the has table. Otherwise,
1.54 +** an error is returned.
1.55 +**
1.56 +** Whether or not the <pointer> argument is specified, the value returned
1.57 +** is a blob containing the pointer stored as the hash data corresponding
1.58 +** to string <key-name> (after the hash-table is updated, if applicable).
1.59 +*/
1.60 +static void scalarFunc(
1.61 + sqlite3_context *context,
1.62 + int argc,
1.63 + sqlite3_value **argv
1.64 +){
1.65 + fts3Hash *pHash;
1.66 + void *pPtr = 0;
1.67 + const unsigned char *zName;
1.68 + int nName;
1.69 +
1.70 + assert( argc==1 || argc==2 );
1.71 +
1.72 + pHash = (fts3Hash *)sqlite3_user_data(context);
1.73 +
1.74 + zName = sqlite3_value_text(argv[0]);
1.75 + nName = sqlite3_value_bytes(argv[0])+1;
1.76 +
1.77 + if( argc==2 ){
1.78 + void *pOld;
1.79 + int n = sqlite3_value_bytes(argv[1]);
1.80 + if( n!=sizeof(pPtr) ){
1.81 + sqlite3_result_error(context, "argument type mismatch", -1);
1.82 + return;
1.83 + }
1.84 + pPtr = *(void **)sqlite3_value_blob(argv[1]);
1.85 + pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr);
1.86 + if( pOld==pPtr ){
1.87 + sqlite3_result_error(context, "out of memory", -1);
1.88 + return;
1.89 + }
1.90 + }else{
1.91 + pPtr = sqlite3Fts3HashFind(pHash, zName, nName);
1.92 + if( !pPtr ){
1.93 + char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
1.94 + sqlite3_result_error(context, zErr, -1);
1.95 + sqlite3_free(zErr);
1.96 + return;
1.97 + }
1.98 + }
1.99 +
1.100 + sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
1.101 +}
1.102 +
1.103 +#ifdef SQLITE_TEST
1.104 +
1.105 +#include "tcl.h"
1.106 +#include <string.h>
1.107 +
1.108 +/*
1.109 +** Implementation of a special SQL scalar function for testing tokenizers
1.110 +** designed to be used in concert with the Tcl testing framework. This
1.111 +** function must be called with two arguments:
1.112 +**
1.113 +** SELECT <function-name>(<key-name>, <input-string>);
1.114 +** SELECT <function-name>(<key-name>, <pointer>);
1.115 +**
1.116 +** where <function-name> is the name passed as the second argument
1.117 +** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer')
1.118 +** concatenated with the string '_test' (e.g. 'fts3_tokenizer_test').
1.119 +**
1.120 +** The return value is a string that may be interpreted as a Tcl
1.121 +** list. For each token in the <input-string>, three elements are
1.122 +** added to the returned list. The first is the token position, the
1.123 +** second is the token text (folded, stemmed, etc.) and the third is the
1.124 +** substring of <input-string> associated with the token. For example,
1.125 +** using the built-in "simple" tokenizer:
1.126 +**
1.127 +** SELECT fts_tokenizer_test('simple', 'I don't see how');
1.128 +**
1.129 +** will return the string:
1.130 +**
1.131 +** "{0 i I 1 dont don't 2 see see 3 how how}"
1.132 +**
1.133 +*/
1.134 +static void testFunc(
1.135 + sqlite3_context *context,
1.136 + int argc,
1.137 + sqlite3_value **argv
1.138 +){
1.139 + fts3Hash *pHash;
1.140 + sqlite3_tokenizer_module *p;
1.141 + sqlite3_tokenizer *pTokenizer = 0;
1.142 + sqlite3_tokenizer_cursor *pCsr = 0;
1.143 +
1.144 + const char *zErr = 0;
1.145 +
1.146 + const char *zName;
1.147 + int nName;
1.148 + const char *zInput;
1.149 + int nInput;
1.150 +
1.151 + const char *zArg = 0;
1.152 +
1.153 + const char *zToken;
1.154 + int nToken;
1.155 + int iStart;
1.156 + int iEnd;
1.157 + int iPos;
1.158 +
1.159 + Tcl_Obj *pRet;
1.160 +
1.161 + assert( argc==2 || argc==3 );
1.162 +
1.163 + nName = sqlite3_value_bytes(argv[0]);
1.164 + zName = (const char *)sqlite3_value_text(argv[0]);
1.165 + nInput = sqlite3_value_bytes(argv[argc-1]);
1.166 + zInput = (const char *)sqlite3_value_text(argv[argc-1]);
1.167 +
1.168 + if( argc==3 ){
1.169 + zArg = (const char *)sqlite3_value_text(argv[1]);
1.170 + }
1.171 +
1.172 + pHash = (fts3Hash *)sqlite3_user_data(context);
1.173 + p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1);
1.174 +
1.175 + if( !p ){
1.176 + char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
1.177 + sqlite3_result_error(context, zErr, -1);
1.178 + sqlite3_free(zErr);
1.179 + return;
1.180 + }
1.181 +
1.182 + pRet = Tcl_NewObj();
1.183 + Tcl_IncrRefCount(pRet);
1.184 +
1.185 + if( SQLITE_OK!=p->xCreate(zArg ? 1 : 0, &zArg, &pTokenizer) ){
1.186 + zErr = "error in xCreate()";
1.187 + goto finish;
1.188 + }
1.189 + pTokenizer->pModule = p;
1.190 + if( SQLITE_OK!=p->xOpen(pTokenizer, zInput, nInput, &pCsr) ){
1.191 + zErr = "error in xOpen()";
1.192 + goto finish;
1.193 + }
1.194 + pCsr->pTokenizer = pTokenizer;
1.195 +
1.196 + while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){
1.197 + Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos));
1.198 + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
1.199 + zToken = &zInput[iStart];
1.200 + nToken = iEnd-iStart;
1.201 + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
1.202 + }
1.203 +
1.204 + if( SQLITE_OK!=p->xClose(pCsr) ){
1.205 + zErr = "error in xClose()";
1.206 + goto finish;
1.207 + }
1.208 + if( SQLITE_OK!=p->xDestroy(pTokenizer) ){
1.209 + zErr = "error in xDestroy()";
1.210 + goto finish;
1.211 + }
1.212 +
1.213 +finish:
1.214 + if( zErr ){
1.215 + sqlite3_result_error(context, zErr, -1);
1.216 + }else{
1.217 + sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT);
1.218 + }
1.219 + Tcl_DecrRefCount(pRet);
1.220 +}
1.221 +
1.222 +static
1.223 +int registerTokenizer(
1.224 + sqlite3 *db,
1.225 + char *zName,
1.226 + const sqlite3_tokenizer_module *p
1.227 +){
1.228 + int rc;
1.229 + sqlite3_stmt *pStmt;
1.230 + const char zSql[] = "SELECT fts3_tokenizer(?, ?)";
1.231 +
1.232 + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
1.233 + if( rc!=SQLITE_OK ){
1.234 + return rc;
1.235 + }
1.236 +
1.237 + sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
1.238 + sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
1.239 + sqlite3_step(pStmt);
1.240 +
1.241 + return sqlite3_finalize(pStmt);
1.242 +}
1.243 +
1.244 +static
1.245 +int queryTokenizer(
1.246 + sqlite3 *db,
1.247 + char *zName,
1.248 + const sqlite3_tokenizer_module **pp
1.249 +){
1.250 + int rc;
1.251 + sqlite3_stmt *pStmt;
1.252 + const char zSql[] = "SELECT fts3_tokenizer(?)";
1.253 +
1.254 + *pp = 0;
1.255 + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
1.256 + if( rc!=SQLITE_OK ){
1.257 + return rc;
1.258 + }
1.259 +
1.260 + sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
1.261 + if( SQLITE_ROW==sqlite3_step(pStmt) ){
1.262 + if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
1.263 + memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
1.264 + }
1.265 + }
1.266 +
1.267 + return sqlite3_finalize(pStmt);
1.268 +}
1.269 +
1.270 +void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
1.271 +
1.272 +/*
1.273 +** Implementation of the scalar function fts3_tokenizer_internal_test().
1.274 +** This function is used for testing only, it is not included in the
1.275 +** build unless SQLITE_TEST is defined.
1.276 +**
1.277 +** The purpose of this is to test that the fts3_tokenizer() function
1.278 +** can be used as designed by the C-code in the queryTokenizer and
1.279 +** registerTokenizer() functions above. These two functions are repeated
1.280 +** in the README.tokenizer file as an example, so it is important to
1.281 +** test them.
1.282 +**
1.283 +** To run the tests, evaluate the fts3_tokenizer_internal_test() scalar
1.284 +** function with no arguments. An assert() will fail if a problem is
1.285 +** detected. i.e.:
1.286 +**
1.287 +** SELECT fts3_tokenizer_internal_test();
1.288 +**
1.289 +*/
1.290 +static void intTestFunc(
1.291 + sqlite3_context *context,
1.292 + int argc,
1.293 + sqlite3_value **argv
1.294 +){
1.295 + int rc;
1.296 + const sqlite3_tokenizer_module *p1;
1.297 + const sqlite3_tokenizer_module *p2;
1.298 + sqlite3 *db = (sqlite3 *)sqlite3_user_data(context);
1.299 +
1.300 + /* Test the query function */
1.301 + sqlite3Fts3SimpleTokenizerModule(&p1);
1.302 + rc = queryTokenizer(db, "simple", &p2);
1.303 + assert( rc==SQLITE_OK );
1.304 + assert( p1==p2 );
1.305 + rc = queryTokenizer(db, "nosuchtokenizer", &p2);
1.306 + assert( rc==SQLITE_ERROR );
1.307 + assert( p2==0 );
1.308 + assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
1.309 +
1.310 + /* Test the storage function */
1.311 + rc = registerTokenizer(db, "nosuchtokenizer", p1);
1.312 + assert( rc==SQLITE_OK );
1.313 + rc = queryTokenizer(db, "nosuchtokenizer", &p2);
1.314 + assert( rc==SQLITE_OK );
1.315 + assert( p2==p1 );
1.316 +
1.317 + sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
1.318 +}
1.319 +
1.320 +#endif
1.321 +
1.322 +/*
1.323 +** Set up SQL objects in database db used to access the contents of
1.324 +** the hash table pointed to by argument pHash. The hash table must
1.325 +** been initialised to use string keys, and to take a private copy
1.326 +** of the key when a value is inserted. i.e. by a call similar to:
1.327 +**
1.328 +** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
1.329 +**
1.330 +** This function adds a scalar function (see header comment above
1.331 +** scalarFunc() in this file for details) and, if ENABLE_TABLE is
1.332 +** defined at compilation time, a temporary virtual table (see header
1.333 +** comment above struct HashTableVtab) to the database schema. Both
1.334 +** provide read/write access to the contents of *pHash.
1.335 +**
1.336 +** The third argument to this function, zName, is used as the name
1.337 +** of both the scalar and, if created, the virtual table.
1.338 +*/
1.339 +int sqlite3Fts3InitHashTable(
1.340 + sqlite3 *db,
1.341 + fts3Hash *pHash,
1.342 + const char *zName
1.343 +){
1.344 + int rc = SQLITE_OK;
1.345 + void *p = (void *)pHash;
1.346 + const int any = SQLITE_ANY;
1.347 + char *zTest = 0;
1.348 + char *zTest2 = 0;
1.349 +
1.350 +#ifdef SQLITE_TEST
1.351 + void *pdb = (void *)db;
1.352 + zTest = sqlite3_mprintf("%s_test", zName);
1.353 + zTest2 = sqlite3_mprintf("%s_internal_test", zName);
1.354 + if( !zTest || !zTest2 ){
1.355 + rc = SQLITE_NOMEM;
1.356 + }
1.357 +#endif
1.358 +
1.359 + if( rc!=SQLITE_OK
1.360 + || (rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0))
1.361 + || (rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0))
1.362 +#ifdef SQLITE_TEST
1.363 + || (rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0))
1.364 + || (rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0))
1.365 + || (rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0))
1.366 +#endif
1.367 + );
1.368 +
1.369 + sqlite3_free(zTest);
1.370 + sqlite3_free(zTest2);
1.371 + return rc;
1.372 +}
1.373 +
1.374 +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */