/*

 ** 2007 June 22

 **

 ** The author disclaims copyright to this source code.  In place of

 ** a legal notice, here is a blessing:

 **

 **    May you do good and not evil.

 **    May you find forgiveness for yourself and forgive others.

 **    May you share freely, never taking more than you give.

 **

 ******************************************************************************

 **

 ** This is part of an SQLite module implementing full-text search.

 ** This particular file implements the generic tokenizer interface.

 */

 /*

 ** The code in this file is only compiled if:

 **

 **     * The FTS2 module is being built as an extension

 **       (in which case SQLITE_CORE is not defined), or

 **

 **     * The FTS2 module is being built into the core of

 **       SQLite (in which case SQLITE_ENABLE_FTS2 is defined).

 */

 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)

 #include "sqlite3.h"

 #include "sqlite3ext.h"

 SQLITE_EXTENSION_INIT1

 #include "fts2_hash.h"

 #include "fts2_tokenizer.h"

 #include <assert.h>

 /*

 ** Implementation of the SQL scalar function for accessing the underlying 

 ** hash table. This function may be called as follows:

 **

 **   SELECT <function-name>(<key-name>);

 **   SELECT <function-name>(<key-name>, <pointer>);

 **

 ** where <function-name> is the name passed as the second argument

 ** to the sqlite3Fts2InitHashTable() function (e.g. 'fts2_tokenizer').

 **

 ** If the <pointer> argument is specified, it must be a blob value

 ** containing a pointer to be stored as the hash data corresponding

 ** to the string <key-name>. If <pointer> is not specified, then

 ** the string <key-name> must already exist in the has table. Otherwise,

 ** an error is returned.

 **

 ** Whether or not the <pointer> argument is specified, the value returned

 ** is a blob containing the pointer stored as the hash data corresponding

 ** to string <key-name> (after the hash-table is updated, if applicable).

 */

 static void scalarFunc(

   sqlite3_context *context,

   int argc,

   sqlite3_value **argv

 ){

   fts2Hash *pHash;

   void *pPtr = 0;

   const unsigned char *zName;

   int nName;

   assert( argc==1 || argc==2 );

   pHash = (fts2Hash *)sqlite3_user_data(context);

   zName = sqlite3_value_text(argv[0]);

   nName = sqlite3_value_bytes(argv[0])+1;

   if( argc==2 ){

     void *pOld;

     int n = sqlite3_value_bytes(argv[1]);

     if( n!=sizeof(pPtr) ){

       sqlite3_result_error(context, "argument type mismatch", -1);

       return;

     }

     pPtr = *(void **)sqlite3_value_blob(argv[1]);

     pOld = sqlite3Fts2HashInsert(pHash, (void *)zName, nName, pPtr);

     if( pOld==pPtr ){

       sqlite3_result_error(context, "out of memory", -1);

       return;

     }

   }else{

     pPtr = sqlite3Fts2HashFind(pHash, zName, nName);

     if( !pPtr ){

       char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);

       sqlite3_result_error(context, zErr, -1);

       sqlite3_free(zErr);

       return;

     }

   }

   sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);

 }

 #ifdef SQLITE_TEST

 #include "tcl.h"

 #include <string.h>

 /*

 ** Implementation of a special SQL scalar function for testing tokenizers 

 ** designed to be used in concert with the Tcl testing framework. This

 ** function must be called with two arguments:

 **

 **   SELECT <function-name>(<key-name>, <input-string>);

 **   SELECT <function-name>(<key-name>, <pointer>);

 **

 ** where <function-name> is the name passed as the second argument

 ** to the sqlite3Fts2InitHashTable() function (e.g. 'fts2_tokenizer')

 ** concatenated with the string '_test' (e.g. 'fts2_tokenizer_test').

 **

 ** The return value is a string that may be interpreted as a Tcl

 ** list. For each token in the <input-string>, three elements are

 ** added to the returned list. The first is the token position, the 

 ** second is the token text (folded, stemmed, etc.) and the third is the

 ** substring of <input-string> associated with the token. For example, 

 ** using the built-in "simple" tokenizer:

 **

 **   SELECT fts_tokenizer_test('simple', 'I don't see how');

 **

 ** will return the string:

 **

 **   "{0 i I 1 dont don't 2 see see 3 how how}"

 **   

 */

 static void testFunc(

   sqlite3_context *context,

   int argc,

   sqlite3_value **argv

 ){

   fts2Hash *pHash;

   sqlite3_tokenizer_module *p;

   sqlite3_tokenizer *pTokenizer = 0;

   sqlite3_tokenizer_cursor *pCsr = 0;

   const char *zErr = 0;

   const char *zName;

   int nName;

   const char *zInput;

   int nInput;

   const char *zArg = 0;

   const char *zToken;

   int nToken;

   int iStart;

   int iEnd;

   int iPos;

   Tcl_Obj *pRet;

   assert( argc==2 || argc==3 );

   nName = sqlite3_value_bytes(argv[0]);

   zName = (const char *)sqlite3_value_text(argv[0]);

   nInput = sqlite3_value_bytes(argv[argc-1]);

   zInput = (const char *)sqlite3_value_text(argv[argc-1]);

   if( argc==3 ){

     zArg = (const char *)sqlite3_value_text(argv[1]);

   }

   pHash = (fts2Hash *)sqlite3_user_data(context);

   p = (sqlite3_tokenizer_module *)sqlite3Fts2HashFind(pHash, zName, nName+1);

   if( !p ){

     char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);

     sqlite3_result_error(context, zErr, -1);

     sqlite3_free(zErr);

     return;

   }

   pRet = Tcl_NewObj();

   Tcl_IncrRefCount(pRet);

   if( SQLITE_OK!=p->xCreate(zArg ? 1 : 0, &zArg, &pTokenizer) ){

     zErr = "error in xCreate()";

     goto finish;

   }

   pTokenizer->pModule = p;

   if( SQLITE_OK!=p->xOpen(pTokenizer, zInput, nInput, &pCsr) ){

     zErr = "error in xOpen()";

     goto finish;

   }

   pCsr->pTokenizer = pTokenizer;

   while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){

     Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos));

     Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));

     zToken = &zInput[iStart];

     nToken = iEnd-iStart;

     Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));

   }

   if( SQLITE_OK!=p->xClose(pCsr) ){

     zErr = "error in xClose()";

     goto finish;

   }

   if( SQLITE_OK!=p->xDestroy(pTokenizer) ){

     zErr = "error in xDestroy()";

     goto finish;

   }

 finish:

   if( zErr ){

     sqlite3_result_error(context, zErr, -1);

   }else{

     sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT);

   }

   Tcl_DecrRefCount(pRet);

 }

 static

 int registerTokenizer(

   sqlite3 *db, 

   char *zName, 

   const sqlite3_tokenizer_module *p

 ){

   int rc;

   sqlite3_stmt *pStmt;

   const char zSql[] = "SELECT fts2_tokenizer(?, ?)";

   rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);

   if( rc!=SQLITE_OK ){

     return rc;

   }

   sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);

   sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);

   sqlite3_step(pStmt);

   return sqlite3_finalize(pStmt);

 }

 static

 int queryTokenizer(

   sqlite3 *db, 

   char *zName,  

   const sqlite3_tokenizer_module **pp

 ){

   int rc;

   sqlite3_stmt *pStmt;

   const char zSql[] = "SELECT fts2_tokenizer(?)";

   *pp = 0;

   rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);

   if( rc!=SQLITE_OK ){

     return rc;

   }

   sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);

   if( SQLITE_ROW==sqlite3_step(pStmt) ){

     if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){

       memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));

     }

   }

   return sqlite3_finalize(pStmt);

 }

 void sqlite3Fts2SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);

 /*

 ** Implementation of the scalar function fts2_tokenizer_internal_test().

 ** This function is used for testing only, it is not included in the

 ** build unless SQLITE_TEST is defined.

 **

 ** The purpose of this is to test that the fts2_tokenizer() function

 ** can be used as designed by the C-code in the queryTokenizer and

 ** registerTokenizer() functions above. These two functions are repeated

 ** in the README.tokenizer file as an example, so it is important to

 ** test them.

 **

 ** To run the tests, evaluate the fts2_tokenizer_internal_test() scalar

 ** function with no arguments. An assert() will fail if a problem is

 ** detected. i.e.:

 **

 **     SELECT fts2_tokenizer_internal_test();

 **

 */

 static void intTestFunc(

   sqlite3_context *context,

   int argc,

   sqlite3_value **argv

 ){

   int rc;

   const sqlite3_tokenizer_module *p1;

   const sqlite3_tokenizer_module *p2;

   sqlite3 *db = (sqlite3 *)sqlite3_user_data(context);

   /* Test the query function */

   sqlite3Fts2SimpleTokenizerModule(&p1);

   rc = queryTokenizer(db, "simple", &p2);

   assert( rc==SQLITE_OK );

   assert( p1==p2 );

   rc = queryTokenizer(db, "nosuchtokenizer", &p2);

   assert( rc==SQLITE_ERROR );

   assert( p2==0 );

   assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );

   /* Test the storage function */

   rc = registerTokenizer(db, "nosuchtokenizer", p1);

   assert( rc==SQLITE_OK );

   rc = queryTokenizer(db, "nosuchtokenizer", &p2);

   assert( rc==SQLITE_OK );

   assert( p2==p1 );

   sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);

 }

 #endif

 /*

 ** Set up SQL objects in database db used to access the contents of

 ** the hash table pointed to by argument pHash. The hash table must

 ** been initialised to use string keys, and to take a private copy 

 ** of the key when a value is inserted. i.e. by a call similar to:

 **

 **    sqlite3Fts2HashInit(pHash, FTS2_HASH_STRING, 1);

 **

 ** This function adds a scalar function (see header comment above

 ** scalarFunc() in this file for details) and, if ENABLE_TABLE is

 ** defined at compilation time, a temporary virtual table (see header 

 ** comment above struct HashTableVtab) to the database schema. Both 

 ** provide read/write access to the contents of *pHash.

 **

 ** The third argument to this function, zName, is used as the name

 ** of both the scalar and, if created, the virtual table.

 */

 int sqlite3Fts2InitHashTable(

   sqlite3 *db, 

   fts2Hash *pHash, 

   const char *zName

 ){

   int rc = SQLITE_OK;

   void *p = (void *)pHash;

   const int any = SQLITE_ANY;

   char *zTest = 0;

   char *zTest2 = 0;

 #ifdef SQLITE_TEST

   void *pdb = (void *)db;

   zTest = sqlite3_mprintf("%s_test", zName);

   zTest2 = sqlite3_mprintf("%s_internal_test", zName);

   if( !zTest || !zTest2 ){

     rc = SQLITE_NOMEM;

   }

 #endif

   if( rc!=SQLITE_OK

    || (rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0))

    || (rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0))

 #ifdef SQLITE_TEST

    || (rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0))

    || (rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0))

    || (rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0))

 #endif

   );

   sqlite3_free(zTest);

   sqlite3_free(zTest2);

   return rc;

 }

 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */