sl@0: /* sl@0: ** 2006 July 10 sl@0: ** sl@0: ** The author disclaims copyright to this source code. sl@0: ** sl@0: ************************************************************************* sl@0: ** Defines the interface to tokenizers used by fulltext-search. There sl@0: ** are three basic components: sl@0: ** sl@0: ** sqlite3_tokenizer_module is a singleton defining the tokenizer sl@0: ** interface functions. This is essentially the class structure for sl@0: ** tokenizers. sl@0: ** sl@0: ** sqlite3_tokenizer is used to define a particular tokenizer, perhaps sl@0: ** including customization information defined at creation time. sl@0: ** sl@0: ** sqlite3_tokenizer_cursor is generated by a tokenizer to generate sl@0: ** tokens from a particular input. sl@0: */ sl@0: #ifndef _FTS3_TOKENIZER_H_ sl@0: #define _FTS3_TOKENIZER_H_ sl@0: sl@0: /* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. sl@0: ** If tokenizers are to be allowed to call sqlite3_*() functions, then sl@0: ** we will need a way to register the API consistently. sl@0: */ sl@0: #include "sqlite3.h" sl@0: sl@0: /* sl@0: ** Structures used by the tokenizer interface. When a new tokenizer sl@0: ** implementation is registered, the caller provides a pointer to sl@0: ** an sqlite3_tokenizer_module containing pointers to the callback sl@0: ** functions that make up an implementation. sl@0: ** sl@0: ** When an fts3 table is created, it passes any arguments passed to sl@0: ** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the sl@0: ** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer sl@0: ** implementation. The xCreate() function in turn returns an sl@0: ** sqlite3_tokenizer structure representing the specific tokenizer to sl@0: ** be used for the fts3 table (customized by the tokenizer clause arguments). sl@0: ** sl@0: ** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen() sl@0: ** method is called. It returns an sqlite3_tokenizer_cursor object sl@0: ** that may be used to tokenize a specific input buffer based on sl@0: ** the tokenization rules supplied by a specific sqlite3_tokenizer sl@0: ** object. sl@0: */ sl@0: typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module; sl@0: typedef struct sqlite3_tokenizer sqlite3_tokenizer; sl@0: typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; sl@0: sl@0: struct sqlite3_tokenizer_module { sl@0: sl@0: /* sl@0: ** Structure version. Should always be set to 0. sl@0: */ sl@0: int iVersion; sl@0: sl@0: /* sl@0: ** Create a new tokenizer. The values in the argv[] array are the sl@0: ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL sl@0: ** TABLE statement that created the fts3 table. For example, if sl@0: ** the following SQL is executed: sl@0: ** sl@0: ** CREATE .. USING fts3( ... , tokenizer arg1 arg2) sl@0: ** sl@0: ** then argc is set to 2, and the argv[] array contains pointers sl@0: ** to the strings "arg1" and "arg2". sl@0: ** sl@0: ** This method should return either SQLITE_OK (0), or an SQLite error sl@0: ** code. If SQLITE_OK is returned, then *ppTokenizer should be set sl@0: ** to point at the newly created tokenizer structure. The generic sl@0: ** sqlite3_tokenizer.pModule variable should not be initialised by sl@0: ** this callback. The caller will do so. sl@0: */ sl@0: int (*xCreate)( sl@0: int argc, /* Size of argv array */ sl@0: const char *const*argv, /* Tokenizer argument strings */ sl@0: sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ sl@0: ); sl@0: sl@0: /* sl@0: ** Destroy an existing tokenizer. The fts3 module calls this method sl@0: ** exactly once for each successful call to xCreate(). sl@0: */ sl@0: int (*xDestroy)(sqlite3_tokenizer *pTokenizer); sl@0: sl@0: /* sl@0: ** Create a tokenizer cursor to tokenize an input buffer. The caller sl@0: ** is responsible for ensuring that the input buffer remains valid sl@0: ** until the cursor is closed (using the xClose() method). sl@0: */ sl@0: int (*xOpen)( sl@0: sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ sl@0: const char *pInput, int nBytes, /* Input buffer */ sl@0: sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */ sl@0: ); sl@0: sl@0: /* sl@0: ** Destroy an existing tokenizer cursor. The fts3 module calls this sl@0: ** method exactly once for each successful call to xOpen(). sl@0: */ sl@0: int (*xClose)(sqlite3_tokenizer_cursor *pCursor); sl@0: sl@0: /* sl@0: ** Retrieve the next token from the tokenizer cursor pCursor. This sl@0: ** method should either return SQLITE_OK and set the values of the sl@0: ** "OUT" variables identified below, or SQLITE_DONE to indicate that sl@0: ** the end of the buffer has been reached, or an SQLite error code. sl@0: ** sl@0: ** *ppToken should be set to point at a buffer containing the sl@0: ** normalized version of the token (i.e. after any case-folding and/or sl@0: ** stemming has been performed). *pnBytes should be set to the length sl@0: ** of this buffer in bytes. The input text that generated the token is sl@0: ** identified by the byte offsets returned in *piStartOffset and sl@0: ** *piEndOffset. sl@0: ** sl@0: ** The buffer *ppToken is set to point at is managed by the tokenizer sl@0: ** implementation. It is only required to be valid until the next call sl@0: ** to xNext() or xClose(). sl@0: */ sl@0: /* TODO(shess) current implementation requires pInput to be sl@0: ** nul-terminated. This should either be fixed, or pInput/nBytes sl@0: ** should be converted to zInput. sl@0: */ sl@0: int (*xNext)( sl@0: sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */ sl@0: const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */ sl@0: int *piStartOffset, /* OUT: Byte offset of token in input buffer */ sl@0: int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ sl@0: int *piPosition /* OUT: Number of tokens returned before this one */ sl@0: ); sl@0: }; sl@0: sl@0: struct sqlite3_tokenizer { sl@0: const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ sl@0: /* Tokenizer implementations will typically add additional fields */ sl@0: }; sl@0: sl@0: struct sqlite3_tokenizer_cursor { sl@0: sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */ sl@0: /* Tokenizer implementations will typically add additional fields */ sl@0: }; sl@0: sl@0: #endif /* _FTS3_TOKENIZER_H_ */