sl@0: /*
sl@0: ** 2006 July 10
sl@0: **
sl@0: ** The author disclaims copyright to this source code.
sl@0: **
sl@0: *************************************************************************
sl@0: ** Defines the interface to tokenizers used by fulltext-search.  There
sl@0: ** are three basic components:
sl@0: **
sl@0: ** sqlite3_tokenizer_module is a singleton defining the tokenizer
sl@0: ** interface functions.  This is essentially the class structure for
sl@0: ** tokenizers.
sl@0: **
sl@0: ** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
sl@0: ** including customization information defined at creation time.
sl@0: **
sl@0: ** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
sl@0: ** tokens from a particular input.
sl@0: */
sl@0: #ifndef _FTS1_TOKENIZER_H_
sl@0: #define _FTS1_TOKENIZER_H_
sl@0: 
sl@0: /* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
sl@0: ** If tokenizers are to be allowed to call sqlite3_*() functions, then
sl@0: ** we will need a way to register the API consistently.
sl@0: */
sl@0: #include "sqlite3.h"
sl@0: 
sl@0: /*
sl@0: ** Structures used by the tokenizer interface.
sl@0: */
sl@0: typedef struct sqlite3_tokenizer sqlite3_tokenizer;
sl@0: typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
sl@0: typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
sl@0: 
sl@0: struct sqlite3_tokenizer_module {
sl@0:   int iVersion;                  /* currently 0 */
sl@0: 
sl@0:   /*
sl@0:   ** Create and destroy a tokenizer.  argc/argv are passed down from
sl@0:   ** the fulltext virtual table creation to allow customization.
sl@0:   */
sl@0:   int (*xCreate)(int argc, const char *const*argv,
sl@0:                  sqlite3_tokenizer **ppTokenizer);
sl@0:   int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
sl@0: 
sl@0:   /*
sl@0:   ** Tokenize a particular input.  Call xOpen() to prepare to
sl@0:   ** tokenize, xNext() repeatedly until it returns SQLITE_DONE, then
sl@0:   ** xClose() to free any internal state.  The pInput passed to
sl@0:   ** xOpen() must exist until the cursor is closed.  The ppToken
sl@0:   ** result from xNext() is only valid until the next call to xNext()
sl@0:   ** or until xClose() is called.
sl@0:   */
sl@0:   /* TODO(shess) current implementation requires pInput to be
sl@0:   ** nul-terminated.  This should either be fixed, or pInput/nBytes
sl@0:   ** should be converted to zInput.
sl@0:   */
sl@0:   int (*xOpen)(sqlite3_tokenizer *pTokenizer,
sl@0:                const char *pInput, int nBytes,
sl@0:                sqlite3_tokenizer_cursor **ppCursor);
sl@0:   int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
sl@0:   int (*xNext)(sqlite3_tokenizer_cursor *pCursor,
sl@0:                const char **ppToken, int *pnBytes,
sl@0:                int *piStartOffset, int *piEndOffset, int *piPosition);
sl@0: };
sl@0: 
sl@0: struct sqlite3_tokenizer {
sl@0:   const sqlite3_tokenizer_module *pModule;  /* The module for this tokenizer */
sl@0:   /* Tokenizer implementations will typically add additional fields */
sl@0: };
sl@0: 
sl@0: struct sqlite3_tokenizer_cursor {
sl@0:   sqlite3_tokenizer *pTokenizer;       /* Tokenizer for this cursor. */
sl@0:   /* Tokenizer implementations will typically add additional fields */
sl@0: };
sl@0: 
sl@0: /*
sl@0: ** Get the module for a tokenizer which generates tokens based on a
sl@0: ** set of non-token characters.  The default is to break tokens at any
sl@0: ** non-alnum character, though the set of delimiters can also be
sl@0: ** specified by the first argv argument to xCreate().
sl@0: */
sl@0: /* TODO(shess) This doesn't belong here.  Need some sort of
sl@0: ** registration process.
sl@0: */
sl@0: void sqlite3Fts1SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
sl@0: void sqlite3Fts1PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
sl@0: 
sl@0: #endif /* _FTS1_TOKENIZER_H_ */