4 ** The author disclaims copyright to this source code.
6 *************************************************************************
7 ** Defines the interface to tokenizers used by fulltext-search. There
8 ** are three basic components:
10 ** sqlite3_tokenizer_module is a singleton defining the tokenizer
11 ** interface functions. This is essentially the class structure for
14 ** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
15 ** including customization information defined at creation time.
17 ** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
18 ** tokens from a particular input.
20 #ifndef _FTS1_TOKENIZER_H_
21 #define _FTS1_TOKENIZER_H_
23 /* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
24 ** If tokenizers are to be allowed to call sqlite3_*() functions, then
25 ** we will need a way to register the API consistently.
30 ** Structures used by the tokenizer interface.
32 typedef struct sqlite3_tokenizer sqlite3_tokenizer;
33 typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
34 typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
36 struct sqlite3_tokenizer_module {
37 int iVersion; /* currently 0 */
40 ** Create and destroy a tokenizer. argc/argv are passed down from
41 ** the fulltext virtual table creation to allow customization.
43 int (*xCreate)(int argc, const char *const*argv,
44 sqlite3_tokenizer **ppTokenizer);
45 int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
48 ** Tokenize a particular input. Call xOpen() to prepare to
49 ** tokenize, xNext() repeatedly until it returns SQLITE_DONE, then
50 ** xClose() to free any internal state. The pInput passed to
51 ** xOpen() must exist until the cursor is closed. The ppToken
52 ** result from xNext() is only valid until the next call to xNext()
53 ** or until xClose() is called.
55 /* TODO(shess) current implementation requires pInput to be
56 ** nul-terminated. This should either be fixed, or pInput/nBytes
57 ** should be converted to zInput.
59 int (*xOpen)(sqlite3_tokenizer *pTokenizer,
60 const char *pInput, int nBytes,
61 sqlite3_tokenizer_cursor **ppCursor);
62 int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
63 int (*xNext)(sqlite3_tokenizer_cursor *pCursor,
64 const char **ppToken, int *pnBytes,
65 int *piStartOffset, int *piEndOffset, int *piPosition);
68 struct sqlite3_tokenizer {
69 const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */
70 /* Tokenizer implementations will typically add additional fields */
73 struct sqlite3_tokenizer_cursor {
74 sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */
75 /* Tokenizer implementations will typically add additional fields */
79 ** Get the module for a tokenizer which generates tokens based on a
80 ** set of non-token characters. The default is to break tokens at any
81 ** non-alnum character, though the set of delimiters can also be
82 ** specified by the first argv argument to xCreate().
84 /* TODO(shess) This doesn't belong here. Need some sort of
85 ** registration process.
87 void sqlite3Fts1SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
88 void sqlite3Fts1PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
90 #endif /* _FTS1_TOKENIZER_H_ */