1.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     1.2 +++ b/os/persistentdata/persistentstorage/sqlite3api/SQLite/fts1_tokenizer.h	Fri Jun 15 03:10:57 2012 +0200
     1.3 @@ -0,0 +1,90 @@
     1.4 +/*
     1.5 +** 2006 July 10
     1.6 +**
     1.7 +** The author disclaims copyright to this source code.
     1.8 +**
     1.9 +*************************************************************************
    1.10 +** Defines the interface to tokenizers used by fulltext-search.  There
    1.11 +** are three basic components:
    1.12 +**
    1.13 +** sqlite3_tokenizer_module is a singleton defining the tokenizer
    1.14 +** interface functions.  This is essentially the class structure for
    1.15 +** tokenizers.
    1.16 +**
    1.17 +** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
    1.18 +** including customization information defined at creation time.
    1.19 +**
    1.20 +** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
    1.21 +** tokens from a particular input.
    1.22 +*/
    1.23 +#ifndef _FTS1_TOKENIZER_H_
    1.24 +#define _FTS1_TOKENIZER_H_
    1.25 +
    1.26 +/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
    1.27 +** If tokenizers are to be allowed to call sqlite3_*() functions, then
    1.28 +** we will need a way to register the API consistently.
    1.29 +*/
    1.30 +#include "sqlite3.h"
    1.31 +
    1.32 +/*
    1.33 +** Structures used by the tokenizer interface.
    1.34 +*/
    1.35 +typedef struct sqlite3_tokenizer sqlite3_tokenizer;
    1.36 +typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
    1.37 +typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
    1.38 +
    1.39 +struct sqlite3_tokenizer_module {
    1.40 +  int iVersion;                  /* currently 0 */
    1.41 +
    1.42 +  /*
    1.43 +  ** Create and destroy a tokenizer.  argc/argv are passed down from
    1.44 +  ** the fulltext virtual table creation to allow customization.
    1.45 +  */
    1.46 +  int (*xCreate)(int argc, const char *const*argv,
    1.47 +                 sqlite3_tokenizer **ppTokenizer);
    1.48 +  int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
    1.49 +
    1.50 +  /*
    1.51 +  ** Tokenize a particular input.  Call xOpen() to prepare to
    1.52 +  ** tokenize, xNext() repeatedly until it returns SQLITE_DONE, then
    1.53 +  ** xClose() to free any internal state.  The pInput passed to
    1.54 +  ** xOpen() must exist until the cursor is closed.  The ppToken
    1.55 +  ** result from xNext() is only valid until the next call to xNext()
    1.56 +  ** or until xClose() is called.
    1.57 +  */
    1.58 +  /* TODO(shess) current implementation requires pInput to be
    1.59 +  ** nul-terminated.  This should either be fixed, or pInput/nBytes
    1.60 +  ** should be converted to zInput.
    1.61 +  */
    1.62 +  int (*xOpen)(sqlite3_tokenizer *pTokenizer,
    1.63 +               const char *pInput, int nBytes,
    1.64 +               sqlite3_tokenizer_cursor **ppCursor);
    1.65 +  int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
    1.66 +  int (*xNext)(sqlite3_tokenizer_cursor *pCursor,
    1.67 +               const char **ppToken, int *pnBytes,
    1.68 +               int *piStartOffset, int *piEndOffset, int *piPosition);
    1.69 +};
    1.70 +
    1.71 +struct sqlite3_tokenizer {
    1.72 +  const sqlite3_tokenizer_module *pModule;  /* The module for this tokenizer */
    1.73 +  /* Tokenizer implementations will typically add additional fields */
    1.74 +};
    1.75 +
    1.76 +struct sqlite3_tokenizer_cursor {
    1.77 +  sqlite3_tokenizer *pTokenizer;       /* Tokenizer for this cursor. */
    1.78 +  /* Tokenizer implementations will typically add additional fields */
    1.79 +};
    1.80 +
    1.81 +/*
    1.82 +** Get the module for a tokenizer which generates tokens based on a
    1.83 +** set of non-token characters.  The default is to break tokens at any
    1.84 +** non-alnum character, though the set of delimiters can also be
    1.85 +** specified by the first argv argument to xCreate().
    1.86 +*/
    1.87 +/* TODO(shess) This doesn't belong here.  Need some sort of
    1.88 +** registration process.
    1.89 +*/
    1.90 +void sqlite3Fts1SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
    1.91 +void sqlite3Fts1PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
    1.92 +
    1.93 +#endif /* _FTS1_TOKENIZER_H_ */