sl@0: /*
sl@0: ** 2006 September 30
sl@0: **
sl@0: ** The author disclaims copyright to this source code.  In place of
sl@0: ** a legal notice, here is a blessing:
sl@0: **
sl@0: **    May you do good and not evil.
sl@0: **    May you find forgiveness for yourself and forgive others.
sl@0: **    May you share freely, never taking more than you give.
sl@0: **
sl@0: *************************************************************************
sl@0: ** Implementation of the full-text-search tokenizer that implements
sl@0: ** a Porter stemmer.
sl@0: */
sl@0: 
sl@0: /*
sl@0: ** The code in this file is only compiled if:
sl@0: **
sl@0: **     * The FTS3 module is being built as an extension
sl@0: **       (in which case SQLITE_CORE is not defined), or
sl@0: **
sl@0: **     * The FTS3 module is being built into the core of
sl@0: **       SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
sl@0: */
sl@0: #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
sl@0: 
sl@0: 
sl@0: #include <assert.h>
sl@0: #include <stdlib.h>
sl@0: #include <stdio.h>
sl@0: #include <string.h>
sl@0: #include <ctype.h>
sl@0: 
sl@0: #include "fts3_tokenizer.h"
sl@0: 
sl@0: /*
sl@0: ** Class derived from sqlite3_tokenizer
sl@0: */
sl@0: typedef struct porter_tokenizer {
sl@0:   sqlite3_tokenizer base;      /* Base class */
sl@0: } porter_tokenizer;
sl@0: 
sl@0: /*
sl@0: ** Class derived from sqlit3_tokenizer_cursor
sl@0: */
sl@0: typedef struct porter_tokenizer_cursor {
sl@0:   sqlite3_tokenizer_cursor base;
sl@0:   const char *zInput;          /* input we are tokenizing */
sl@0:   int nInput;                  /* size of the input */
sl@0:   int iOffset;                 /* current position in zInput */
sl@0:   int iToken;                  /* index of next token to be returned */
sl@0:   char *zToken;                /* storage for current token */
sl@0:   int nAllocated;              /* space allocated to zToken buffer */
sl@0: } porter_tokenizer_cursor;
sl@0: 
sl@0: 
sl@0: /* Forward declaration */
sl@0: static const sqlite3_tokenizer_module porterTokenizerModule;
sl@0: 
sl@0: 
sl@0: /*
sl@0: ** Create a new tokenizer instance.
sl@0: */
sl@0: static int porterCreate(
sl@0:   int argc, const char * const *argv,
sl@0:   sqlite3_tokenizer **ppTokenizer
sl@0: ){
sl@0:   porter_tokenizer *t;
sl@0:   t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t));
sl@0:   if( t==NULL ) return SQLITE_NOMEM;
sl@0:   memset(t, 0, sizeof(*t));
sl@0:   *ppTokenizer = &t->base;
sl@0:   return SQLITE_OK;
sl@0: }
sl@0: 
sl@0: /*
sl@0: ** Destroy a tokenizer
sl@0: */
sl@0: static int porterDestroy(sqlite3_tokenizer *pTokenizer){
sl@0:   sqlite3_free(pTokenizer);
sl@0:   return SQLITE_OK;
sl@0: }
sl@0: 
sl@0: /*
sl@0: ** Prepare to begin tokenizing a particular string.  The input
sl@0: ** string to be tokenized is zInput[0..nInput-1].  A cursor
sl@0: ** used to incrementally tokenize this string is returned in 
sl@0: ** *ppCursor.
sl@0: */
sl@0: static int porterOpen(
sl@0:   sqlite3_tokenizer *pTokenizer,         /* The tokenizer */
sl@0:   const char *zInput, int nInput,        /* String to be tokenized */
sl@0:   sqlite3_tokenizer_cursor **ppCursor    /* OUT: Tokenization cursor */
sl@0: ){
sl@0:   porter_tokenizer_cursor *c;
sl@0: 
sl@0:   c = (porter_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
sl@0:   if( c==NULL ) return SQLITE_NOMEM;
sl@0: 
sl@0:   c->zInput = zInput;
sl@0:   if( zInput==0 ){
sl@0:     c->nInput = 0;
sl@0:   }else if( nInput<0 ){
sl@0:     c->nInput = (int)strlen(zInput);
sl@0:   }else{
sl@0:     c->nInput = nInput;
sl@0:   }
sl@0:   c->iOffset = 0;                 /* start tokenizing at the beginning */
sl@0:   c->iToken = 0;
sl@0:   c->zToken = NULL;               /* no space allocated, yet. */
sl@0:   c->nAllocated = 0;
sl@0: 
sl@0:   *ppCursor = &c->base;
sl@0:   return SQLITE_OK;
sl@0: }
sl@0: 
sl@0: /*
sl@0: ** Close a tokenization cursor previously opened by a call to
sl@0: ** porterOpen() above.
sl@0: */
sl@0: static int porterClose(sqlite3_tokenizer_cursor *pCursor){
sl@0:   porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
sl@0:   sqlite3_free(c->zToken);
sl@0:   sqlite3_free(c);
sl@0:   return SQLITE_OK;
sl@0: }
sl@0: /*
sl@0: ** Vowel or consonant
sl@0: */
sl@0: static const char cType[] = {
sl@0:    0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0,
sl@0:    1, 1, 1, 2, 1
sl@0: };
sl@0: 
sl@0: /*
sl@0: ** isConsonant() and isVowel() determine if their first character in
sl@0: ** the string they point to is a consonant or a vowel, according
sl@0: ** to Porter ruls.  
sl@0: **
sl@0: ** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'.
sl@0: ** 'Y' is a consonant unless it follows another consonant,
sl@0: ** in which case it is a vowel.
sl@0: **
sl@0: ** In these routine, the letters are in reverse order.  So the 'y' rule
sl@0: ** is that 'y' is a consonant unless it is followed by another
sl@0: ** consonent.
sl@0: */
sl@0: static int isVowel(const char*);
sl@0: static int isConsonant(const char *z){
sl@0:   int j;
sl@0:   char x = *z;
sl@0:   if( x==0 ) return 0;
sl@0:   assert( x>='a' && x<='z' );
sl@0:   j = cType[x-'a'];
sl@0:   if( j<2 ) return j;
sl@0:   return z[1]==0 || isVowel(z + 1);
sl@0: }
sl@0: static int isVowel(const char *z){
sl@0:   int j;
sl@0:   char x = *z;
sl@0:   if( x==0 ) return 0;
sl@0:   assert( x>='a' && x<='z' );
sl@0:   j = cType[x-'a'];
sl@0:   if( j<2 ) return 1-j;
sl@0:   return isConsonant(z + 1);
sl@0: }
sl@0: 
sl@0: /*
sl@0: ** Let any sequence of one or more vowels be represented by V and let
sl@0: ** C be sequence of one or more consonants.  Then every word can be
sl@0: ** represented as:
sl@0: **
sl@0: **           [C] (VC){m} [V]
sl@0: **
sl@0: ** In prose:  A word is an optional consonant followed by zero or
sl@0: ** vowel-consonant pairs followed by an optional vowel.  "m" is the
sl@0: ** number of vowel consonant pairs.  This routine computes the value
sl@0: ** of m for the first i bytes of a word.
sl@0: **
sl@0: ** Return true if the m-value for z is 1 or more.  In other words,
sl@0: ** return true if z contains at least one vowel that is followed
sl@0: ** by a consonant.
sl@0: **
sl@0: ** In this routine z[] is in reverse order.  So we are really looking
sl@0: ** for an instance of of a consonant followed by a vowel.
sl@0: */
sl@0: static int m_gt_0(const char *z){
sl@0:   while( isVowel(z) ){ z++; }
sl@0:   if( *z==0 ) return 0;
sl@0:   while( isConsonant(z) ){ z++; }
sl@0:   return *z!=0;
sl@0: }
sl@0: 
sl@0: /* Like mgt0 above except we are looking for a value of m which is
sl@0: ** exactly 1
sl@0: */
sl@0: static int m_eq_1(const char *z){
sl@0:   while( isVowel(z) ){ z++; }
sl@0:   if( *z==0 ) return 0;
sl@0:   while( isConsonant(z) ){ z++; }
sl@0:   if( *z==0 ) return 0;
sl@0:   while( isVowel(z) ){ z++; }
sl@0:   if( *z==0 ) return 1;
sl@0:   while( isConsonant(z) ){ z++; }
sl@0:   return *z==0;
sl@0: }
sl@0: 
sl@0: /* Like mgt0 above except we are looking for a value of m>1 instead
sl@0: ** or m>0
sl@0: */
sl@0: static int m_gt_1(const char *z){
sl@0:   while( isVowel(z) ){ z++; }
sl@0:   if( *z==0 ) return 0;
sl@0:   while( isConsonant(z) ){ z++; }
sl@0:   if( *z==0 ) return 0;
sl@0:   while( isVowel(z) ){ z++; }
sl@0:   if( *z==0 ) return 0;
sl@0:   while( isConsonant(z) ){ z++; }
sl@0:   return *z!=0;
sl@0: }
sl@0: 
sl@0: /*
sl@0: ** Return TRUE if there is a vowel anywhere within z[0..n-1]
sl@0: */
sl@0: static int hasVowel(const char *z){
sl@0:   while( isConsonant(z) ){ z++; }
sl@0:   return *z!=0;
sl@0: }
sl@0: 
sl@0: /*
sl@0: ** Return TRUE if the word ends in a double consonant.
sl@0: **
sl@0: ** The text is reversed here. So we are really looking at
sl@0: ** the first two characters of z[].
sl@0: */
sl@0: static int doubleConsonant(const char *z){
sl@0:   return isConsonant(z) && z[0]==z[1] && isConsonant(z+1);
sl@0: }
sl@0: 
sl@0: /*
sl@0: ** Return TRUE if the word ends with three letters which
sl@0: ** are consonant-vowel-consonent and where the final consonant
sl@0: ** is not 'w', 'x', or 'y'.
sl@0: **
sl@0: ** The word is reversed here.  So we are really checking the
sl@0: ** first three letters and the first one cannot be in [wxy].
sl@0: */
sl@0: static int star_oh(const char *z){
sl@0:   return
sl@0:     z[0]!=0 && isConsonant(z) &&
sl@0:     z[0]!='w' && z[0]!='x' && z[0]!='y' &&
sl@0:     z[1]!=0 && isVowel(z+1) &&
sl@0:     z[2]!=0 && isConsonant(z+2);
sl@0: }
sl@0: 
sl@0: /*
sl@0: ** If the word ends with zFrom and xCond() is true for the stem
sl@0: ** of the word that preceeds the zFrom ending, then change the 
sl@0: ** ending to zTo.
sl@0: **
sl@0: ** The input word *pz and zFrom are both in reverse order.  zTo
sl@0: ** is in normal order. 
sl@0: **
sl@0: ** Return TRUE if zFrom matches.  Return FALSE if zFrom does not
sl@0: ** match.  Not that TRUE is returned even if xCond() fails and
sl@0: ** no substitution occurs.
sl@0: */
sl@0: static int stem(
sl@0:   char **pz,             /* The word being stemmed (Reversed) */
sl@0:   const char *zFrom,     /* If the ending matches this... (Reversed) */
sl@0:   const char *zTo,       /* ... change the ending to this (not reversed) */
sl@0:   int (*xCond)(const char*)   /* Condition that must be true */
sl@0: ){
sl@0:   char *z = *pz;
sl@0:   while( *zFrom && *zFrom==*z ){ z++; zFrom++; }
sl@0:   if( *zFrom!=0 ) return 0;
sl@0:   if( xCond && !xCond(z) ) return 1;
sl@0:   while( *zTo ){
sl@0:     *(--z) = *(zTo++);
sl@0:   }
sl@0:   *pz = z;
sl@0:   return 1;
sl@0: }
sl@0: 
sl@0: /*
sl@0: ** This is the fallback stemmer used when the porter stemmer is
sl@0: ** inappropriate.  The input word is copied into the output with
sl@0: ** US-ASCII case folding.  If the input word is too long (more
sl@0: ** than 20 bytes if it contains no digits or more than 6 bytes if
sl@0: ** it contains digits) then word is truncated to 20 or 6 bytes
sl@0: ** by taking 10 or 3 bytes from the beginning and end.
sl@0: */
sl@0: static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
sl@0:   int i, mx, j;
sl@0:   int hasDigit = 0;
sl@0:   for(i=0; i<nIn; i++){
sl@0:     int c = zIn[i];
sl@0:     if( c>='A' && c<='Z' ){
sl@0:       zOut[i] = c - 'A' + 'a';
sl@0:     }else{
sl@0:       if( c>='0' && c<='9' ) hasDigit = 1;
sl@0:       zOut[i] = c;
sl@0:     }
sl@0:   }
sl@0:   mx = hasDigit ? 3 : 10;
sl@0:   if( nIn>mx*2 ){
sl@0:     for(j=mx, i=nIn-mx; i<nIn; i++, j++){
sl@0:       zOut[j] = zOut[i];
sl@0:     }
sl@0:     i = j;
sl@0:   }
sl@0:   zOut[i] = 0;
sl@0:   *pnOut = i;
sl@0: }
sl@0: 
sl@0: 
sl@0: /*
sl@0: ** Stem the input word zIn[0..nIn-1].  Store the output in zOut.
sl@0: ** zOut is at least big enough to hold nIn bytes.  Write the actual
sl@0: ** size of the output word (exclusive of the '\0' terminator) into *pnOut.
sl@0: **
sl@0: ** Any upper-case characters in the US-ASCII character set ([A-Z])
sl@0: ** are converted to lower case.  Upper-case UTF characters are
sl@0: ** unchanged.
sl@0: **
sl@0: ** Words that are longer than about 20 bytes are stemmed by retaining
sl@0: ** a few bytes from the beginning and the end of the word.  If the
sl@0: ** word contains digits, 3 bytes are taken from the beginning and
sl@0: ** 3 bytes from the end.  For long words without digits, 10 bytes
sl@0: ** are taken from each end.  US-ASCII case folding still applies.
sl@0: ** 
sl@0: ** If the input word contains not digits but does characters not 
sl@0: ** in [a-zA-Z] then no stemming is attempted and this routine just 
sl@0: ** copies the input into the input into the output with US-ASCII
sl@0: ** case folding.
sl@0: **
sl@0: ** Stemming never increases the length of the word.  So there is
sl@0: ** no chance of overflowing the zOut buffer.
sl@0: */
sl@0: static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
sl@0:   int i, j, c;
sl@0:   char zReverse[28];
sl@0:   char *z, *z2;
sl@0:   if( nIn<3 || nIn>=sizeof(zReverse)-7 ){
sl@0:     /* The word is too big or too small for the porter stemmer.
sl@0:     ** Fallback to the copy stemmer */
sl@0:     copy_stemmer(zIn, nIn, zOut, pnOut);
sl@0:     return;
sl@0:   }
sl@0:   for(i=0, j=sizeof(zReverse)-6; i<nIn; i++, j--){
sl@0:     c = zIn[i];
sl@0:     if( c>='A' && c<='Z' ){
sl@0:       zReverse[j] = c + 'a' - 'A';
sl@0:     }else if( c>='a' && c<='z' ){
sl@0:       zReverse[j] = c;
sl@0:     }else{
sl@0:       /* The use of a character not in [a-zA-Z] means that we fallback
sl@0:       ** to the copy stemmer */
sl@0:       copy_stemmer(zIn, nIn, zOut, pnOut);
sl@0:       return;
sl@0:     }
sl@0:   }
sl@0:   memset(&zReverse[sizeof(zReverse)-5], 0, 5);
sl@0:   z = &zReverse[j+1];
sl@0: 
sl@0: 
sl@0:   /* Step 1a */
sl@0:   if( z[0]=='s' ){
sl@0:     if(
sl@0:      !stem(&z, "sess", "ss", 0) &&
sl@0:      !stem(&z, "sei", "i", 0)  &&
sl@0:      !stem(&z, "ss", "ss", 0)
sl@0:     ){
sl@0:       z++;
sl@0:     }
sl@0:   }
sl@0: 
sl@0:   /* Step 1b */  
sl@0:   z2 = z;
sl@0:   if( stem(&z, "dee", "ee", m_gt_0) ){
sl@0:     /* Do nothing.  The work was all in the test */
sl@0:   }else if( 
sl@0:      (stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel))
sl@0:       && z!=z2
sl@0:   ){
sl@0:      if( stem(&z, "ta", "ate", 0) ||
sl@0:          stem(&z, "lb", "ble", 0) ||
sl@0:          stem(&z, "zi", "ize", 0) ){
sl@0:        /* Do nothing.  The work was all in the test */
sl@0:      }else if( doubleConsonant(z) && (*z!='l' && *z!='s' && *z!='z') ){
sl@0:        z++;
sl@0:      }else if( m_eq_1(z) && star_oh(z) ){
sl@0:        *(--z) = 'e';
sl@0:      }
sl@0:   }
sl@0: 
sl@0:   /* Step 1c */
sl@0:   if( z[0]=='y' && hasVowel(z+1) ){
sl@0:     z[0] = 'i';
sl@0:   }
sl@0: 
sl@0:   /* Step 2 */
sl@0:   switch( z[1] ){
sl@0:    case 'a':
sl@0:      stem(&z, "lanoita", "ate", m_gt_0) ||
sl@0:      stem(&z, "lanoit", "tion", m_gt_0);
sl@0:      break;
sl@0:    case 'c':
sl@0:      stem(&z, "icne", "ence", m_gt_0) ||
sl@0:      stem(&z, "icna", "ance", m_gt_0);
sl@0:      break;
sl@0:    case 'e':
sl@0:      stem(&z, "rezi", "ize", m_gt_0);
sl@0:      break;
sl@0:    case 'g':
sl@0:      stem(&z, "igol", "log", m_gt_0);
sl@0:      break;
sl@0:    case 'l':
sl@0:      stem(&z, "ilb", "ble", m_gt_0) ||
sl@0:      stem(&z, "illa", "al", m_gt_0) ||
sl@0:      stem(&z, "iltne", "ent", m_gt_0) ||
sl@0:      stem(&z, "ile", "e", m_gt_0) ||
sl@0:      stem(&z, "ilsuo", "ous", m_gt_0);
sl@0:      break;
sl@0:    case 'o':
sl@0:      stem(&z, "noitazi", "ize", m_gt_0) ||
sl@0:      stem(&z, "noita", "ate", m_gt_0) ||
sl@0:      stem(&z, "rota", "ate", m_gt_0);
sl@0:      break;
sl@0:    case 's':
sl@0:      stem(&z, "msila", "al", m_gt_0) ||
sl@0:      stem(&z, "ssenevi", "ive", m_gt_0) ||
sl@0:      stem(&z, "ssenluf", "ful", m_gt_0) ||
sl@0:      stem(&z, "ssensuo", "ous", m_gt_0);
sl@0:      break;
sl@0:    case 't':
sl@0:      stem(&z, "itila", "al", m_gt_0) ||
sl@0:      stem(&z, "itivi", "ive", m_gt_0) ||
sl@0:      stem(&z, "itilib", "ble", m_gt_0);
sl@0:      break;
sl@0:   }
sl@0: 
sl@0:   /* Step 3 */
sl@0:   switch( z[0] ){
sl@0:    case 'e':
sl@0:      stem(&z, "etaci", "ic", m_gt_0) ||
sl@0:      stem(&z, "evita", "", m_gt_0)   ||
sl@0:      stem(&z, "ezila", "al", m_gt_0);
sl@0:      break;
sl@0:    case 'i':
sl@0:      stem(&z, "itici", "ic", m_gt_0);
sl@0:      break;
sl@0:    case 'l':
sl@0:      stem(&z, "laci", "ic", m_gt_0) ||
sl@0:      stem(&z, "luf", "", m_gt_0);
sl@0:      break;
sl@0:    case 's':
sl@0:      stem(&z, "ssen", "", m_gt_0);
sl@0:      break;
sl@0:   }
sl@0: 
sl@0:   /* Step 4 */
sl@0:   switch( z[1] ){
sl@0:    case 'a':
sl@0:      if( z[0]=='l' && m_gt_1(z+2) ){
sl@0:        z += 2;
sl@0:      }
sl@0:      break;
sl@0:    case 'c':
sl@0:      if( z[0]=='e' && z[2]=='n' && (z[3]=='a' || z[3]=='e')  && m_gt_1(z+4)  ){
sl@0:        z += 4;
sl@0:      }
sl@0:      break;
sl@0:    case 'e':
sl@0:      if( z[0]=='r' && m_gt_1(z+2) ){
sl@0:        z += 2;
sl@0:      }
sl@0:      break;
sl@0:    case 'i':
sl@0:      if( z[0]=='c' && m_gt_1(z+2) ){
sl@0:        z += 2;
sl@0:      }
sl@0:      break;
sl@0:    case 'l':
sl@0:      if( z[0]=='e' && z[2]=='b' && (z[3]=='a' || z[3]=='i') && m_gt_1(z+4) ){
sl@0:        z += 4;
sl@0:      }
sl@0:      break;
sl@0:    case 'n':
sl@0:      if( z[0]=='t' ){
sl@0:        if( z[2]=='a' ){
sl@0:          if( m_gt_1(z+3) ){
sl@0:            z += 3;
sl@0:          }
sl@0:        }else if( z[2]=='e' ){
sl@0:          stem(&z, "tneme", "", m_gt_1) ||
sl@0:          stem(&z, "tnem", "", m_gt_1) ||
sl@0:          stem(&z, "tne", "", m_gt_1);
sl@0:        }
sl@0:      }
sl@0:      break;
sl@0:    case 'o':
sl@0:      if( z[0]=='u' ){
sl@0:        if( m_gt_1(z+2) ){
sl@0:          z += 2;
sl@0:        }
sl@0:      }else if( z[3]=='s' || z[3]=='t' ){
sl@0:        stem(&z, "noi", "", m_gt_1);
sl@0:      }
sl@0:      break;
sl@0:    case 's':
sl@0:      if( z[0]=='m' && z[2]=='i' && m_gt_1(z+3) ){
sl@0:        z += 3;
sl@0:      }
sl@0:      break;
sl@0:    case 't':
sl@0:      stem(&z, "eta", "", m_gt_1) ||
sl@0:      stem(&z, "iti", "", m_gt_1);
sl@0:      break;
sl@0:    case 'u':
sl@0:      if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){
sl@0:        z += 3;
sl@0:      }
sl@0:      break;
sl@0:    case 'v':
sl@0:    case 'z':
sl@0:      if( z[0]=='e' && z[2]=='i' && m_gt_1(z+3) ){
sl@0:        z += 3;
sl@0:      }
sl@0:      break;
sl@0:   }
sl@0: 
sl@0:   /* Step 5a */
sl@0:   if( z[0]=='e' ){
sl@0:     if( m_gt_1(z+1) ){
sl@0:       z++;
sl@0:     }else if( m_eq_1(z+1) && !star_oh(z+1) ){
sl@0:       z++;
sl@0:     }
sl@0:   }
sl@0: 
sl@0:   /* Step 5b */
sl@0:   if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){
sl@0:     z++;
sl@0:   }
sl@0: 
sl@0:   /* z[] is now the stemmed word in reverse order.  Flip it back
sl@0:   ** around into forward order and return.
sl@0:   */
sl@0:   *pnOut = i = strlen(z);
sl@0:   zOut[i] = 0;
sl@0:   while( *z ){
sl@0:     zOut[--i] = *(z++);
sl@0:   }
sl@0: }
sl@0: 
sl@0: /*
sl@0: ** Characters that can be part of a token.  We assume any character
sl@0: ** whose value is greater than 0x80 (any UTF character) can be
sl@0: ** part of a token.  In other words, delimiters all must have
sl@0: ** values of 0x7f or lower.
sl@0: */
sl@0: static const char porterIdChar[] = {
sl@0: /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
sl@0:     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,  /* 3x */
sl@0:     0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 4x */
sl@0:     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,  /* 5x */
sl@0:     0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 6x */
sl@0:     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,  /* 7x */
sl@0: };
sl@0: #define isDelim(C) (((ch=C)&0x80)==0 && (ch<0x30 || !porterIdChar[ch-0x30]))
sl@0: 
sl@0: /*
sl@0: ** Extract the next token from a tokenization cursor.  The cursor must
sl@0: ** have been opened by a prior call to porterOpen().
sl@0: */
sl@0: static int porterNext(
sl@0:   sqlite3_tokenizer_cursor *pCursor,  /* Cursor returned by porterOpen */
sl@0:   const char **pzToken,               /* OUT: *pzToken is the token text */
sl@0:   int *pnBytes,                       /* OUT: Number of bytes in token */
sl@0:   int *piStartOffset,                 /* OUT: Starting offset of token */
sl@0:   int *piEndOffset,                   /* OUT: Ending offset of token */
sl@0:   int *piPosition                     /* OUT: Position integer of token */
sl@0: ){
sl@0:   porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
sl@0:   const char *z = c->zInput;
sl@0: 
sl@0:   while( c->iOffset<c->nInput ){
sl@0:     int iStartOffset, ch;
sl@0: 
sl@0:     /* Scan past delimiter characters */
sl@0:     while( c->iOffset<c->nInput && isDelim(z[c->iOffset]) ){
sl@0:       c->iOffset++;
sl@0:     }
sl@0: 
sl@0:     /* Count non-delimiter characters. */
sl@0:     iStartOffset = c->iOffset;
sl@0:     while( c->iOffset<c->nInput && !isDelim(z[c->iOffset]) ){
sl@0:       c->iOffset++;
sl@0:     }
sl@0: 
sl@0:     if( c->iOffset>iStartOffset ){
sl@0:       int n = c->iOffset-iStartOffset;
sl@0:       if( n>c->nAllocated ){
sl@0:         c->nAllocated = n+20;
sl@0:         c->zToken = sqlite3_realloc(c->zToken, c->nAllocated);
sl@0:         if( c->zToken==NULL ) return SQLITE_NOMEM;
sl@0:       }
sl@0:       porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes);
sl@0:       *pzToken = c->zToken;
sl@0:       *piStartOffset = iStartOffset;
sl@0:       *piEndOffset = c->iOffset;
sl@0:       *piPosition = c->iToken++;
sl@0:       return SQLITE_OK;
sl@0:     }
sl@0:   }
sl@0:   return SQLITE_DONE;
sl@0: }
sl@0: 
sl@0: /*
sl@0: ** The set of routines that implement the porter-stemmer tokenizer
sl@0: */
sl@0: static const sqlite3_tokenizer_module porterTokenizerModule = {
sl@0:   0,
sl@0:   porterCreate,
sl@0:   porterDestroy,
sl@0:   porterOpen,
sl@0:   porterClose,
sl@0:   porterNext,
sl@0: };
sl@0: 
sl@0: /*
sl@0: ** Allocate a new porter tokenizer.  Return a pointer to the new
sl@0: ** tokenizer in *ppModule
sl@0: */
sl@0: void sqlite3Fts3PorterTokenizerModule(
sl@0:   sqlite3_tokenizer_module const**ppModule
sl@0: ){
sl@0:   *ppModule = &porterTokenizerModule;
sl@0: }
sl@0: 
sl@0: #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */