1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/os/persistentdata/persistentstorage/dbms/usql/UQ_LEXER.CPP Fri Jun 15 03:10:57 2012 +0200
1.3 @@ -0,0 +1,360 @@
1.4 +// Copyright (c) 1998-2009 Nokia Corporation and/or its subsidiary(-ies).
1.5 +// All rights reserved.
1.6 +// This component and the accompanying materials are made available
1.7 +// under the terms of "Eclipse Public License v1.0"
1.8 +// which accompanies this distribution, and is available
1.9 +// at the URL "http://www.eclipse.org/legal/epl-v10.html".
1.10 +//
1.11 +// Initial Contributors:
1.12 +// Nokia Corporation - initial contribution.
1.13 +//
1.14 +// Contributors:
1.15 +//
1.16 +// Description:
1.17 +// SQL parser tokeniser
1.18 +//
1.19 +//
1.20 +
1.21 +#include "UQ_STD.H"
1.22 +
1.23 +// optimised tables for ASCII character set
1.24 +
1.25 +const TUint8 KAlpha=0x1;
1.26 +const TUint8 KDigitOr_=0x2;
1.27 +
1.28 +const TUint8 KCharAttrib[]=
1.29 + {
1.30 + KDigitOr_,KDigitOr_,KDigitOr_,KDigitOr_,KDigitOr_,KDigitOr_,KDigitOr_,KDigitOr_,
1.31 + KDigitOr_,KDigitOr_,0,0,0,0,0,0,
1.32 + 0,KAlpha,KAlpha,KAlpha,KAlpha,KAlpha,KAlpha,KAlpha,
1.33 + KAlpha,KAlpha,KAlpha,KAlpha,KAlpha,KAlpha,KAlpha,KAlpha,
1.34 + KAlpha,KAlpha,KAlpha,KAlpha,KAlpha,KAlpha,KAlpha,KAlpha,
1.35 + KAlpha,KAlpha,KAlpha,0,0,0,0,KDigitOr_,
1.36 + 0,KAlpha,KAlpha,KAlpha,KAlpha,KAlpha,KAlpha,KAlpha,
1.37 + KAlpha,KAlpha,KAlpha,KAlpha,KAlpha,KAlpha,KAlpha,KAlpha,
1.38 + KAlpha,KAlpha,KAlpha,KAlpha,KAlpha,KAlpha,KAlpha,KAlpha,
1.39 + KAlpha,KAlpha,KAlpha
1.40 + };
1.41 +
1.42 +#define ISALPHA(aChar) (TUint(aChar-'0')<=TUint('z'-'0') && KCharAttrib[aChar-'0']&KAlpha)
1.43 +#define ISALPHA_DIGIT_OR_(aChar) (TUint(aChar-'0')<=TUint('z'-'0') && KCharAttrib[aChar-'0']&(KAlpha|KDigitOr_))
1.44 +#define LCASE(aChar) (aChar|0x20)
1.45 +
1.46 +// The keywords
1.47 +// These are always stored as ASCII because DBMS has its own
1.48 +
1.49 +const TInt KMaxKeywordLength=9;
1.50 +
1.51 +static const TText8 KSqlKeywords[][KMaxKeywordLength+1]=
1.52 + {
1.53 +#define KEYWORD(s) #s
1.54 +#include "UQ_KEYWD.H"
1.55 +#undef KEYWORD
1.56 + };
1.57 +const TInt KSqlKeywordCount=sizeof(KSqlKeywords)/sizeof(*KSqlKeywords);
1.58 +
1.59 +#if defined(_ASSERTIONS)
1.60 +TInt CheckKeywords()
1.61 +//
1.62 +// ensure that the keyword table is in alphabetical order
1.63 +//
1.64 + {
1.65 + for (TInt ii=1;ii<KSqlKeywordCount;++ii)
1.66 + __ASSERT(TPtrC8(KSqlKeywords[ii-1])<TPtrC8(KSqlKeywords[ii]));
1.67 + return 1;
1.68 + }
1.69 +#endif
1.70 +
1.71 +// class TSqlLexer
1.72 +
1.73 +TInt TSqlLexer::CompareKeyword(TInt aKeyword,const RSqlLiteral& aIdentifier)
1.74 +//
1.75 +// Check if the identifer in aIdentifier is a keyword
1.76 +// uses a case-insensitive match, not folding
1.77 +//
1.78 + {
1.79 + __ASSERT(TUint(aKeyword)<TUint(KSqlKeywordCount));
1.80 +//
1.81 + const TText* ptr=aIdentifier.Ptr();
1.82 + const TText* end=aIdentifier.End();
1.83 + const TText8* pk=&KSqlKeywords[aKeyword][0];
1.84 + for (;;)
1.85 + {
1.86 + TUint ck=*pk++;
1.87 + if (ptr==end)
1.88 + return ck;
1.89 + if (!ck)
1.90 + return -1;
1.91 + TInt d=ck-LCASE(TUint(*ptr++));
1.92 + if (d)
1.93 + return d;
1.94 + }
1.95 + }
1.96 +
1.97 +TSqlKeyword TSqlLexer::Keyword(const TSqlToken& aToken)
1.98 +//
1.99 +// non-member function: Return the keyword value
1.100 +//
1.101 + {
1.102 + if (aToken==ESqlIdentifier)
1.103 + {
1.104 + TInt r=KSqlKeywordCount;
1.105 + TInt l=0;
1.106 + while (r>l)
1.107 + {
1.108 + TInt m=(l+r)>>1;
1.109 + TInt k=CompareKeyword(m,aToken.Literal());
1.110 + if (k>0)
1.111 + r=m;
1.112 + else if (k<0)
1.113 + l=m+1;
1.114 + else
1.115 + return TSqlKeyword(m); // keyword
1.116 + }
1.117 + }
1.118 + // identifier
1.119 + return ESqlNotKeyword;
1.120 + }
1.121 +
1.122 +TSqlLexer::TSqlLexer(const TDesC& aSql)
1.123 + : iNext(aSql.Ptr()),iEnd(iNext+aSql.Length())
1.124 + {
1.125 + __ASSERT(CheckKeywords());
1.126 + }
1.127 +
1.128 +TSqlTokenType TSqlLexer::GetIdentifier(TSqlToken& aToken)
1.129 +//
1.130 +// Get a keyword or identifier. Do not resolve a keyword at this stage
1.131 +//
1.132 + {
1.133 + const TText* end=iEnd;
1.134 + const TText* next=iNext-1;
1.135 + while (++next<end)
1.136 + {
1.137 + TUint ch=*next;
1.138 + if (ISALPHA_DIGIT_OR_(ch))
1.139 + continue;
1.140 + if (!TChar(ch).IsAlphaDigit())
1.141 + break;
1.142 + }
1.143 + aToken.iLiteral.SetText(iNext-1,next);
1.144 + iNext=next;
1.145 + return ESqlIdentifier;
1.146 + }
1.147 +
1.148 +TInt TSqlLexer::GetInteger(TInt64 &aVal)
1.149 +//
1.150 +// A rather more optimised version of TLex::Val(TInt64&)
1.151 +// initially accumulate the value in a TUint32, and only switch to 64-bit arithmetic
1.152 +// if the value overflows. Always return a 64-bit value
1.153 +//
1.154 + {
1.155 + const TUint KPreMultiplyLimit32=429496728; //(KMaxTUint-9)/10
1.156 + const TUint KPreMultiplyLimit64=214748364; //(KMaxTInt+1)/10
1.157 +//
1.158 + const TText* ptr=iNext;
1.159 + const TText* const end=iEnd;
1.160 + __ASSERT(ptr<end);
1.161 + TUint sign=0;
1.162 + TUint c=*ptr;
1.163 + if (c=='-')
1.164 + {
1.165 + sign=1;
1.166 + if (++ptr==end)
1.167 + return KErrGeneral;
1.168 + c=*ptr;
1.169 + }
1.170 + else if (c=='+')
1.171 + {
1.172 + if (++ptr==end)
1.173 + return KErrGeneral;
1.174 + c=*ptr;
1.175 + }
1.176 + c-='0';
1.177 + if (c>=10u)
1.178 + return KErrGeneral; // no digits at all
1.179 + TUint val32=c;
1.180 + while (++ptr<end)
1.181 + {
1.182 + c=*ptr-'0';
1.183 + if (c>=10u)
1.184 + break;
1.185 + if (val32>KPreMultiplyLimit32)
1.186 + goto overflow64; // will not fit into 32 bit arithmetic
1.187 + val32*=10;
1.188 + val32+=c;
1.189 + }
1.190 +// we have result, just set the sign and finish
1.191 + aVal=val32;
1.192 + goto checksign;
1.193 +//
1.194 +// continue the accumulation with a 64-bit integer
1.195 +overflow64:
1.196 + aVal=val32;
1.197 + for (;;)
1.198 + {
1.199 + I64MUL10(aVal);
1.200 + aVal+=c;
1.201 + if (++ptr==end)
1.202 + break;
1.203 + c=*ptr-'0';
1.204 + if (c>=10u)
1.205 + break;
1.206 + if (I64HIGH(aVal)>KPreMultiplyLimit64)
1.207 + return KErrOverflow; // the value is certain to overflow
1.208 + }
1.209 + if (I64HIGH(aVal)&0x80000000u)
1.210 + { // greater than the "half way mark"
1.211 + if (!sign)
1.212 + return KErrOverflow;
1.213 + if (I64LOW(aVal)!=0)
1.214 + return KErrOverflow;
1.215 + }
1.216 +checksign:
1.217 + iNext=ptr;
1.218 + if (sign)
1.219 + aVal=-aVal;
1.220 + return KErrNone;
1.221 + }
1.222 +
1.223 +TSqlTokenType TSqlLexer::GetNumber(TSqlToken& aToken)
1.224 + {
1.225 + const TText* mark=--iNext; // rewind past initial character
1.226 + // attempt to parse a integer
1.227 + TInt r=GetInteger(aToken.iLiteral.SetInt());
1.228 + if (r==KErrNone)
1.229 + {
1.230 + if (iNext<iEnd)
1.231 + {
1.232 + TUint c=*iNext;
1.233 + if (c!='.' && c!='e' && c!='E')
1.234 + return ESqlLiteralInt; // it is an integer
1.235 + }
1.236 + else
1.237 + return ESqlLiteralInt; // it is an integer
1.238 + }
1.239 + TLex lex(TPtrC(mark,iEnd-mark));
1.240 + r=lex.Val(aToken.iLiteral.SetReal(),TChar('.'));
1.241 + if (r!=KErrNone)
1.242 + return SqlError(r);
1.243 + iNext=mark+lex.Offset();
1.244 + return ESqlLiteralReal;
1.245 + }
1.246 +
1.247 +TSqlTokenType TSqlLexer::GetString(TSqlToken& aToken)
1.248 + {
1.249 + const TText* next=iNext;
1.250 + const TText* end=iEnd;
1.251 + for (;;)
1.252 + {
1.253 + if (next==end)
1.254 + return SqlError();
1.255 + TUint c=*next++;
1.256 + if (c=='\'')
1.257 + {
1.258 + if (next==end)
1.259 + break;
1.260 + if (*next!='\'')
1.261 + break;
1.262 + next++;
1.263 + }
1.264 + }
1.265 + aToken.iLiteral.SetText(iNext,next-1);
1.266 + iNext=next;
1.267 + return ESqlLiteralText;
1.268 + }
1.269 +
1.270 +TSqlTokenType TSqlLexer::GetDate(TSqlToken& aToken)
1.271 + {
1.272 + const TText* end=iEnd;
1.273 + const TText* next=iNext;
1.274 + do
1.275 + {
1.276 + if (next==end)
1.277 + return SqlError();
1.278 + } while (*next++!='#');
1.279 + TInt r=aToken.iLiteral.SetTime().Parse(TPtrC(iNext,(next-1)-iNext));
1.280 + if (r<0)
1.281 + return SqlError(r);
1.282 + iNext=next;
1.283 + return ESqlLiteralTime;
1.284 + }
1.285 +
1.286 +TSqlTokenType TSqlLexer::GetNextToken(TSqlToken& aToken)
1.287 + {
1.288 + const TText* ptr=iNext;
1.289 + const TText* const end=iEnd;
1.290 + for (;;)
1.291 + {
1.292 + if (ptr==end)
1.293 + return ESqlEos;
1.294 + TUint ch=*ptr++;
1.295 + iNext=ptr;
1.296 + switch (ch)
1.297 + {
1.298 + case ' ': // a "normal" space
1.299 + continue;
1.300 + case '0': case '1': case '2': case '3': case '4': // literal number
1.301 + case '5': case '6': case '7': case '8': case '9':
1.302 + case '+': case '-': case '.':
1.303 + return GetNumber(aToken);
1.304 + case '\'':
1.305 + return GetString(aToken);
1.306 + case '#':
1.307 + return GetDate(aToken);
1.308 + case '*':
1.309 + return ESqlAsterisk;
1.310 + case ',':
1.311 + return ESqlComma;
1.312 + case '(':
1.313 + return ESqlLeftBracket;
1.314 + case ')':
1.315 + return ESqlRightBracket;
1.316 + case '=':
1.317 + return ESqlEqual;
1.318 + case '<':
1.319 + {
1.320 + ch=*ptr++;
1.321 + if (ch=='=')
1.322 + {
1.323 + iNext=ptr;
1.324 + return ESqlLessEqual;
1.325 + }
1.326 + if (ch=='>')
1.327 + {
1.328 + iNext=ptr;
1.329 + return ESqlNotEqual;
1.330 + }
1.331 + return ESqlLess;
1.332 + }
1.333 + case '>':
1.334 + {
1.335 + ch=*ptr++;
1.336 + if (ch=='=')
1.337 + {
1.338 + iNext=ptr;
1.339 + return ESqlGreaterEqual;
1.340 + }
1.341 + return ESqlGreater;
1.342 + }
1.343 + default:
1.344 + break;
1.345 + }
1.346 + if (ISALPHA(ch))
1.347 + return GetIdentifier(aToken); // keyword or identifier
1.348 + TChar cc(ch);
1.349 + if (cc.IsAlpha())
1.350 + return GetIdentifier(aToken); // keyword or identifier
1.351 + if (!cc.IsSpace())
1.352 + return SqlError();
1.353 + }
1.354 + }
1.355 +
1.356 +const TText* TSqlLexer::Next() const
1.357 + {
1.358 + return iNext;
1.359 + }
1.360 +void TSqlLexer::Set(const TText* aNext)
1.361 + {
1.362 + iNext = aNext ;
1.363 + }