// Copyright (c) 1999-2009 Nokia Corporation and/or its subsidiary(-ies).

 // All rights reserved.

 // This component and the accompanying materials are made available

 // under the terms of "Eclipse Public License v1.0"

 // which accompanies this distribution, and is available

 // at the URL "http://www.eclipse.org/legal/epl-v10.html".

 //

 // Initial Contributors:

 // Nokia Corporation - initial contribution.

 //

 // Contributors:

 //

 // Description:

 // Reads and parses the Unicode collation value table and writes out a C++ source file

 // containing the data in a form that can be used by the EPOC collation system.

 //

 // The program reads three files or one compositive files:

 //

 // Three files (by default):

 // 1. Base keys (maps single Unicode values to single collation key values): must be in the same format as

 // basekeys.txt, supplied with the Standard Unicode Collation system

 //

 // 2. Composite keys (maps single Unicode values to strings of collation keys): must be in the same format as

 // compkeys.txt, supplied with the Standard Unicode Collation system

 //

 // 3. Strings (maps strings of Unicode values to single collation keys OR strings of collation keys): must be in the

 // same format as compkeys.txt, except that there can be any number of Unicode characters at the start of the line,

 // space-separated and each exactly 4 hex digits.

 //

 // One compositive files (with option /a):

 // 1. All Keys (combine above three files into one file): must be in the same format as allkeys.txt, supplied with the Standard Unicode Collation system (after Unicode 3.0).

 //

 //

 #include <assert.h>

 #include <ctype.h>

 #ifdef __MSVCDOTNET__

 #include <fstream>

 #include <iostream>

 using namespace std;

 #else //!__MSVCDOTNET__

 #include <fstream.h>

 #include <iostream.h>

 #endif //__MSVCDOTNET__

 #include <stdlib.h>

 #include <string.h>

 #include <stdio.h>

 /*

 Constants constraining the range of level-1 and level-2 keys so that they can be packed.

 Non-zero values are reduced by one less than the minimum value.

 */

 const unsigned int KLevel1Bits = 8;

 const unsigned int KLevel1Min = 0x20;

 const unsigned int KLevel1Max = KLevel1Min + (1 << KLevel1Bits) - 2;

 const unsigned int KLevel2Bits = 6;

 const unsigned int KLevel2Min = 1;

 const unsigned int KLevel2Max = KLevel2Min + (1 << KLevel2Bits) - 2;

 /*

 Table of characters in the WGL4 set, plus characters in canonical decompositions of

 those characters, plus commonly used control characters and space characters,

 given as ranges of Unicode characters. In each pair, the first code is the first in the range,

 and the second is the first code NOT in the range.

 The extra characters are added mainly to ensure that control characters and spaces are

 normally ignored. The extra characters are:

 0x0000-0x001F: ASCII control characters

 0x2000-0x2012: spaces, hyphen variants, figure dash

 0x2028-0x202E: line and paragraph separator, bidirectional control characters

 0xFEFF		 : byte-order mark

 0xFFFC-0xFFFD: object replacement character, replacement character

 */

 const unsigned int Wgl4Range[] =

 	{

 	0x00, 0x7f,		// All ASCII

 	0xa0, 0x180,		// Non-breaking space, Latin-1, Latin Extended-A

 	0x192,0x193,		// Latin f with hook

 	0x1fa,0x200,		// A-ring, a-ring, AE, ae, O slash, o slash all with acute accent

 	0x2c6,0x2c8,		// non-combining circumflex and caron

 	0x2c9,0x2ca,		// non-combining macron

 	0x2d8,0x2dc,		// non-combining breve, dot above, ring above, ogonek

 	0x2dd,0x2de,		// non-combining double acute

 	0x300,0x305,		// combining grave, acute, circumflex, tilde, macron

 	0x306,0x309,		// combining breve, dot above, double dot above

 	0x30a,0x30e,		// combining ring above, double acute, caron, vertical line above

 	0x327,0x329,		// combining cedilla, ogonek

 	0x384,0x38b,		// Greek

 	0x38c,0x38d,		// Greek

 	0x38e,0x3a2,		// Greek

 	0x3a3,0x3cf,		// Greek

 	0x401,0x40d,		// Cyrillic

 	0x40e,0x450,		// Cyrillic

 	0x451,0x45d,		// Cyrillic

 	0x45e,0x460,		// Cyrillic

 	0x490,0x492,		// Cyrillic

 	0x1e80,0x1e86,		// Both W and w with each of grave, acute and diaeresis

 	0x1ef2,0x1ef4,		// Y with grave, y with grave

 	0x2000,0x2016,		// various space and horizontal lines

 	0x2017,0x201f,		//double vertical line, double low line, various quotation marks

 	0x2020,0x2023,		// dagger, double dagger, bullet

 	0x2026,0x2027,		//ellipsis

 	0x2028,0x202F,		// line & paragraph separators and directional formatting

 	0x2030,0x2031,		// per mille

 	0x2032,0x2034,		// prime

 	0x2039,0x203b,		// single angle quotation marks

 	0x203c,0x203d,		// double exclamation mark

 	0x203e,0x203f,		// non-combining overscore

 	0x2044,0x2045,		// fraction slash

 	0x207f,0x2080,		// superscript n

 	0x20a3,0x20a5,		// French Franc, Italian/Turkish Lira

 	0x20a7,0x20a8,		// Spanish Peseta

 	0x20ac,0x20ad,		// Euro symbol

 	0x2105,0x2106,		// care of

 	0x2113,0x2114,		// script l

 	0x2116,0x2117,		// numero

 	0x2122,0x2123,		// trade mark

 	0x2126,0x2127,		// ohm

 	0x212e,0x212f,		// estimated (net weight)

 	0x215b,0x215f,		// 1/8, 3/8, 5/8, 7/8

 	0x2190,0x2196,		// horizontal and vertical arrows

 	0x21a8,0x21a9,		// up down arrow with base

 	0x2202,0x2203,		// partial differential

 	0x2206,0x2207,		// increment (delta)

 	0x220f,0x2210,		// n-ary product (pi)

 	0x2211,0x2213,		// n-ary sum (sigma), minus

 	0x2215,0x2216,		// division (slash)

 	0x2219,0x221b,		// bullet operator, square root

 	0x221e,0x2220,		// infinity, right angle

 	0x2229,0x222a,		// intersection

 	0x222b,0x222c,		// union

 	0x2248,0x2249,		// almost equal to

 	0x2260,0x2262,		// not equal to, identical to

 	0x2264,0x2266,		// less-than-or-equal-to, greater-than-or-equal-to

 	0x2302,0x2303,		// house

 	0x2310,0x2311,		// rversed not sign

 	0x2320,0x2322,		// top and bottom of integral

 	0x2500,0x2501,		// box drawing

 	0x2502,0x2503,		// box drawing

 	0x250c,0x250d,		// box drawing

 	0x2510,0x2511,		// box drawing

 	0x2514,0x2515,		// box drawing

 	0x2518,0x2519,		// box drawing

 	0x251c,0x251d,		// box drawing

 	0x2524,0x2525,		// box drawing

 	0x252c,0x252d,		// box drawing

 	0x2534,0x2535,		// box drawing

 	0x253c,0x253d,		// box drawing

 	0x2550,0x256d,		// box drawing

 	0x2580,0x2581,		// block element

 	0x2584,0x2585,		// block element

 	0x2588,0x2589,		// block element

 	0x258c,0x258d,		// block element

 	0x2590,0x2594,		// block element

 	0x25a0,0x25a2,		// geometric shapes

 	0x25aa,0x25ad,		// geometric shapes

 	0x25b2,0x25b3,		// geometric shapes

 	0x25ba,0x25bb,		// geometric shapes

 	0x25bc,0x25bd,		// geometric shapes

 	0x25c4,0x25c5,		// geometric shapes

 	0x25ca,0x25cc,		// geometric shapes

 	0x25cf,0x25d0,		// geometric shapes

 	0x25d8,0x25da,		// geometric shapes

 	0x25e6,0x25e7,		// geometric shapes

 	0x263a,0x263d,		// smilies, sun

 	0x2640,0x2641,		// female

 	0x2642,0x2643,		// male

 	0x2660,0x2661,		// spade

 	0x2663,0x2664,		// club

 	0x2665,0x2667,		// heart

 	0x266a,0x266c,		// quaver, beamed quavers

 	0xfb01,0xfb03,		// fi, fl ligatures

 	0xfeff,0xff00,		// zero-width non-breaking space

 	0xfffc, 0xfffe		// object replacement character and replacement character

 	};

 const int Wgl4Ranges = sizeof(Wgl4Range) / sizeof(Wgl4Range[0]) / 2;

 int CompareWgl4Ranges(const void* aRange1,const void* aRange2)

 	{

 	unsigned int* p = (unsigned int*)aRange1;

 	unsigned int* q = (unsigned int*)aRange2;

 	if (q[0] == q[1])

 		{

 		unsigned int* temp = p;

 		p = q;

 		q = temp;

 		}

 	if (*p < *q)

 		return -1;

 	else if (*p >= q[1])

 		return 1;

 	else

 		return 0;

 	}

 // Determine if a character is in the WGL4 character repertoire.

 static bool InWgl4(unsigned int aChar)

 	{

 	unsigned int key[2];

 	key[0] = key[1] = aChar;

 	return bsearch(key,Wgl4Range,Wgl4Ranges,sizeof(Wgl4Range[0]) * 2,CompareWgl4Ranges) != NULL;

 	}

 // A collation key.

 class CollationKey

 	{

 public:

 	bool operator==(const CollationKey& k) const

 		{ return iLevel[0] == k.iLevel[0] && iLevel[1] == k.iLevel[1] && iLevel[2] == k.iLevel[2] &&

 		  iIgnorable == k.iIgnorable && iStop == k.iStop; }

 	enum

 		{

 		ELevels = 3

 		};

 	int iLevel[ELevels];// the keys at the various levels

 	bool iIgnorable;	// TRUE if this key can normally be ignored

 	bool iStop;			// TRUE if this is the last key in a string of keys

 	};

 // The collation index for a single Unicode value.

 class CollationIndex

 	{

 public:

 	static int Compare(const void* aIndex1,const void* aIndex2);

 	int iCode;			// Unicode value

 	int iIndex;			// index into the key table

 	};

 class Reader

 	{

 public:

 	Reader(bool aWgl4,bool aStandard,const char* aLocaleName, const char* aUidString);

 	~Reader();

 	void ReadBaseKeys(const char* aFileName);

 	void ReadCompKeys(const char* aFileName);

 	void ReadStrings(const char* aFileName);

 	void ReadAllKeys(const char* aFileName);

 	void WriteOutput(const char* aFileName, bool aCopyrightMessage);

 	int CompareStringIndices(int aIndex1,int aIndex2) const;

 private:

 	Reader(const Reader&);

 	int Hex(const char *aString, int &aCharConsumed, bool aTolerate = false);

 	void GetCollationKey(const char* aString, int& aCharConsumed, CollationKey* aKey=NULL);

 	void GetMultipleCollationKeys(const char* aString);

 	unsigned int PackKey(const CollationKey& aValue);

 	int PackIndex(const CollationIndex& aValue, unsigned int result[2]);

 	bool ParseLine(const char* aLine, int aCode[16], int& aCodeCount, int& aKeyStart, int& aKeyCount);

 	void AddKeyOneToOne(const char* aLine, const int aCode, const int aKeyStart);

 	void AddKeyOneToMuch(const char* aLine, const int aCode, const int aKeyStart);

 	void AddKeyMuchToMuch(const char* aLine, const int aCode[16], const int aCodeCount, const int aKeyStart);

 	enum

 		{

 		EMaxCollationKeys = 0x110000 * 2, /*more elements considering composite keys */

 		EMaxCollationIndices = 0x110000,

 		EMaxStringElements = 65536,

 		EMaxStringIndices = 65536

 		};

 	CollationKey iCollationKey[EMaxCollationKeys];

 	int iKeys;

 	CollationIndex iCollationIndex[EMaxCollationIndices];

 	int iIndices;

 	int iStringElement[EMaxStringElements];

 	int iStringElements;

 	unsigned int iStringIndex[EMaxStringIndices];

 	int iStringIndices;

 	const char* iInputFileName;

 	int iLineNumber;

 	bool iSuppressCanonseqWarning;		// have we issued the canonseq warning yet?

 	bool iWgl4;				// true if writing keys for wgl4 characters only

 	bool iStandard;			// true if reading standard files, not tailoring files

 	const char* iLocaleName;

 	const char* iUidString;

 	char* iCPlusPlusIdentifier;		// iLocaleName in title case with difficult characters removed

 	};

 bool isValidHexDigit(char c)

 	{

 	if ('0' <= c && c <= '9')

 		return true;

 	if ('a' <= c && c <= 'f')

 		return true;

 	if ('A' <= c && c <= 'F')

 		return true;

 	return false;

 	}

 void PrintUsage()

 	{

 	cout << "Usage: coltab [/u<uid>] [/c] [/a] [/h<topic>] <locale>\n";

 	cout << "By Default (without /a option), for the locales 'standard' and 'wgl4' coltab reads basekeys.txt & compkeys.txt\n";

 	cout << "For any other locale name <name> coltab reads <name>_basekeys.txt,\n";

 	cout << "<name>_compkeys.txt and <name>_strings.txt.\n";

 	cout << "Use the /a option, for the locales 'standard' and 'wgl4' coltab reads allkeys.txt\n";

 	cout << "For any other locale name <name> coltab reads <name>_allkeys.txt.\n"; 

 	cout << "The output file is always ls_<name>.cpp.\n";

 	cout << "Use the /u option to specify the UID that the collation table should have.\n";

 	cout << "A hex number must follow /u immediately, for example /u800ACBDE\n";

 	cout << "this hex number must not exceed eight digits. If this is not specified,\n";

 	cout << "the output file will have to be edited to make it compilable.\n";

 	cout << "Specify /c to prefix the output with a Nokia copyright message.\n";

 	cout << "Specify /h for in-depth help.";

 	}

 void UsageError()

 	{

 	PrintUsage();

 	exit(1);

 	}

 void PrintHelp(char* aTopic)

 	{

 	int topic = 0;

 	while ('0' <= *aTopic && *aTopic <= '9')

 		{

 		topic = topic * 10 + (*aTopic - '0');

 		++aTopic;

 		}

 	switch(topic)

 		{

 	case 1:

 		cout << "How Coltab interprets CANONSEQ:\n\n"\

 			"If the CANONSEQ specifier is used in a line, Coltab will ignore the mapping.\n"\

 			"This because, on the Symbian platform, any canonically composed character is\n"\

 			"decomposed before the key mapping is applied, so characters with canonical\n"\

 			"decompositions do not need keys. In files supplied by the Unicode Consortium,\n"\

 			"all mappings for composed characters are flagged by CANONSEQ, so it is useful\n"\

 			"if Coltab can just ignore these so that Unicode Consortium files can be used\n"\

 			"unedited.\n\n"\

 			"This can cause problems if a localizer copies a line from a Unicode file into,\n"\

 			"say, the <lang>_strings.txt file, in order to give a mapping for an accented\n"\

 			"character. The localizer replaces the composed character code with the\n"\

 			"decomposition and changes the keys but forgets to remove the CANONSEQ\n"\

 			"specifier. In this case the key would be ignored. Coltab provides a warning so\n"\

 			"that this can be put right.\n\n"\

 			"Coltab will only warn about the first CANONSEQ in each file, and does not warn\n"\

 			"if the 'standard' or 'wgl4' options are used.";

 		exit(1);

 		break;

 	case 2:

 		cout << "How to ensure coltab's output files are compilable.\n\n"\

 			"By default, Coltab's files for locales need to be edited before they are\n"\

 			"compilable. The UID for the collation method needs to be filled in. This UID\n"\

 			"is added so that the collation table can be searched for later. At present,\n"\

 			"this UID is not necessary for the correct functioning of the Symbian platform\n"\

 			"and so a value of 0 can be safely used.\n\n"\

 			"To insert this value into the file directly, use the /u option, for example\n"\

 			"coltab /u0 french\n"\

 			"If the /u option is used, the file should be compilable as is. If it is not,\n"\

 			"please raise it as a defect with Symbian's internationalization team,\n"\

 			"supplying the files that caused the problem if this is possible.\n"\

 			"If the 'standard' or 'wgl4' options are used, no UID is output, so the /u\n"\

 			"option is not required.";

 		exit(1);

 		break;

 	case 3:

 		cout << "How to ensure collation key values are inside the supported range. \n\n"\

 			"According to Unicode Standard, the range suppored by tool COLTAB:\n"\

 			" Level 0 (primary):   0000 - FFFF, \n"\

 			" Level 1 (Secondary): 0020 - 011E, \n"\

 			" Level 2 (Tertiary):  0001 - 003F. \n"\

 			"Please edit your collation files and make sure key values are inside the above range";

 		exit(1);

 		break;

 	default:

 		PrintUsage();

 		cout << "\n\nSpecify /h1 for help on the use of CANONSEQ\n";

 		cout << "Specify /h2 for help on making compilable files that do not need editing\n";

 		exit(1);

 		break;

 		}

 	}

 short HighSurrogate(int aCode)

 	{

 	return static_cast<short>(0xD7C0 + (aCode >> 10));

 	}

 short LowSurrogate(int aCode)

 	{

 	return static_cast<short>(0xDC00 | (aCode & 0x3FF));

 	}

 int main(int argc,char** argv)

 	{

 	bool copyright = false;

 	bool wgl4 = false;

 	bool allKeys = false;

 	const char* prefix = "";

 	const char* infix = "";

 	const char* locale = "";

 	char* localeArg = 0;

 	char* uidArg = 0;

 	for (int i = 1; i < argc; ++i)

 		{

 		if (argv[i][0] == '/' || argv[i][0] == '-')

 			{

 			switch (argv[i][1])

 				{

 			case 'u':

 			case 'U':

 				{

 				uidArg = argv[i] + 2;

 				const char* uidCheck = uidArg;

 				while (*uidCheck)

 					{

 					if (!isValidHexDigit(*uidCheck))

 						UsageError();

 					++uidCheck;

 					}

 				if (uidCheck == uidArg || 8 < uidCheck - uidArg)

 					UsageError();

 				break;

 				}

 			case 'c':

 			case 'C':

 				copyright = true;

 				break;

 			case 'a':

 				allKeys = true;

 				break;

 			case 'h':

 			case 'H':

 				PrintHelp(argv[i] + 2);

 				break;

 			default:

 				UsageError();

 				break;

 				}

 			}

 		else if (!localeArg)

 			localeArg = argv[i];

 		else

 			UsageError();

 		}

 	if (!localeArg)

 		UsageError();

 	bool standard = false;

 	if (!_stricmp(localeArg, "standard"))

 		{

 		locale = "Standard";

 		standard = true;

 		}

 	else if (!_stricmp(localeArg, "wgl4"))

 		{

 		locale = "Wgl4";

 		wgl4 = true;

 		standard = true;

 		}

 	else

 		{

 		locale = prefix = localeArg;

 		infix = "_";

 		}

 	Reader* reader = new Reader(wgl4, standard, locale, uidArg);

 	if (!reader)

 		{

 		cout << "out of memory\n";

 		exit(1);

 		}

 	char* filename = new char[strlen(prefix) + strlen(infix) + 64];

 	if (allKeys == false)

 		{

 		sprintf(filename,"%s%scompkeys.txt",prefix,infix);

 		reader->ReadCompKeys(filename);

 		if (!standard)

 			{

 			sprintf(filename,"%s%sstrings.txt",prefix,infix);

 			reader->ReadStrings(filename);

 			}

 		sprintf(filename,"%s%sbasekeys.txt",prefix,infix);

 		reader->ReadBaseKeys(filename);

 		}

 	else

 		{

 		sprintf(filename,"%s%sAllKeys.txt",prefix,infix);

 		reader->ReadAllKeys(filename);

 		}

 	sprintf(filename,"ls_%s.cpp", localeArg);

 	reader->WriteOutput(filename, copyright);

 	delete reader;

 	delete [] filename;

 	return 0;

 	}

 Reader::Reader(bool aWgl4, bool aStandard,

 	const char* aLocaleName, const char* aUidString):

 	iKeys(0),

 	iIndices(0),

 	iStringElements(0),

 	iStringIndices(0),

 	iInputFileName(NULL),

 	iLineNumber(0),

 	iSuppressCanonseqWarning(false),

 	iWgl4(aWgl4),

 	iStandard(aStandard),

 	iLocaleName(aLocaleName),

 	iUidString(aUidString)

 	{

 	if (iStandard)

 		{

 		iCPlusPlusIdentifier = new char[9];

 		strcpy(iCPlusPlusIdentifier, "Standard");

 		return;

 		}

 	char* p = iCPlusPlusIdentifier = new char[strlen(aLocaleName) + 2];

 	int current = toupper(aLocaleName[0]);

 	if (current < 'A' || 'Z' < current)

 		*p++ = 'C';

 	else

 		{

 		*p++ = static_cast<char>(current);

 		++aLocaleName;

 		}

 	bool inUnderScore = false;

 	while (*aLocaleName)

 		{

 		current = tolower(*aLocaleName++);

 		if (current < 'a' || 'z' < current)

 			{

 			if (!inUnderScore)

 				{

 				inUnderScore = true;

 				*p++ = '_';

 				}

 			}

 		else

 			{

 			inUnderScore = false;

 			*p++ = static_cast<char>(current);

 			}

 		}

 	*p = 0;

 	}

 Reader::~Reader()

 	{

 	delete [] iCPlusPlusIdentifier;

 	}

 // Get a hex number of exactly four digits from aString. Return -1 if none is found and aTolerate is true.

 int Reader::Hex(const char *aString, int &aCharConsumed, bool aTolerate)

 	{

 	char *end;

 	unsigned long x = strtoul(aString,&end,16);

 	aCharConsumed = end - aString;

 	if ((aCharConsumed != 4) && (aCharConsumed != 5) && (aCharConsumed != 6))

 		{

 		if (!aTolerate)

 			{

 			cout << "bad hex number on line " << iLineNumber << " of file " << iInputFileName << '\n';

 			exit(1);

 			}

 		return -1;

 		}

 	return x;

 	}

 // Get a collation value from a string of the form [.xxxx.xxxx.xxxx.xxxx]

 void Reader::GetCollationKey(const char* aString, int& aCharConsumed, CollationKey* aKey)

 	{

 	aCharConsumed = 0;

 	const char *end = strchr(aString, ']');

 	if (end != NULL){

 		aCharConsumed = end - aString;

 	}

 	if (aString[0] != '[' || (aCharConsumed != 21 && aCharConsumed != 22 && aCharConsumed != 23))

 		{

 		cout << "syntax error on line " << iLineNumber << " of file " << iInputFileName << '\n';

 		exit(1);

 		}

 	if (aKey == NULL)

 		{

 		if (iKeys >= EMaxCollationKeys)

 			{

 			cout << "too many keys";

 			exit(1);

 			}

 		aKey = &iCollationKey[iKeys++];

 		}

 	aKey->iIgnorable = aString[1] == '*'; // asterisk means that this character is normally ignored

 	int charConsumed = 0;

 	for (int i = 0; i < CollationKey::ELevels; i++)

 		aKey->iLevel[i] = Hex(aString + 2 + i * 5, charConsumed);

 	if (aKey->iLevel[1] > 0 && (aKey->iLevel[1] < KLevel1Min || aKey->iLevel[1] > KLevel1Max))

 		{

 		aKey->iLevel[1] = KLevel1Max;

 		cout << "illegal level-1 key value on line " << iLineNumber << "; outside the range " << KLevel1Min << ".." << KLevel1Max << "\n";

 		cout << "Error: illegal key value in file, please see coltab /h3 for details.\n";

 		exit(1);

 		}

 	if (aKey->iLevel[2] > 0 && (aKey->iLevel[2] < KLevel2Min || aKey->iLevel[2] > KLevel2Max))

 		{

 		cout << "illegal level-2 key value on line " << iLineNumber << "; outside the range " << KLevel2Min << ".." << KLevel2Max << "\n";

 		cout << "Error: illegal key value in file, please see coltab /h3 for details.\n";

 		exit(1);

 		}

 	aKey->iStop = true;

 	}

 void Reader::GetMultipleCollationKeys(const char* aString)

 	{

 	int keyCount = 0;

 	int charConsumed =0;

 	while (aString[0] == '[')

 		{

 		GetCollationKey(aString, charConsumed);

 		keyCount++;

 		iCollationKey[iKeys - 1].iStop = false;

 		int length = strlen(aString);

 		if (length <= charConsumed + 1)

 			break;

 		aString += charConsumed + 1;

 		if (aString[0] == ' ') //a space is put between collation keys in keys files provided by previous Unicode Standard (i.e 3.1)

 			aString++;

 		}

 	iCollationKey[iKeys - 1].iStop = true;

 	}

 /*

 Partially parse a line, returning its key code and the start of its first block of key data.

 Return false if it is not a data line, or not relevant.

 */

 bool Reader::ParseLine(const char* aLine, int aCode[16], int& aCodeCount, int& aKeyStart, int& aKeyCount)

 	{

 	int lineLength = strlen(aLine);

 	int charConsumed = 0;

 	aCodeCount = 0;

 	aCode[0] = Hex(aLine,charConsumed,true);

 	/*

 	A data line must start with a hex number and be at least 27 characters long.

 	Canonically decomposable Unicode characters are skipped.

 	Skip non-WGL4 characters if doing WGL4 only.

 	*/

 	if (aCode[0] != -1)

 		{

 		aCodeCount = 1;

 		if (!strcmp(aLine + lineLength - 8,"CANONSEQ"))

 			{

 			if (!iSuppressCanonseqWarning)

 				{

 				cout << "Warning: CANONSEQ used in file " << iInputFileName

 					<< " on line " << iLineNumber << ".\nWarning: All mappings specifying CANONSEQ are ignored.\n"

 					<< "Warning: Use coltab /h1 for more details.";

 				iSuppressCanonseqWarning = true;

 				}

 			aCodeCount = 0;

 			}

 		else if (lineLength < 27 ||

 			(iWgl4 && !InWgl4((unsigned int)aCode))) 

 			aCodeCount = 0;

 		}

 	if (aCode[0] != -1)

 		{

 		// find '['

 		aKeyStart = charConsumed;

 		while (aKeyStart < lineLength && aLine[aKeyStart] != '[')

 			aKeyStart++;

 		// read all hex before '['

 		int index = charConsumed + 1;

 		while (index < aKeyStart)

 			{

 			aCode[aCodeCount] = Hex(aLine+index, charConsumed, true);

 			if (aCode[aCodeCount] == -1)

 				break;

 			index += charConsumed + 1;

 			aCodeCount++;

 			}

 		// find number of collation keys

 		aKeyCount = 0;

 		index = aKeyStart;

 		while (index < lineLength && aLine[index] != '%' && aLine[index] != '#')

 			{

 			if (aLine[index] == '[')

 				aKeyCount++;

 			index++;

 			}

 		}

 	return aCodeCount > 0;

 	}

 void Reader::AddKeyOneToOne(const char* aLine, const int aCode, const int aKeyStart)

 	{

 	if (iIndices >= EMaxCollationIndices)

 		{

 		cout << "too many Unicode values";

 		exit(1);

 		}

 	CollationIndex& index = iCollationIndex[iIndices++];

 	index.iCode = aCode;

 	index.iIndex = -1;

 	/*

 	First try to find the key in the array of keys found so far.

 	Search backwards to use the fact that runs of the same key occur together.

 	*/

 	CollationKey key;

 	int charConsumed = 0;

 	GetCollationKey(aLine + aKeyStart, charConsumed, &key);

 	for (int i = iKeys - 1; i >= 0 && index.iIndex == -1; i--)

 		if (iCollationKey[i] == key)

 			index.iIndex = i;

 	// If that fails, add a new key.

 	if (index.iIndex == -1)

 		{

 		index.iIndex = iKeys++;

 		if (iKeys > EMaxCollationKeys)

 			{

 			cout << "too many keys";

 			exit(1);

 			} 

 		iCollationKey[index.iIndex] = key;

 		}

 	}

 /*

 Read 1-to-1 mapping. Sample:

 02B9 ; [*02A5.0020.0002.02B9] % MODIFIER LETTER PRIME

 aCombinedFile = true: aFileName is combined file, which contains base keys, comp keys, and string keys.

 */

 void Reader::ReadBaseKeys(const char* aFileName)

 	{

 	iSuppressCanonseqWarning = iStandard || iWgl4;

 	iLineNumber = 0;

 	iInputFileName = aFileName;

 	ifstream input_file;

 #ifdef __MSVCDOTNET__

 	input_file.open(iInputFileName, ios::in);

 #else //!__MSVCDOTNET__

 	input_file.open(iInputFileName, ios::in | ios::nocreate);

 #endif //__MSVCDOTNET__

 	if (input_file.fail())

 		{

 		cout << "cannot open input file '" << iInputFileName << "'\n";

 		exit(1);

 		}

 	cout << "reading base keys from '" << iInputFileName << "'\n";

 	char line[1024];

 	for (;;)

 		{

 		input_file.getline(line,sizeof(line));

 		if (input_file.eof())

 			break;

 		iLineNumber++;

 		// line number counting

 		if (iLineNumber % 100 == 0)

 			{

 			cout << "line " << iLineNumber << '\n';

 			cout.flush();

 			}

 		int code[16];

 		int codeCount = 0;

 		int key_start = 0;

 		int keyCount = 0;

 		if (ParseLine(line, code, codeCount, key_start, keyCount)) 

 			{

 			if (codeCount != 1 || keyCount != 1)

 				continue;	// goto next line

 			AddKeyOneToOne(line, code[0], key_start);

 			}

 		}

 	input_file.close();

 	}

 void Reader::AddKeyOneToMuch(const char* aLine, const int aCode, const int aKeyStart)

 	{

 	if (iIndices >= EMaxCollationIndices)

 		{

 		cout << "too many Unicode values";

 		exit(1);

 		}

 	CollationIndex& index = iCollationIndex[iIndices++];

 	index.iCode = aCode;

 	index.iIndex = iKeys;

 	GetMultipleCollationKeys(aLine + aKeyStart);

 	}

 /*

 Read 1-to-much mapping.

 3303  ; [.279F.0020.001C.3303][.1114.0020.001C.3303][.27C7.0020.001F.3303] # SQUARE AARU; QQKN

 */

 void Reader::ReadCompKeys(const char* aFileName)

 	{

 	iSuppressCanonseqWarning = iStandard || iWgl4;

 	iLineNumber = 0;

 	iInputFileName = aFileName;

 	ifstream input_file;

 #ifdef __MSVCDOTNET__

 	input_file.open(iInputFileName, ios::in);

 #else //!__MSVCDOTNET__

 	input_file.open(iInputFileName, ios::in | ios::nocreate);

 #endif //__MSVCDOTNET__

 	if (input_file.fail())

 		{

 		cout << "there are no composite keys; '" << iInputFileName << "' not found\n";

 		return;

 		}

 	cout << "reading composite keys from '" << iInputFileName << "'\n";

 	char line[1024];

 	for (;;)

 		{

 		input_file.getline(line,sizeof(line));

 		if (input_file.eof())

 			break;

 		iLineNumber++;

 		// line number counting

 		if (iLineNumber % 100 == 0)

 			{

 			cout << "line " << iLineNumber << '\n';

 			cout.flush();

 			}

 		int code[16];

 		int codeCount = 0;

 		int key_start = 0;

 		int keyCount = 0;

 		if (ParseLine(line, code, codeCount, key_start, keyCount)) 

 			{

 			if (codeCount != 1 || keyCount < 2)

 				continue;	// goto next line

 			AddKeyOneToMuch(line, code[0], key_start);

 			}

 		}

 	input_file.close();

 	}

 void Reader::AddKeyMuchToMuch(const char* aLine, const int aCode[16], const int aCodeCount, const int aKeyStart)

 	{

 	// Store the index to the Unicode string and the key sequence.

 	if (iStringIndices > EMaxStringIndices)

 		{

 		cout << "too many string indices";

 		exit(1);

 		}

 	iStringIndex[iStringIndices++] = (iStringElements << 16) | iKeys;

 	// Reserve space for the length.

 	if (iStringElements >= EMaxStringElements)

 		{

 		cout << "too many string elements";

 		exit(1);

 		}

 	iStringElements++;

 	// Read the Unicode string.

 	int length = 0;		// in unit of int16

 	int charCount = 0;	// in unit of char. for debug.

 	for (int i=0; i<aCodeCount; i++)

 		{	

 		if (iStringElements >= EMaxStringElements)

 			{

 			cout << "too many string elements";

 			exit(1);

 			}

 		if (aCode[i] > 0xFFFF)

 			{

 			// UCS4 --> UTF-16

 			iStringElement[iStringElements++] = 0xD7C0 + (aCode[i] >> 10);

 			iStringElement[iStringElements++] = 0xDC00 | (aCode[i] & 0x3FF);

 			length += 2;

 			}

 		else

 			{

 			iStringElement[iStringElements++] = aCode[i];

 			length++;

 			}

 		charCount++;

 		}

 	iStringElement[iStringElements - length - 1] = (unsigned int)length;

 	// Read the key sequence.

 	GetMultipleCollationKeys(aLine + aKeyStart);

 	}

 /*

 Read much-to-much mapping. Sample:

 004F 0338 [.08EA.0020.0008.00D8] % capital O-stroke

 0E40 0E08 ; [.1E2B.0020.0002.0E08][.1E5E.0020.001F.0E40] # <THAI CHARACTER SARA E, THAI CHARACTER CHO CHAN>

 */

 void Reader::ReadStrings(const char* aFileName)

 	{

 	iSuppressCanonseqWarning = iStandard || iWgl4;

 	iLineNumber = 0;

 	iInputFileName = aFileName;

 	ifstream input_file;

 #ifdef __MSVCDOTNET__

 	input_file.open(iInputFileName, ios::in);

 #else //!__MSVCDOTNET__

 	input_file.open(iInputFileName, ios::in | ios::nocreate);

 #endif //__MSVCDOTNET__

 	if (input_file.fail())

 		{

 		cout << "there are no strings; '" << iInputFileName << "' not found\n";

 		return;

 		}

 	cout << "reading strings from '" << iInputFileName << "'\n";

 	char line[1024];

 	for (;;)

 		{

 		input_file.getline(line,sizeof(line));

 		if (input_file.eof())

 			break;

 		iLineNumber++;

 		// line number counting

 		if (iLineNumber % 100 == 0)

 			{

 			cout << "line " << iLineNumber << '\n';

 			cout.flush();

 			}

 		int code[16];

 		int codeCount = 0;

 		int key_start = 0;

 		int keyCount = 0;

 		if (ParseLine(line, code, codeCount, key_start, keyCount)) 

 			{

 			if (codeCount < 2 || keyCount < 1)

 				continue;	// goto next line

 			AddKeyMuchToMuch(line, code, codeCount, key_start);

 			}

 		}

 	input_file.close();

 	}

 /*

 Read combined key table. Sample:

 1-to-1 mapping:

 02B9 ; [*02A5.0020.0002.02B9] % MODIFIER LETTER PRIME

 1-to-much mapping:

 3303  ; [.279F.0020.001C.3303][.1114.0020.001C.3303][.27C7.0020.001F.3303] # SQUARE AARU; QQKN

 much-to-much mapping:

 004F 0338 [.08EA.0020.0008.00D8] % capital O-stroke

 0E40 0E08 ; [.1E2B.0020.0002.0E08][.1E5E.0020.001F.0E40] # <THAI CHARACTER SARA E, THAI CHARACTER CHO CHAN>

 */

 void Reader::ReadAllKeys(const char* aFileName)

 	{

 	iSuppressCanonseqWarning = iStandard || iWgl4;

 	iLineNumber = 0;

 	iInputFileName = aFileName;

 	ifstream input_file;

 #ifdef __MSVCDOTNET__

 	input_file.open(iInputFileName, ios::in);

 #else //!__MSVCDOTNET__

 	input_file.open(iInputFileName, ios::in | ios::nocreate);

 #endif //__MSVCDOTNET__

 	if (input_file.fail())

 		{

 		cout << "there are no keys; '" << iInputFileName << "' not found\n";

 		return;

 		}

 	cout << "reading all keys from '" << iInputFileName << "'\n";

 	char line[1024];

 	for (;;)

 		{

 		if (input_file.eof())

 			break;

 		input_file.getline(line,sizeof(line));

 		iLineNumber++;

 		int code[16];

 		int codeCount = 0;

 		int key_start = 0;

 		int keyCount = 0;

 		if (ParseLine(line, code, codeCount, key_start, keyCount)) 

 			{

 			if (codeCount == 1 && keyCount == 1)

 				AddKeyOneToOne(line, code[0], key_start);

 			else if (codeCount == 1 && keyCount > 1)

 				AddKeyOneToMuch(line, code[0], key_start);

 			else if (codeCount > 1 && keyCount > 0)

 			AddKeyMuchToMuch(line, code, codeCount, key_start);

 			else

 				cout << "ignore line: " << line << "\n";

 			}

 		}

 	input_file.close();

 	}

 // Pack the 3 collation key levels into a single 32-bit integer.

 unsigned int Reader::PackKey(const CollationKey& aValue)

 	{

 	unsigned int level0 = aValue.iLevel[0];

 	unsigned int level1 = aValue.iLevel[1];

 	if (level1 > 0)

 		level1 -= (KLevel1Min - 1);

 	unsigned int level2 = aValue.iLevel[2];

 	if (level2 > 0)

 		level2 -= (KLevel2Min - 1);

 	unsigned int key = level0 << 16 | level1 << 8 | level2 << 2;

 	if (aValue.iIgnorable)

 		key |= 2;

 	if (aValue.iStop)

 		key |= 1;

 	return key;

 	}

 // Pack a collation index value into a single 32-bit integer.

 int Reader::PackIndex(const CollationIndex& aValue, unsigned int result[2])

 	{

 	unsigned int code = aValue.iCode;

 	unsigned int index = aValue.iIndex;

 	if (code <= 0xFFFF)

 		{

 		result[0] = (code << 16 | index);

 		return 1;

 		}

 	else

 		{

 		result[0] = (::HighSurrogate(code) << 16 | index);

 		result[1] = (::LowSurrogate(code) << 16 | index);

 		return 2;

 		}

 	}

 const Reader* TheReader;

 static int CompareStringIndices(const void* aIndex1,const void* aIndex2)

 	{

 	return TheReader->CompareStringIndices(*(unsigned int*)aIndex1 >> 16,*(unsigned int*)aIndex2 >> 16);

 	}

 int CompareUnicodeStrings(const int *aString1,int aLength1,const int *aString2,int aLength2)

 	{

 	for (int i = 0; i < aLength1 || i < aLength2; i++, aString1++, aString2++)

 		{

 		int x = i < aLength1 ? *aString1 : -1;

 		int y = i < aLength2 ? *aString2 : -1;

 		if (x != y)

 			return x - y;

 		}

 	return 0;

 	}

 int Reader::CompareStringIndices(int aIndex1,int aIndex2) const

 	{

 	return CompareUnicodeStrings(iStringElement + aIndex1 + 1,iStringElement[aIndex1],

 								 iStringElement + aIndex2 + 1,iStringElement[aIndex2]);

 	}

 void Reader::WriteOutput(const char* aFileName, bool aCopyright)

 	{

 	int i;

 	ofstream output_file;

 	output_file.open(aFileName);

 	if (output_file.fail())

 		{

 		cout << "cannot open output file '" << aFileName << "'\n";

 		exit(1);

 		}

 	cout << "writing output to '" << aFileName << "'\n";

 	char *locale = NULL;

 	if (iStandard)

 		locale = _strdup("Standard");

 	else

 		locale = _strdup(iLocaleName);

 	if (!iStandard)

 		{

 		_strlwr(locale);

 		locale[0] = (char)toupper(locale[0]);

 		if (aCopyright)

 			{

 			char* capsFileName = new char[strlen(aFileName) + 1];

 			strcpy(capsFileName, aFileName);

 			_strupr(capsFileName);

 			output_file << "/*\n" << capsFileName << "\n\nCopyright (C) 2000-2009 Nokia Corporation and/or its subsidiary(-ies). All rights reserved.\n*/\n";

 			delete [] capsFileName;

 			output_file << "\n/*\nThe LCharSet object used by the " << locale << " locale.\n";

 			output_file << "Generated by COLTAB.\n*/\n";

 			}

 		output_file << "\n#include \"ls_std.h\"\n#include <collate.h>\n";

 		output_file << "\nconst TUint KUid" << iCPlusPlusIdentifier << "CollationMethod = ";

 		if (iUidString)

 			output_file << "0x" << iUidString << ";\n";

 		else

 			{

 			output_file << "/* FILL THIS IN */;\n";

 			cout << "Warning: File will need editing\nWarning: see coltab /h2 for details.\n";

 			}

 		}

 	/*

 	Write the unique collation keys.

 	Each one has the format, going from highest to lowest bit:

 	16 bits:	level-0 key

 	8 bits:		level-1 key

 	6 bits:		level-2 key

 	1 bit:		set if this key is optionally ignorable

 	1 bit:		set if this is the last key in the string of keys for a single Unicode value

 	*/

 	if (iKeys != 0)

 		{

 		output_file << "\nstatic const TUint32 The" << iCPlusPlusIdentifier << "Key[] = \n\t{";

 		CollationKey* ck = iCollationKey;

 		output_file << "\t // " << iKeys << " keys";

 		output_file << hex;

 		for (i = 0; i < iKeys; i++, ck++)

 			{

 			unsigned int key = PackKey(*ck);

 			if (i % 8 == 0)

 				output_file << "\n\t";

 			output_file << "0x";

 			output_file << key << ",";

 			}

 		output_file << dec;

 		output_file << "\n\t};\n\n";

 		}

 	if (iIndices != 0)

 		{

 		// Sort then write the collation index values - these relate Unicode values to collation keys.

 		qsort(iCollationIndex,iIndices,sizeof(CollationIndex),CollationIndex::Compare);

 		output_file << "static const TUint32 The" << iCPlusPlusIdentifier << "Index[] = \n\t{";

 		CollationIndex* ci = iCollationIndex;

 		int entry=0;

 		output_file << "\t // " << iIndices << " indices";

 		output_file << hex;

 		for (i = 0; i < iIndices; i++, ci++, entry++)

 			{

 			unsigned int key[2];

 			int bytecount = PackIndex(*ci, key);

 			if (entry % 8 == 0)

 				output_file << "\n\t";

 			output_file << "0x";

 			output_file << key[0] << ",";

 			if (bytecount == 2)

 				{

 				entry++;

 				if (entry % 8 == 0)

 					output_file << "\n\t";

 				output_file << "0x";

 				output_file << key[1] << ",";

 				}

 			}

 		output_file << dec;

 		output_file << "\n\t};";

 		output_file << "\t // " << entry << " entries";

 		output_file << "\n\n";

 		iIndices = entry; //One surrogate pair occupies 2 entries 

 		}

 	if (iStringElements)

 		{

 		// Write the Unicode strings; these are preceded by their lengths.

 		output_file << "static const TUint16 The" << iCPlusPlusIdentifier << "StringElement[] = \n\t{";

 		output_file << hex;

 		for (i = 0; i < iStringElements; i++)

 			{

 			if (i % 8 == 0)

 				output_file << "\n\t";

 			output_file << "0x" << iStringElement[i] << ",";

 			}

 		output_file << dec;

 		if (iStringElements==0)

 			output_file << "0";

 		output_file << "\n\t};\n\n";

 		/*

 		Sort then write the string index values - these relate Unicode strings to collation keys.

 		Each one has the string index in the upper word and the key index in the lower word.

 		*/

 		TheReader = this;

 		qsort(iStringIndex,iStringIndices,sizeof(iStringIndex[0]),::CompareStringIndices);

 		output_file << "static const TUint32 The" << iCPlusPlusIdentifier << "StringIndex[] = \n\t{";

 		output_file << hex;

 		for (i = 0; i < iStringIndices; i++)

 			{

 			if (i % 8 == 0)

 				output_file << "\n\t";

 			output_file << "0x" << iStringIndex[i] << ",";

 			}

 		output_file << dec;

 		if (iStringIndices ==0)

 			output_file << "0";

 		output_file << "\n\t};\n\n";

 		}

 	// Write the collation table structure.

 	output_file << "static const TCollationKeyTable The" << iCPlusPlusIdentifier << "Table = \n\t{ ";

 	if (iKeys)

 		output_file << "The" << iCPlusPlusIdentifier << "Key";

 	else

 		output_file << "0";

 	if (iIndices)

 		output_file << ", The" << iCPlusPlusIdentifier << "Index, " << iIndices;

 	else

 		output_file << ", 0, 0";

 	if (iStringElements)

 		output_file << ", The" << iCPlusPlusIdentifier << "StringElement, The" << iCPlusPlusIdentifier << "StringIndex, " << iStringIndices << " };\n";

 	else

 		output_file << ", 0, 0, 0 };\n";

 	if (!iStandard)

 		output_file << "\nstatic const TCollationMethod TheCollationMethod[] = \n"\

 			"	{\n"\

 			"		{\n"\

 			"		KUid" << iCPlusPlusIdentifier << "CollationMethod, // the method for the locale\n"\

 			"		NULL, // use the standard table as the main table\n"\

 			"		&The" << iCPlusPlusIdentifier << "Table, // the locale values override the standard values\n"\

 			"		0 // the flags are standard\n"\

 			"		},\n"\

 			"		{\n"\

 			"		KUidBasicCollationMethod, // the standard unlocalised method\n"\

 			"		NULL, // null means use the standard table\n"\

 			"		NULL, // there's no override table\n"\

 			"		0 // the flags are standard\n"\

 			"		}\n"\

 			"	};\n"\

 			"\n"\

 			"static const TCollationDataSet TheCollationDataSet =\n"\

 			"	{\n"\

 			"	TheCollationMethod,\n"\

 			"	2\n"\

 			"	};"\

 			"\n\n"\

 			"// The one and only locale character set object.\n"\

 			"const LCharSet TheCharSet =\n"\

 			"	{\n"\

 			"	NULL,\n"\

 			"	&TheCollationDataSet\n"\

 			"	};\n";

 	output_file.close();

 	delete [] locale;

 	}

 int CollationIndex::Compare(const void* aIndex1,const void* aIndex2)

 	{

 	return ((CollationIndex*)aIndex1)->iCode - ((CollationIndex*)aIndex2)->iCode;

 	}