sl@0: /* sl@0: ****************************************************************************** sl@0: * sl@0: * Copyright (C) 2003-2004, International Business Machines sl@0: * Corporation and others. All Rights Reserved. sl@0: * sl@0: ****************************************************************************** sl@0: * file name: ucnv_ext.h sl@0: * encoding: US-ASCII sl@0: * tab size: 8 (not used) sl@0: * indentation:4 sl@0: * sl@0: * created on: 2003jun13 sl@0: * created by: Markus W. Scherer sl@0: * sl@0: * Conversion extensions sl@0: */ sl@0: sl@0: #ifndef __UCNV_EXT_H__ sl@0: #define __UCNV_EXT_H__ sl@0: sl@0: #include "unicode/utypes.h" sl@0: sl@0: #if !UCONFIG_NO_CONVERSION sl@0: sl@0: #include "unicode/ucnv.h" sl@0: #include "ucnv_cnv.h" sl@0: sl@0: /* sl@0: * See icuhtml/design/conversion/conversion_extensions.html sl@0: * sl@0: * Conversion extensions serve two purposes: sl@0: * 1. They support m:n mappings. sl@0: * 2. They support extension-only conversion files that are used together sl@0: * with the regular conversion data in base files. sl@0: * sl@0: * A base file may contain an extension table (explicitly requested or sl@0: * implicitly generated for m:n mappings), but its extension table is not sl@0: * used when an extension-only file is used. sl@0: * sl@0: * It is an error if a base file contains any regular (not extension) mapping sl@0: * from the same sequence as a mapping in the extension file sl@0: * because the base mapping would hide the extension mapping. sl@0: * sl@0: * sl@0: * Data for conversion extensions: sl@0: * sl@0: * One set of data structures per conversion direction (to/from Unicode). sl@0: * The data structures are sorted by input units to allow for binary search. sl@0: * Input sequences of more than one unit are handled like contraction tables sl@0: * in collation: sl@0: * The lookup value of a unit points to another table that is to be searched sl@0: * for the next unit, recursively. sl@0: * sl@0: * For conversion from Unicode, the initial code point is looked up in sl@0: * a 3-stage trie for speed, sl@0: * with an additional table of unique results to save space. sl@0: * sl@0: * Long output strings are stored in separate arrays, with length and index sl@0: * in the lookup tables. sl@0: * Output results also include a flag distinguishing roundtrip from sl@0: * (reverse) fallback mappings. sl@0: * sl@0: * Input Unicode strings must not begin or end with unpaired surrogates sl@0: * to avoid problems with matches on parts of surrogate pairs. sl@0: * sl@0: * Mappings from multiple characters (code points or codepage state sl@0: * table sequences) must be searched preferring the longest match. sl@0: * For this to work and be efficient, the variable-width table must contain sl@0: * all mappings that contain prefixes of the multiple characters. sl@0: * If an extension table is built on top of a base table in another file sl@0: * and a base table entry is a prefix of a multi-character mapping, then sl@0: * this is an error. sl@0: * sl@0: * sl@0: * Implementation note: sl@0: * sl@0: * Currently, the parser and several checks in the code limit the number sl@0: * of UChars or bytes in a mapping to sl@0: * UCNV_EXT_MAX_UCHARS and UCNV_EXT_MAX_BYTES, respectively, sl@0: * which are output value limits in the data structure. sl@0: * sl@0: * For input, this is not strictly necessary - it is a hard limit only for the sl@0: * buffers in UConverter that are used to store partial matches. sl@0: * sl@0: * Input sequences could otherwise be arbitrarily long if partial matches sl@0: * need not be stored (i.e., if a sequence does not span several buffers with too sl@0: * many units before the last buffer), although then results would differ sl@0: * depending on whether partial matches exceed the limits or not, sl@0: * which depends on the pattern of buffer sizes. sl@0: * sl@0: * sl@0: * Data structure: sl@0: * sl@0: * int32_t indexes[>=32]; sl@0: * sl@0: * Array of indexes and lengths etc. The length of the array is at least 32. sl@0: * The actual length is stored in indexes[0] to be forward compatible. sl@0: * sl@0: * Each index to another array is the number of bytes from indexes[]. sl@0: * Each length of an array is the number of array base units in that array. sl@0: * sl@0: * Some of the structures may not be present, in which case their indexes sl@0: * and lengths are 0. sl@0: * sl@0: * Usage of indexes[i]: sl@0: * [0] length of indexes[] sl@0: * sl@0: * // to Unicode table sl@0: * [1] index of toUTable[] (array of uint32_t) sl@0: * [2] length of toUTable[] sl@0: * [3] index of toUUChars[] (array of UChar) sl@0: * [4] length of toUUChars[] sl@0: * sl@0: * // from Unicode table, not for the initial code point sl@0: * [5] index of fromUTableUChars[] (array of UChar) sl@0: * [6] index of fromUTableValues[] (array of uint32_t) sl@0: * [7] length of fromUTableUChars[] and fromUTableValues[] sl@0: * [8] index of fromUBytes[] (array of char) sl@0: * [9] length of fromUBytes[] sl@0: * sl@0: * // from Unicode trie for initial-code point lookup sl@0: * [10] index of fromUStage12[] (combined array of uint16_t for stages 1 & 2) sl@0: * [11] length of stage 1 portion of fromUStage12[] sl@0: * [12] length of fromUStage12[] sl@0: * [13] index of fromUStage3[] (array of uint16_t indexes into fromUStage3b[]) sl@0: * [14] length of fromUStage3[] sl@0: * [15] index of fromUStage3b[] (array of uint32_t like fromUTableValues[]) sl@0: * [16] length of fromUStage3b[] sl@0: * sl@0: * [17] Bit field containing numbers of bytes: sl@0: * 31..24 reserved, 0 sl@0: * 23..16 maximum input bytes sl@0: * 15.. 8 maximum output bytes sl@0: * 7.. 0 maximum bytes per UChar sl@0: * sl@0: * [18] Bit field containing numbers of UChars: sl@0: * 31..24 reserved, 0 sl@0: * 23..16 maximum input UChars sl@0: * 15.. 8 maximum output UChars sl@0: * 7.. 0 maximum UChars per byte sl@0: * sl@0: * [19] Bit field containing flags: sl@0: * (extension table unicodeMask) sl@0: * 1 UCNV_HAS_SURROGATES flag for the extension table sl@0: * 0 UCNV_HAS_SUPPLEMENTARY flag for the extension table sl@0: * sl@0: * [20]..[30] reserved, 0 sl@0: * [31] number of bytes for the entire extension structure sl@0: * [>31] reserved; there are indexes[0] indexes sl@0: * sl@0: * sl@0: * uint32_t toUTable[]; sl@0: * sl@0: * Array of byte/value pairs for lookups for toUnicode conversion. sl@0: * The array is partitioned into sections like collation contraction tables. sl@0: * Each section contains one word with the number of following words and sl@0: * a default value for when the lookup in this section yields no match. sl@0: * sl@0: * A section is sorted in ascending order of input bytes, sl@0: * allowing for fast linear or binary searches. sl@0: * The builder may store entries for a contiguous range of byte values sl@0: * (compare difference between the first and last one with count), sl@0: * which then allows for direct array access. sl@0: * The builder should always do this for the initial table section. sl@0: * sl@0: * Entries may have 0 values, see below. sl@0: * No two entries in a section have the same byte values. sl@0: * sl@0: * Each uint32_t contains an input byte value in bits 31..24 and the sl@0: * corresponding lookup value in bits 23..0. sl@0: * Interpret the value as follows: sl@0: * if(value==0) { sl@0: * no match, see below sl@0: * } else if(value<0x1f0000) { sl@0: * partial match - use value as index to the next toUTable section sl@0: * and match the next unit; (value indexes toUTable[value]) sl@0: * } else { sl@0: * if(bit 23 set) { sl@0: * roundtrip; sl@0: * } else { sl@0: * fallback; sl@0: * } sl@0: * unset value bit 23; sl@0: * if(value<=0x2fffff) { sl@0: * (value-0x1f0000) is a code point; (BMP: value<=0x1fffff) sl@0: * } else { sl@0: * bits 17..0 (value&0x3ffff) is an index to sl@0: * the result UChars in toUUChars[]; (0 indexes toUUChars[0]) sl@0: * length of the result=((value>>18)-12); (length=0..19) sl@0: * } sl@0: * } sl@0: * sl@0: * The first word in a section contains the number of following words in the sl@0: * input byte position (bits 31..24, number=1..0xff). sl@0: * The value of the initial word is used when the current byte is not found sl@0: * in this section. sl@0: * If the value is not 0, then it represents a result as above. sl@0: * If the value is 0, then the search has to return a shorter match with an sl@0: * earlier default value as the result, or result in "unmappable" even for the sl@0: * initial bytes. sl@0: * If the value is 0 for the initial toUTable entry, then the initial byte sl@0: * does not start any mapping input. sl@0: * sl@0: * sl@0: * UChar toUUChars[]; sl@0: * sl@0: * Contains toUnicode mapping results, stored as sequences of UChars. sl@0: * Indexes and lengths stored in the toUTable[]. sl@0: * sl@0: * sl@0: * UChar fromUTableUChars[]; sl@0: * uint32_t fromUTableValues[]; sl@0: * sl@0: * The fromUTable is split into two arrays, but works otherwise much like sl@0: * the toUTable. The array is partitioned into sections like collation sl@0: * contraction tables and toUTable. sl@0: * A row in the table consists of same-index entries in fromUTableUChars[] sl@0: * and fromUTableValues[]. sl@0: * sl@0: * Interpret a value as follows: sl@0: * if(value==0) { sl@0: * no match, see below sl@0: * } else if(value<=0xffffff) { (bits 31..24 are 0) sl@0: * partial match - use value as index to the next fromUTable section sl@0: * and match the next unit; (value indexes fromUTable[value]) sl@0: * } else { sl@0: * if(value==0x80000001) { sl@0: * return no mapping, but request for ; sl@0: * } sl@0: * if(bit 31 set) { sl@0: * roundtrip; sl@0: * } else { sl@0: * fallback; sl@0: * } sl@0: * // bits 30..29 reserved, 0 sl@0: * length=(value>>24)&0x1f; (bits 28..24) sl@0: * if(length==1..3) { sl@0: * bits 23..0 contain 1..3 bytes, padded with 00s on the left; sl@0: * } else { sl@0: * bits 23..0 (value&0xffffff) is an index to sl@0: * the result bytes in fromUBytes[]; (0 indexes fromUBytes[0]) sl@0: * } sl@0: * } sl@0: * sl@0: * The first pair in a section contains the number of following pairs in the sl@0: * UChar position (16 bits, number=1..0xffff). sl@0: * The value of the initial pair is used when the current UChar is not found sl@0: * in this section. sl@0: * If the value is not 0, then it represents a result as above. sl@0: * If the value is 0, then the search has to return a shorter match with an sl@0: * earlier default value as the result, or result in "unmappable" even for the sl@0: * initial UChars. sl@0: * sl@0: * If the from Unicode trie is present, then the from Unicode search tables sl@0: * are not used for initial code points. sl@0: * In this case, the first entries (index 0) in the tables are not used sl@0: * (reserved, set to 0) because a value of 0 is used in trie results sl@0: * to indicate no mapping. sl@0: * sl@0: * sl@0: * uint16_t fromUStage12[]; sl@0: * sl@0: * Stages 1 & 2 of a trie that maps an initial code point. sl@0: * Indexes in stage 1 are all offset by the length of stage 1 so that the sl@0: * same array pointer can be used for both stages. sl@0: * If (c>>10)>=(length of stage 1) then c does not start any mapping. sl@0: * Same bit distribution as for regular conversion tries. sl@0: * sl@0: * sl@0: * uint16_t fromUStage3[]; sl@0: * uint32_t fromUStage3b[]; sl@0: * sl@0: * Stage 3 of the trie. The first array simply contains indexes to the second, sl@0: * which contains words in the same format as fromUTableValues[]. sl@0: * Use a stage 3 granularity of 4, which allows for 256k stage 3 entries, sl@0: * and 16-bit entries in stage 3 allow for 64k stage 3b entries. sl@0: * The stage 3 granularity means that the stage 2 entry needs to be left-shifted. sl@0: * sl@0: * Two arrays are used because it is expected that more than half of the stage 3 sl@0: * entries will be zero. The 16-bit index stage 3 array saves space even sl@0: * considering storing a total of 6 bytes per non-zero entry in both arrays sl@0: * together. sl@0: * Using a stage 3 granularity of >1 diminishes the compactability in that stage sl@0: * but provides a larger effective addressing space in stage 2. sl@0: * All but the final result stage use 16-bit entries to save space. sl@0: * sl@0: * fromUStage3b[] contains a zero for "no mapping" at its index 0, sl@0: * and may contain UCNV_EXT_FROM_U_SUBCHAR1 at index 1 for " SUB mapping" sl@0: * (i.e., "no mapping" with preference for rather than ), sl@0: * and all other items are unique non-zero results. sl@0: * sl@0: * The default value of a fromUTableValues[] section that is referenced sl@0: * _directly_ from a fromUStage3b[] item may also be UCNV_EXT_FROM_U_SUBCHAR1, sl@0: * but this value must not occur anywhere else in fromUTableValues[] sl@0: * because "no mapping" is always a property of a single code point, sl@0: * never of multiple. sl@0: * sl@0: * sl@0: * char fromUBytes[]; sl@0: * sl@0: * Contains fromUnicode mapping results, stored as sequences of chars. sl@0: * Indexes and lengths stored in the fromUTableValues[]. sl@0: */ sl@0: enum { sl@0: UCNV_EXT_INDEXES_LENGTH, /* 0 */ sl@0: sl@0: UCNV_EXT_TO_U_INDEX, /* 1 */ sl@0: UCNV_EXT_TO_U_LENGTH, sl@0: UCNV_EXT_TO_U_UCHARS_INDEX, sl@0: UCNV_EXT_TO_U_UCHARS_LENGTH, sl@0: sl@0: UCNV_EXT_FROM_U_UCHARS_INDEX, /* 5 */ sl@0: UCNV_EXT_FROM_U_VALUES_INDEX, sl@0: UCNV_EXT_FROM_U_LENGTH, sl@0: UCNV_EXT_FROM_U_BYTES_INDEX, sl@0: UCNV_EXT_FROM_U_BYTES_LENGTH, sl@0: sl@0: UCNV_EXT_FROM_U_STAGE_12_INDEX, /* 10 */ sl@0: UCNV_EXT_FROM_U_STAGE_1_LENGTH, sl@0: UCNV_EXT_FROM_U_STAGE_12_LENGTH, sl@0: UCNV_EXT_FROM_U_STAGE_3_INDEX, sl@0: UCNV_EXT_FROM_U_STAGE_3_LENGTH, sl@0: UCNV_EXT_FROM_U_STAGE_3B_INDEX, sl@0: UCNV_EXT_FROM_U_STAGE_3B_LENGTH, sl@0: sl@0: UCNV_EXT_COUNT_BYTES, /* 17 */ sl@0: UCNV_EXT_COUNT_UCHARS, sl@0: UCNV_EXT_FLAGS, sl@0: sl@0: UCNV_EXT_RESERVED_INDEX, /* 20, moves with additional indexes */ sl@0: sl@0: UCNV_EXT_SIZE=31, sl@0: UCNV_EXT_INDEXES_MIN_LENGTH=32 sl@0: }; sl@0: sl@0: /* get the pointer to an extension array from indexes[index] */ sl@0: #define UCNV_EXT_ARRAY(indexes, index, itemType) \ sl@0: ((const itemType *)((const char *)(indexes)+(indexes)[index])) sl@0: sl@0: #define UCNV_GET_MAX_BYTES_PER_UCHAR(indexes) \ sl@0: ((indexes)[UCNV_EXT_COUNT_BYTES]&0xff) sl@0: sl@0: /* internal API ------------------------------------------------------------- */ sl@0: sl@0: U_CFUNC UBool sl@0: ucnv_extInitialMatchToU(UConverter *cnv, const int32_t *cx, sl@0: int32_t firstLength, sl@0: const char **src, const char *srcLimit, sl@0: UChar **target, const UChar *targetLimit, sl@0: int32_t **offsets, int32_t srcIndex, sl@0: UBool flush, sl@0: UErrorCode *pErrorCode); sl@0: sl@0: U_CFUNC UChar32 sl@0: ucnv_extSimpleMatchToU(const int32_t *cx, sl@0: const char *source, int32_t length, sl@0: UBool useFallback); sl@0: sl@0: U_CFUNC void sl@0: ucnv_extContinueMatchToU(UConverter *cnv, sl@0: UConverterToUnicodeArgs *pArgs, int32_t srcIndex, sl@0: UErrorCode *pErrorCode); sl@0: sl@0: sl@0: U_CFUNC UBool sl@0: ucnv_extInitialMatchFromU(UConverter *cnv, const int32_t *cx, sl@0: UChar32 cp, sl@0: const UChar **src, const UChar *srcLimit, sl@0: char **target, const char *targetLimit, sl@0: int32_t **offsets, int32_t srcIndex, sl@0: UBool flush, sl@0: UErrorCode *pErrorCode); sl@0: sl@0: U_CFUNC int32_t sl@0: ucnv_extSimpleMatchFromU(const int32_t *cx, sl@0: UChar32 cp, uint32_t *pValue, sl@0: UBool useFallback); sl@0: sl@0: U_CFUNC void sl@0: ucnv_extContinueMatchFromU(UConverter *cnv, sl@0: UConverterFromUnicodeArgs *pArgs, int32_t srcIndex, sl@0: UErrorCode *pErrorCode); sl@0: sl@0: U_CFUNC void sl@0: ucnv_extGetUnicodeSet(const UConverterSharedData *sharedData, sl@0: const USetAdder *sa, sl@0: UConverterUnicodeSet which, sl@0: UErrorCode *pErrorCode); sl@0: sl@0: /* toUnicode helpers -------------------------------------------------------- */ sl@0: sl@0: #define UCNV_EXT_TO_U_BYTE_SHIFT 24 sl@0: #define UCNV_EXT_TO_U_VALUE_MASK 0xffffff sl@0: #define UCNV_EXT_TO_U_MIN_CODE_POINT 0x1f0000 sl@0: #define UCNV_EXT_TO_U_MAX_CODE_POINT 0x2fffff sl@0: #define UCNV_EXT_TO_U_ROUNDTRIP_FLAG ((uint32_t)1<<23) sl@0: #define UCNV_EXT_TO_U_INDEX_MASK 0x3ffff sl@0: #define UCNV_EXT_TO_U_LENGTH_SHIFT 18 sl@0: #define UCNV_EXT_TO_U_LENGTH_OFFSET 12 sl@0: sl@0: /* maximum number of indexed UChars */ sl@0: #define UCNV_EXT_MAX_UCHARS 19 sl@0: sl@0: #define UCNV_EXT_TO_U_MAKE_WORD(byte, value) (((uint32_t)(byte)<>UCNV_EXT_TO_U_BYTE_SHIFT) sl@0: #define UCNV_EXT_TO_U_GET_VALUE(word) ((word)&UCNV_EXT_TO_U_VALUE_MASK) sl@0: sl@0: #define UCNV_EXT_TO_U_IS_PARTIAL(value) ((value)>UCNV_EXT_TO_U_LENGTH_SHIFT)-UCNV_EXT_TO_U_LENGTH_OFFSET) sl@0: sl@0: /* fromUnicode helpers ------------------------------------------------------ */ sl@0: sl@0: /* most trie constants are shared with ucnvmbcs.h */ sl@0: sl@0: /* see similar utrie.h UTRIE_INDEX_SHIFT and UTRIE_DATA_GRANULARITY */ sl@0: #define UCNV_EXT_STAGE_2_LEFT_SHIFT 2 sl@0: #define UCNV_EXT_STAGE_3_GRANULARITY 4 sl@0: sl@0: /* trie access, returns the stage 3 value=index to stage 3b; s1Index=c>>10 */ sl@0: #define UCNV_EXT_FROM_U(stage12, stage3, s1Index, c) \ sl@0: (stage3)[ ((int32_t)(stage12)[ (stage12)[s1Index] +(((c)>>4)&0x3f) ]< (impossible roundtrip to 0 bytes, value 01) */ sl@0: #define UCNV_EXT_FROM_U_SUBCHAR1 0x80000001 sl@0: sl@0: /* at most 3 bytes in the lower part of the value */ sl@0: #define UCNV_EXT_FROM_U_MAX_DIRECT_LENGTH 3 sl@0: sl@0: /* maximum number of indexed bytes */ sl@0: #define UCNV_EXT_MAX_BYTES 0x1f sl@0: sl@0: #define UCNV_EXT_FROM_U_IS_PARTIAL(value) (((value)>>UCNV_EXT_FROM_U_LENGTH_SHIFT)==0) sl@0: #define UCNV_EXT_FROM_U_GET_PARTIAL_INDEX(value) (value) sl@0: sl@0: #define UCNV_EXT_FROM_U_IS_ROUNDTRIP(value) (((value)&UCNV_EXT_FROM_U_ROUNDTRIP_FLAG)!=0) sl@0: #define UCNV_EXT_FROM_U_MASK_ROUNDTRIP(value) ((value)&~UCNV_EXT_FROM_U_ROUNDTRIP_FLAG) sl@0: sl@0: /* use after masking off the roundtrip flag */ sl@0: #define UCNV_EXT_FROM_U_GET_LENGTH(value) (int32_t)(((value)>>UCNV_EXT_FROM_U_LENGTH_SHIFT)&UCNV_EXT_MAX_BYTES) sl@0: sl@0: /* get bytes or bytes index */ sl@0: #define UCNV_EXT_FROM_U_GET_DATA(value) ((value)&UCNV_EXT_FROM_U_DATA_MASK) sl@0: sl@0: #endif sl@0: sl@0: #endif