sl@0: /* sl@0: ****************************************************************************** sl@0: * sl@0: * Copyright (C) 2000-2004, International Business Machines sl@0: * Corporation and others. All Rights Reserved. sl@0: * sl@0: ****************************************************************************** sl@0: * file name: ucnvmbcs.h sl@0: * encoding: US-ASCII sl@0: * tab size: 8 (not used) sl@0: * indentation:4 sl@0: * sl@0: * created on: 2000jul07 sl@0: * created by: Markus W. Scherer sl@0: */ sl@0: sl@0: #ifndef __UCNVMBCS_H__ sl@0: #define __UCNVMBCS_H__ sl@0: sl@0: #include "unicode/utypes.h" sl@0: sl@0: #if !UCONFIG_NO_CONVERSION sl@0: sl@0: #include "unicode/ucnv.h" sl@0: #include "ucnv_cnv.h" sl@0: sl@0: /** sl@0: * ICU conversion (.cnv) data file structure, following the usual UDataInfo sl@0: * header. sl@0: * sl@0: * Format version: 6.2 sl@0: * sl@0: * struct UConverterStaticData -- struct containing the converter name, IBM CCSID, sl@0: * min/max bytes per character, etc. sl@0: * see ucnv_bld.h sl@0: * sl@0: * -------------------- sl@0: * sl@0: * The static data is followed by conversionType-specific data structures. sl@0: * At the moment, there are only variations of MBCS converters. They all have sl@0: * the same toUnicode structures, while the fromUnicode structures for SBCS sl@0: * differ from those for other MBCS-style converters. sl@0: * sl@0: * _MBCSHeader.version 4.2 adds an optional conversion extension data structure. sl@0: * If it is present, then an ICU version reading header versions 4.0 or 4.1 sl@0: * will be able to use the base table and ignore the extension. sl@0: * sl@0: * The unicodeMask in the static data is part of the base table data structure. sl@0: * Especially, the UCNV_HAS_SUPPLEMENTARY flag determines the length of the sl@0: * fromUnicode stage 1 array. sl@0: * The static data unicodeMask refers only to the base table's properties if sl@0: * a base table is included. sl@0: * In an extension-only file, the static data unicodeMask is 0. sl@0: * The extension data indexes have a separate field with the unicodeMask flags. sl@0: * sl@0: * MBCS-style data structure following the static data. sl@0: * Offsets are counted in bytes from the beginning of the MBCS header structure. sl@0: * Details about usage in comments in ucnvmbcs.c. sl@0: * sl@0: * struct _MBCSHeader (see the definition in this header file below) sl@0: * contains 32-bit fields as follows: sl@0: * 8 values: sl@0: * 0 uint8_t[4] MBCS version in UVersionInfo format (currently 4.2.0.0) sl@0: * 1 uint32_t countStates sl@0: * 2 uint32_t countToUFallbacks sl@0: * 3 uint32_t offsetToUCodeUnits sl@0: * 4 uint32_t offsetFromUTable sl@0: * 5 uint32_t offsetFromUBytes sl@0: * 6 uint32_t flags, bits: sl@0: * 31.. 8 offsetExtension -- _MBCSHeader.version 4.2 (ICU 2.8) and higher sl@0: * 0 for older versions and if sl@0: * there is not extension structure sl@0: * 7.. 0 outputType sl@0: * 7 uint32_t fromUBytesLength -- _MBCSHeader.version 4.1 (ICU 2.4) and higher sl@0: * counts bytes in fromUBytes[] sl@0: * sl@0: * if(outputType==MBCS_OUTPUT_EXT_ONLY) { sl@0: * -- base table name for extension-only table sl@0: * char baseTableName[variable]; -- with NUL plus padding for 4-alignment sl@0: * sl@0: * -- all _MBCSHeader fields except for version and flags are 0 sl@0: * } else { sl@0: * -- normal base table with optional extension sl@0: * sl@0: * int32_t stateTable[countStates][256]; sl@0: * sl@0: * struct _MBCSToUFallback { (fallbacks are sorted by offset) sl@0: * uint32_t offset; sl@0: * UChar32 codePoint; sl@0: * } toUFallbacks[countToUFallbacks]; sl@0: * sl@0: * uint16_t unicodeCodeUnits[(offsetFromUTable-offsetToUCodeUnits)/2]; sl@0: * (padded to an even number of units) sl@0: * sl@0: * -- stage 1 tables sl@0: * if(staticData.unicodeMask&UCNV_HAS_SUPPLEMENTARY) { sl@0: * -- stage 1 table for all of Unicode sl@0: * uint16_t fromUTable[0x440]; (32-bit-aligned) sl@0: * } else { sl@0: * -- BMP-only tables have a smaller stage 1 table sl@0: * uint16_t fromUTable[0x40]; (32-bit-aligned) sl@0: * } sl@0: * sl@0: * -- stage 2 tables sl@0: * length determined by top of stage 1 and bottom of stage 3 tables sl@0: * if(outputType==MBCS_OUTPUT_1) { sl@0: * -- SBCS: pure indexes sl@0: * uint16_t stage 2 indexes[?]; sl@0: * } else { sl@0: * -- DBCS, MBCS, EBCDIC_STATEFUL, ...: roundtrip flags and indexes sl@0: * uint32_t stage 2 flags and indexes[?]; sl@0: * } sl@0: * sl@0: * -- stage 3 tables with byte results sl@0: * if(outputType==MBCS_OUTPUT_1) { sl@0: * -- SBCS: each 16-bit result contains flags and the result byte, see ucnvmbcs.c sl@0: * uint16_t fromUBytes[fromUBytesLength/2]; sl@0: * } else { sl@0: * -- DBCS, MBCS, EBCDIC_STATEFUL, ... 2/3/4 bytes result, see ucnvmbcs.c sl@0: * uint8_t fromUBytes[fromUBytesLength]; or sl@0: * uint16_t fromUBytes[fromUBytesLength/2]; or sl@0: * uint32_t fromUBytes[fromUBytesLength/4]; sl@0: * } sl@0: * } sl@0: * sl@0: * -- extension table, details see ucnv_ext.h sl@0: * int32_t indexes[>=32]; ... sl@0: */ sl@0: sl@0: /* MBCS converter data and state -------------------------------------------- */ sl@0: sl@0: enum { sl@0: MBCS_MAX_STATE_COUNT=128 sl@0: }; sl@0: sl@0: /** sl@0: * MBCS action codes for conversions to Unicode. sl@0: * These values are in bits 23..20 of the state table entries. sl@0: */ sl@0: enum { sl@0: MBCS_STATE_VALID_DIRECT_16, sl@0: MBCS_STATE_VALID_DIRECT_20, sl@0: sl@0: MBCS_STATE_FALLBACK_DIRECT_16, sl@0: MBCS_STATE_FALLBACK_DIRECT_20, sl@0: sl@0: MBCS_STATE_VALID_16, sl@0: MBCS_STATE_VALID_16_PAIR, sl@0: sl@0: MBCS_STATE_UNASSIGNED, sl@0: MBCS_STATE_ILLEGAL, sl@0: sl@0: MBCS_STATE_CHANGE_ONLY sl@0: }; sl@0: sl@0: /* Macros for state table entries */ sl@0: #define MBCS_ENTRY_TRANSITION(state, offset) (int32_t)(((int32_t)(state)<<24L)|(offset)) sl@0: #define MBCS_ENTRY_TRANSITION_SET_OFFSET(entry, offset) (int32_t)(((entry)&0xff000000)|(offset)) sl@0: #define MBCS_ENTRY_TRANSITION_ADD_OFFSET(entry, offset) (int32_t)((entry)+(offset)) sl@0: sl@0: #define MBCS_ENTRY_FINAL(state, action, value) (int32_t)(0x80000000|((int32_t)(state)<<24L)|((action)<<20L)|(value)) sl@0: #define MBCS_ENTRY_SET_FINAL(entry) (int32_t)((entry)|0x80000000) sl@0: #define MBCS_ENTRY_FINAL_SET_ACTION(entry, action) (int32_t)(((entry)&0xff0fffff)|((int32_t)(action)<<20L)) sl@0: #define MBCS_ENTRY_FINAL_SET_VALUE(entry, value) (int32_t)(((entry)&0xfff00000)|(value)) sl@0: #define MBCS_ENTRY_FINAL_SET_ACTION_VALUE(entry, action, value) (int32_t)(((entry)&0xff000000)|((int32_t)(action)<<20L)|(value)) sl@0: sl@0: #define MBCS_ENTRY_SET_STATE(entry, state) (int32_t)(((entry)&0x80ffffff)|((int32_t)(state)<<24L)) sl@0: sl@0: #define MBCS_ENTRY_STATE(entry) (((entry)>>24)&0x7f) sl@0: sl@0: #define MBCS_ENTRY_IS_TRANSITION(entry) ((entry)>=0) sl@0: #define MBCS_ENTRY_IS_FINAL(entry) ((entry)<0) sl@0: sl@0: #define MBCS_ENTRY_TRANSITION_STATE(entry) ((entry)>>24) sl@0: #define MBCS_ENTRY_TRANSITION_OFFSET(entry) ((entry)&0xffffff) sl@0: sl@0: #define MBCS_ENTRY_FINAL_STATE(entry) (((entry)>>24)&0x7f) sl@0: #define MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry) ((entry)<(int32_t)0x80100000) sl@0: #define MBCS_ENTRY_FINAL_ACTION(entry) (((entry)>>20)&0xf) sl@0: #define MBCS_ENTRY_FINAL_VALUE(entry) ((entry)&0xfffff) sl@0: #define MBCS_ENTRY_FINAL_VALUE_16(entry) (uint16_t)(entry) sl@0: sl@0: /* single-byte fromUnicode: get the 16-bit result word */ sl@0: #define MBCS_SINGLE_RESULT_FROM_U(table, results, c) (results)[ (table)[ (table)[(c)>>10] +(((c)>>4)&0x3f) ] +((c)&0xf) ] sl@0: sl@0: /* multi-byte fromUnicode: get the 32-bit stage 2 entry */ sl@0: #define MBCS_STAGE_2_FROM_U(table, c) ((const uint32_t *)(table))[ (table)[(c)>>10] +(((c)>>4)&0x3f) ] sl@0: #define MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) ( ((stage2Entry) & ((uint32_t)1<< (16+((c)&0xf)) )) !=0) sl@0: sl@0: #define MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c) ((uint16_t *)(bytes))[16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf)] sl@0: #define MBCS_VALUE_4_FROM_STAGE_2(bytes, stage2Entry, c) ((uint32_t *)(bytes))[16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf)] sl@0: sl@0: #define MBCS_POINTER_3_FROM_STAGE_2(bytes, stage2Entry, c) ((bytes)+(16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf))*3) sl@0: sl@0: sl@0: /** sl@0: * MBCS output types for conversions from Unicode. sl@0: * These per-converter types determine the storage method in stage 3 of the lookup table, sl@0: * mostly how many bytes are stored per entry. sl@0: */ sl@0: enum { sl@0: MBCS_OUTPUT_1, /* 0 */ sl@0: MBCS_OUTPUT_2, /* 1 */ sl@0: MBCS_OUTPUT_3, /* 2 */ sl@0: MBCS_OUTPUT_4, /* 3 */ sl@0: sl@0: MBCS_OUTPUT_3_EUC=8, /* 8 */ sl@0: MBCS_OUTPUT_4_EUC, /* 9 */ sl@0: sl@0: MBCS_OUTPUT_2_SISO=12, /* c */ sl@0: MBCS_OUTPUT_2_HZ, /* d */ sl@0: sl@0: MBCS_OUTPUT_EXT_ONLY, /* e */ sl@0: sl@0: MBCS_OUTPUT_COUNT, sl@0: sl@0: MBCS_OUTPUT_DBCS_ONLY=0xdb /* runtime-only type for DBCS-only handling of SISO tables */ sl@0: }; sl@0: sl@0: /** sl@0: * Fallbacks to Unicode are stored outside the normal state table and code point structures sl@0: * in a vector of items of this type. They are sorted by offset. sl@0: */ sl@0: typedef struct { sl@0: uint32_t offset; sl@0: UChar32 codePoint; sl@0: } _MBCSToUFallback; sl@0: sl@0: /** sl@0: * This is the MBCS part of the UConverterTable union (a runtime data structure). sl@0: * It keeps all the per-converter data and points into the loaded mapping tables. sl@0: */ sl@0: typedef struct UConverterMBCSTable { sl@0: /* toUnicode */ sl@0: uint8_t countStates, dbcsOnlyState, stateTableOwned; sl@0: uint32_t countToUFallbacks; sl@0: sl@0: const int32_t (*stateTable)/*[countStates]*/[256]; sl@0: int32_t (*swapLFNLStateTable)/*[countStates]*/[256]; /* for swaplfnl */ sl@0: const uint16_t *unicodeCodeUnits/*[countUnicodeResults]*/; sl@0: const _MBCSToUFallback *toUFallbacks; sl@0: sl@0: /* fromUnicode */ sl@0: const uint16_t *fromUnicodeTable; sl@0: const uint8_t *fromUnicodeBytes; sl@0: uint8_t *swapLFNLFromUnicodeBytes; /* for swaplfnl */ sl@0: uint32_t fromUBytesLength; sl@0: uint8_t outputType, unicodeMask; sl@0: sl@0: /* converter name for swaplfnl */ sl@0: char *swapLFNLName; sl@0: sl@0: /* extension data */ sl@0: struct UConverterSharedData *baseSharedData; sl@0: const int32_t *extIndexes; sl@0: } UConverterMBCSTable; sl@0: sl@0: /** sl@0: * MBCS data header. See data format description above. sl@0: */ sl@0: typedef struct { sl@0: UVersionInfo version; sl@0: uint32_t countStates, sl@0: countToUFallbacks, sl@0: offsetToUCodeUnits, sl@0: offsetFromUTable, sl@0: offsetFromUBytes, sl@0: flags, sl@0: fromUBytesLength; sl@0: } _MBCSHeader; sl@0: sl@0: /* sl@0: * This is a simple version of _MBCSGetNextUChar() that is used sl@0: * by other converter implementations. sl@0: * It only returns an "assigned" result if it consumes the entire input. sl@0: * It does not use state from the converter, nor error codes. sl@0: * It does not handle the EBCDIC swaplfnl option (set in UConverter). sl@0: * It handles conversion extensions but not GB 18030. sl@0: * sl@0: * Return value: sl@0: * U+fffe unassigned sl@0: * U+ffff illegal sl@0: * otherwise the Unicode code point sl@0: */ sl@0: U_CFUNC UChar32 sl@0: ucnv_MBCSSimpleGetNextUChar(UConverterSharedData *sharedData, sl@0: const char *source, int32_t length, sl@0: UBool useFallback); sl@0: sl@0: /** sl@0: * This version of _MBCSSimpleGetNextUChar() is optimized for single-byte, single-state codepages. sl@0: * It does not handle the EBCDIC swaplfnl option (set in UConverter). sl@0: * It does not handle conversion extensions (_extToU()). sl@0: */ sl@0: U_CFUNC UChar32 sl@0: ucnv_MBCSSingleSimpleGetNextUChar(UConverterSharedData *sharedData, sl@0: uint8_t b, UBool useFallback); sl@0: sl@0: /** sl@0: * This macro version of _MBCSSingleSimpleGetNextUChar() gets a code point from a byte. sl@0: * It works for single-byte, single-state codepages that only map sl@0: * to and from BMP code points, and it always sl@0: * returns fallback values. sl@0: */ sl@0: #define _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP(sharedData, b) \ sl@0: (UChar)MBCS_ENTRY_FINAL_VALUE_16((sharedData)->mbcs.stateTable[0][(uint8_t)(b)]) sl@0: sl@0: /** sl@0: * This is an internal function that allows other converter implementations sl@0: * to check whether a byte is a lead byte. sl@0: */ sl@0: U_CFUNC UBool sl@0: ucnv_MBCSIsLeadByte(UConverterSharedData *sharedData, char byte); sl@0: sl@0: /** This is a macro version of _MBCSIsLeadByte(). */ sl@0: #define _MBCS_IS_LEAD_BYTE(sharedData, byte) \ sl@0: (UBool)MBCS_ENTRY_IS_TRANSITION((sharedData)->mbcs.stateTable[0][(uint8_t)(byte)]) sl@0: sl@0: /* sl@0: * This is another simple conversion function for internal use by other sl@0: * conversion implementations. sl@0: * It does not use the converter state nor call callbacks. sl@0: * It does not handle the EBCDIC swaplfnl option (set in UConverter). sl@0: * It handles conversion extensions but not GB 18030. sl@0: * sl@0: * It converts one single Unicode code point into codepage bytes, encoded sl@0: * as one 32-bit value. The function returns the number of bytes in *pValue: sl@0: * 1..4 the number of bytes in *pValue sl@0: * 0 unassigned (*pValue undefined) sl@0: * -1 illegal (currently not used, *pValue undefined) sl@0: * sl@0: * *pValue will contain the resulting bytes with the last byte in bits 7..0, sl@0: * the second to last byte in bits 15..8, etc. sl@0: * Currently, the function assumes but does not check that 0<=c<=0x10ffff. sl@0: */ sl@0: U_CFUNC int32_t sl@0: ucnv_MBCSFromUChar32(UConverterSharedData *sharedData, sl@0: UChar32 c, uint32_t *pValue, sl@0: UBool useFallback); sl@0: sl@0: /** sl@0: * This version of _MBCSFromUChar32() is optimized for single-byte codepages. sl@0: * It does not handle the EBCDIC swaplfnl option (set in UConverter). sl@0: * sl@0: * It returns the codepage byte for the code point, or -1 if it is unassigned. sl@0: */ sl@0: U_CFUNC int32_t sl@0: ucnv_MBCSSingleFromUChar32(UConverterSharedData *sharedData, sl@0: UChar32 c, sl@0: UBool useFallback); sl@0: sl@0: /** sl@0: * SBCS, DBCS, and EBCDIC_STATEFUL are replaced by MBCS, but sl@0: * we cheat a little about the type, returning the old types if appropriate. sl@0: */ sl@0: U_CFUNC UConverterType sl@0: ucnv_MBCSGetType(const UConverter* converter); sl@0: sl@0: U_CFUNC void sl@0: ucnv_MBCSFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs, sl@0: UErrorCode *pErrorCode); sl@0: U_CFUNC void sl@0: ucnv_MBCSToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs, sl@0: UErrorCode *pErrorCode); sl@0: sl@0: /* sl@0: * Internal function returning a UnicodeSet for toUnicode() conversion. sl@0: * Currently only used for ISO-2022-CN, and only handles roundtrip mappings. sl@0: * In the future, if we add support for reverse-fallback sets, this function sl@0: * needs to be updated, and called for each initial state. sl@0: * Does not currently handle extensions. sl@0: * Does not empty the set first. sl@0: */ sl@0: U_CFUNC void sl@0: ucnv_MBCSGetUnicodeSetForBytes(const UConverterSharedData *sharedData, sl@0: const USetAdder *sa, sl@0: UConverterUnicodeSet which, sl@0: uint8_t state, int32_t lowByte, int32_t highByte, sl@0: UErrorCode *pErrorCode); sl@0: sl@0: /* sl@0: * Internal function returning a UnicodeSet for toUnicode() conversion. sl@0: * Currently only used for ISO-2022-CN, and only handles roundtrip mappings. sl@0: * In the future, if we add support for fallback sets, this function sl@0: * needs to be updated. sl@0: * Handles extensions. sl@0: * Does not empty the set first. sl@0: */ sl@0: U_CFUNC void sl@0: ucnv_MBCSGetUnicodeSetForUnicode(const UConverterSharedData *sharedData, sl@0: const USetAdder *sa, sl@0: UConverterUnicodeSet which, sl@0: UErrorCode *pErrorCode); sl@0: sl@0: #endif sl@0: sl@0: #endif