1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/os/textandloc/fontservices/textshaperplugin/IcuSource/common/ucnvmbcs.h Fri Jun 15 03:10:57 2012 +0200
1.3 @@ -0,0 +1,397 @@
1.4 +/*
1.5 +******************************************************************************
1.6 +*
1.7 +* Copyright (C) 2000-2004, International Business Machines
1.8 +* Corporation and others. All Rights Reserved.
1.9 +*
1.10 +******************************************************************************
1.11 +* file name: ucnvmbcs.h
1.12 +* encoding: US-ASCII
1.13 +* tab size: 8 (not used)
1.14 +* indentation:4
1.15 +*
1.16 +* created on: 2000jul07
1.17 +* created by: Markus W. Scherer
1.18 +*/
1.19 +
1.20 +#ifndef __UCNVMBCS_H__
1.21 +#define __UCNVMBCS_H__
1.22 +
1.23 +#include "unicode/utypes.h"
1.24 +
1.25 +#if !UCONFIG_NO_CONVERSION
1.26 +
1.27 +#include "unicode/ucnv.h"
1.28 +#include "ucnv_cnv.h"
1.29 +
1.30 +/**
1.31 + * ICU conversion (.cnv) data file structure, following the usual UDataInfo
1.32 + * header.
1.33 + *
1.34 + * Format version: 6.2
1.35 + *
1.36 + * struct UConverterStaticData -- struct containing the converter name, IBM CCSID,
1.37 + * min/max bytes per character, etc.
1.38 + * see ucnv_bld.h
1.39 + *
1.40 + * --------------------
1.41 + *
1.42 + * The static data is followed by conversionType-specific data structures.
1.43 + * At the moment, there are only variations of MBCS converters. They all have
1.44 + * the same toUnicode structures, while the fromUnicode structures for SBCS
1.45 + * differ from those for other MBCS-style converters.
1.46 + *
1.47 + * _MBCSHeader.version 4.2 adds an optional conversion extension data structure.
1.48 + * If it is present, then an ICU version reading header versions 4.0 or 4.1
1.49 + * will be able to use the base table and ignore the extension.
1.50 + *
1.51 + * The unicodeMask in the static data is part of the base table data structure.
1.52 + * Especially, the UCNV_HAS_SUPPLEMENTARY flag determines the length of the
1.53 + * fromUnicode stage 1 array.
1.54 + * The static data unicodeMask refers only to the base table's properties if
1.55 + * a base table is included.
1.56 + * In an extension-only file, the static data unicodeMask is 0.
1.57 + * The extension data indexes have a separate field with the unicodeMask flags.
1.58 + *
1.59 + * MBCS-style data structure following the static data.
1.60 + * Offsets are counted in bytes from the beginning of the MBCS header structure.
1.61 + * Details about usage in comments in ucnvmbcs.c.
1.62 + *
1.63 + * struct _MBCSHeader (see the definition in this header file below)
1.64 + * contains 32-bit fields as follows:
1.65 + * 8 values:
1.66 + * 0 uint8_t[4] MBCS version in UVersionInfo format (currently 4.2.0.0)
1.67 + * 1 uint32_t countStates
1.68 + * 2 uint32_t countToUFallbacks
1.69 + * 3 uint32_t offsetToUCodeUnits
1.70 + * 4 uint32_t offsetFromUTable
1.71 + * 5 uint32_t offsetFromUBytes
1.72 + * 6 uint32_t flags, bits:
1.73 + * 31.. 8 offsetExtension -- _MBCSHeader.version 4.2 (ICU 2.8) and higher
1.74 + * 0 for older versions and if
1.75 + * there is not extension structure
1.76 + * 7.. 0 outputType
1.77 + * 7 uint32_t fromUBytesLength -- _MBCSHeader.version 4.1 (ICU 2.4) and higher
1.78 + * counts bytes in fromUBytes[]
1.79 + *
1.80 + * if(outputType==MBCS_OUTPUT_EXT_ONLY) {
1.81 + * -- base table name for extension-only table
1.82 + * char baseTableName[variable]; -- with NUL plus padding for 4-alignment
1.83 + *
1.84 + * -- all _MBCSHeader fields except for version and flags are 0
1.85 + * } else {
1.86 + * -- normal base table with optional extension
1.87 + *
1.88 + * int32_t stateTable[countStates][256];
1.89 + *
1.90 + * struct _MBCSToUFallback { (fallbacks are sorted by offset)
1.91 + * uint32_t offset;
1.92 + * UChar32 codePoint;
1.93 + * } toUFallbacks[countToUFallbacks];
1.94 + *
1.95 + * uint16_t unicodeCodeUnits[(offsetFromUTable-offsetToUCodeUnits)/2];
1.96 + * (padded to an even number of units)
1.97 + *
1.98 + * -- stage 1 tables
1.99 + * if(staticData.unicodeMask&UCNV_HAS_SUPPLEMENTARY) {
1.100 + * -- stage 1 table for all of Unicode
1.101 + * uint16_t fromUTable[0x440]; (32-bit-aligned)
1.102 + * } else {
1.103 + * -- BMP-only tables have a smaller stage 1 table
1.104 + * uint16_t fromUTable[0x40]; (32-bit-aligned)
1.105 + * }
1.106 + *
1.107 + * -- stage 2 tables
1.108 + * length determined by top of stage 1 and bottom of stage 3 tables
1.109 + * if(outputType==MBCS_OUTPUT_1) {
1.110 + * -- SBCS: pure indexes
1.111 + * uint16_t stage 2 indexes[?];
1.112 + * } else {
1.113 + * -- DBCS, MBCS, EBCDIC_STATEFUL, ...: roundtrip flags and indexes
1.114 + * uint32_t stage 2 flags and indexes[?];
1.115 + * }
1.116 + *
1.117 + * -- stage 3 tables with byte results
1.118 + * if(outputType==MBCS_OUTPUT_1) {
1.119 + * -- SBCS: each 16-bit result contains flags and the result byte, see ucnvmbcs.c
1.120 + * uint16_t fromUBytes[fromUBytesLength/2];
1.121 + * } else {
1.122 + * -- DBCS, MBCS, EBCDIC_STATEFUL, ... 2/3/4 bytes result, see ucnvmbcs.c
1.123 + * uint8_t fromUBytes[fromUBytesLength]; or
1.124 + * uint16_t fromUBytes[fromUBytesLength/2]; or
1.125 + * uint32_t fromUBytes[fromUBytesLength/4];
1.126 + * }
1.127 + * }
1.128 + *
1.129 + * -- extension table, details see ucnv_ext.h
1.130 + * int32_t indexes[>=32]; ...
1.131 + */
1.132 +
1.133 +/* MBCS converter data and state -------------------------------------------- */
1.134 +
1.135 +enum {
1.136 + MBCS_MAX_STATE_COUNT=128
1.137 +};
1.138 +
1.139 +/**
1.140 + * MBCS action codes for conversions to Unicode.
1.141 + * These values are in bits 23..20 of the state table entries.
1.142 + */
1.143 +enum {
1.144 + MBCS_STATE_VALID_DIRECT_16,
1.145 + MBCS_STATE_VALID_DIRECT_20,
1.146 +
1.147 + MBCS_STATE_FALLBACK_DIRECT_16,
1.148 + MBCS_STATE_FALLBACK_DIRECT_20,
1.149 +
1.150 + MBCS_STATE_VALID_16,
1.151 + MBCS_STATE_VALID_16_PAIR,
1.152 +
1.153 + MBCS_STATE_UNASSIGNED,
1.154 + MBCS_STATE_ILLEGAL,
1.155 +
1.156 + MBCS_STATE_CHANGE_ONLY
1.157 +};
1.158 +
1.159 +/* Macros for state table entries */
1.160 +#define MBCS_ENTRY_TRANSITION(state, offset) (int32_t)(((int32_t)(state)<<24L)|(offset))
1.161 +#define MBCS_ENTRY_TRANSITION_SET_OFFSET(entry, offset) (int32_t)(((entry)&0xff000000)|(offset))
1.162 +#define MBCS_ENTRY_TRANSITION_ADD_OFFSET(entry, offset) (int32_t)((entry)+(offset))
1.163 +
1.164 +#define MBCS_ENTRY_FINAL(state, action, value) (int32_t)(0x80000000|((int32_t)(state)<<24L)|((action)<<20L)|(value))
1.165 +#define MBCS_ENTRY_SET_FINAL(entry) (int32_t)((entry)|0x80000000)
1.166 +#define MBCS_ENTRY_FINAL_SET_ACTION(entry, action) (int32_t)(((entry)&0xff0fffff)|((int32_t)(action)<<20L))
1.167 +#define MBCS_ENTRY_FINAL_SET_VALUE(entry, value) (int32_t)(((entry)&0xfff00000)|(value))
1.168 +#define MBCS_ENTRY_FINAL_SET_ACTION_VALUE(entry, action, value) (int32_t)(((entry)&0xff000000)|((int32_t)(action)<<20L)|(value))
1.169 +
1.170 +#define MBCS_ENTRY_SET_STATE(entry, state) (int32_t)(((entry)&0x80ffffff)|((int32_t)(state)<<24L))
1.171 +
1.172 +#define MBCS_ENTRY_STATE(entry) (((entry)>>24)&0x7f)
1.173 +
1.174 +#define MBCS_ENTRY_IS_TRANSITION(entry) ((entry)>=0)
1.175 +#define MBCS_ENTRY_IS_FINAL(entry) ((entry)<0)
1.176 +
1.177 +#define MBCS_ENTRY_TRANSITION_STATE(entry) ((entry)>>24)
1.178 +#define MBCS_ENTRY_TRANSITION_OFFSET(entry) ((entry)&0xffffff)
1.179 +
1.180 +#define MBCS_ENTRY_FINAL_STATE(entry) (((entry)>>24)&0x7f)
1.181 +#define MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry) ((entry)<(int32_t)0x80100000)
1.182 +#define MBCS_ENTRY_FINAL_ACTION(entry) (((entry)>>20)&0xf)
1.183 +#define MBCS_ENTRY_FINAL_VALUE(entry) ((entry)&0xfffff)
1.184 +#define MBCS_ENTRY_FINAL_VALUE_16(entry) (uint16_t)(entry)
1.185 +
1.186 +/* single-byte fromUnicode: get the 16-bit result word */
1.187 +#define MBCS_SINGLE_RESULT_FROM_U(table, results, c) (results)[ (table)[ (table)[(c)>>10] +(((c)>>4)&0x3f) ] +((c)&0xf) ]
1.188 +
1.189 +/* multi-byte fromUnicode: get the 32-bit stage 2 entry */
1.190 +#define MBCS_STAGE_2_FROM_U(table, c) ((const uint32_t *)(table))[ (table)[(c)>>10] +(((c)>>4)&0x3f) ]
1.191 +#define MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) ( ((stage2Entry) & ((uint32_t)1<< (16+((c)&0xf)) )) !=0)
1.192 +
1.193 +#define MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c) ((uint16_t *)(bytes))[16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf)]
1.194 +#define MBCS_VALUE_4_FROM_STAGE_2(bytes, stage2Entry, c) ((uint32_t *)(bytes))[16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf)]
1.195 +
1.196 +#define MBCS_POINTER_3_FROM_STAGE_2(bytes, stage2Entry, c) ((bytes)+(16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf))*3)
1.197 +
1.198 +
1.199 +/**
1.200 + * MBCS output types for conversions from Unicode.
1.201 + * These per-converter types determine the storage method in stage 3 of the lookup table,
1.202 + * mostly how many bytes are stored per entry.
1.203 + */
1.204 +enum {
1.205 + MBCS_OUTPUT_1, /* 0 */
1.206 + MBCS_OUTPUT_2, /* 1 */
1.207 + MBCS_OUTPUT_3, /* 2 */
1.208 + MBCS_OUTPUT_4, /* 3 */
1.209 +
1.210 + MBCS_OUTPUT_3_EUC=8, /* 8 */
1.211 + MBCS_OUTPUT_4_EUC, /* 9 */
1.212 +
1.213 + MBCS_OUTPUT_2_SISO=12, /* c */
1.214 + MBCS_OUTPUT_2_HZ, /* d */
1.215 +
1.216 + MBCS_OUTPUT_EXT_ONLY, /* e */
1.217 +
1.218 + MBCS_OUTPUT_COUNT,
1.219 +
1.220 + MBCS_OUTPUT_DBCS_ONLY=0xdb /* runtime-only type for DBCS-only handling of SISO tables */
1.221 +};
1.222 +
1.223 +/**
1.224 + * Fallbacks to Unicode are stored outside the normal state table and code point structures
1.225 + * in a vector of items of this type. They are sorted by offset.
1.226 + */
1.227 +typedef struct {
1.228 + uint32_t offset;
1.229 + UChar32 codePoint;
1.230 +} _MBCSToUFallback;
1.231 +
1.232 +/**
1.233 + * This is the MBCS part of the UConverterTable union (a runtime data structure).
1.234 + * It keeps all the per-converter data and points into the loaded mapping tables.
1.235 + */
1.236 +typedef struct UConverterMBCSTable {
1.237 + /* toUnicode */
1.238 + uint8_t countStates, dbcsOnlyState, stateTableOwned;
1.239 + uint32_t countToUFallbacks;
1.240 +
1.241 + const int32_t (*stateTable)/*[countStates]*/[256];
1.242 + int32_t (*swapLFNLStateTable)/*[countStates]*/[256]; /* for swaplfnl */
1.243 + const uint16_t *unicodeCodeUnits/*[countUnicodeResults]*/;
1.244 + const _MBCSToUFallback *toUFallbacks;
1.245 +
1.246 + /* fromUnicode */
1.247 + const uint16_t *fromUnicodeTable;
1.248 + const uint8_t *fromUnicodeBytes;
1.249 + uint8_t *swapLFNLFromUnicodeBytes; /* for swaplfnl */
1.250 + uint32_t fromUBytesLength;
1.251 + uint8_t outputType, unicodeMask;
1.252 +
1.253 + /* converter name for swaplfnl */
1.254 + char *swapLFNLName;
1.255 +
1.256 + /* extension data */
1.257 + struct UConverterSharedData *baseSharedData;
1.258 + const int32_t *extIndexes;
1.259 +} UConverterMBCSTable;
1.260 +
1.261 +/**
1.262 + * MBCS data header. See data format description above.
1.263 + */
1.264 +typedef struct {
1.265 + UVersionInfo version;
1.266 + uint32_t countStates,
1.267 + countToUFallbacks,
1.268 + offsetToUCodeUnits,
1.269 + offsetFromUTable,
1.270 + offsetFromUBytes,
1.271 + flags,
1.272 + fromUBytesLength;
1.273 +} _MBCSHeader;
1.274 +
1.275 +/*
1.276 + * This is a simple version of _MBCSGetNextUChar() that is used
1.277 + * by other converter implementations.
1.278 + * It only returns an "assigned" result if it consumes the entire input.
1.279 + * It does not use state from the converter, nor error codes.
1.280 + * It does not handle the EBCDIC swaplfnl option (set in UConverter).
1.281 + * It handles conversion extensions but not GB 18030.
1.282 + *
1.283 + * Return value:
1.284 + * U+fffe unassigned
1.285 + * U+ffff illegal
1.286 + * otherwise the Unicode code point
1.287 + */
1.288 +U_CFUNC UChar32
1.289 +ucnv_MBCSSimpleGetNextUChar(UConverterSharedData *sharedData,
1.290 + const char *source, int32_t length,
1.291 + UBool useFallback);
1.292 +
1.293 +/**
1.294 + * This version of _MBCSSimpleGetNextUChar() is optimized for single-byte, single-state codepages.
1.295 + * It does not handle the EBCDIC swaplfnl option (set in UConverter).
1.296 + * It does not handle conversion extensions (_extToU()).
1.297 + */
1.298 +U_CFUNC UChar32
1.299 +ucnv_MBCSSingleSimpleGetNextUChar(UConverterSharedData *sharedData,
1.300 + uint8_t b, UBool useFallback);
1.301 +
1.302 +/**
1.303 + * This macro version of _MBCSSingleSimpleGetNextUChar() gets a code point from a byte.
1.304 + * It works for single-byte, single-state codepages that only map
1.305 + * to and from BMP code points, and it always
1.306 + * returns fallback values.
1.307 + */
1.308 +#define _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP(sharedData, b) \
1.309 + (UChar)MBCS_ENTRY_FINAL_VALUE_16((sharedData)->mbcs.stateTable[0][(uint8_t)(b)])
1.310 +
1.311 +/**
1.312 + * This is an internal function that allows other converter implementations
1.313 + * to check whether a byte is a lead byte.
1.314 + */
1.315 +U_CFUNC UBool
1.316 +ucnv_MBCSIsLeadByte(UConverterSharedData *sharedData, char byte);
1.317 +
1.318 +/** This is a macro version of _MBCSIsLeadByte(). */
1.319 +#define _MBCS_IS_LEAD_BYTE(sharedData, byte) \
1.320 + (UBool)MBCS_ENTRY_IS_TRANSITION((sharedData)->mbcs.stateTable[0][(uint8_t)(byte)])
1.321 +
1.322 +/*
1.323 + * This is another simple conversion function for internal use by other
1.324 + * conversion implementations.
1.325 + * It does not use the converter state nor call callbacks.
1.326 + * It does not handle the EBCDIC swaplfnl option (set in UConverter).
1.327 + * It handles conversion extensions but not GB 18030.
1.328 + *
1.329 + * It converts one single Unicode code point into codepage bytes, encoded
1.330 + * as one 32-bit value. The function returns the number of bytes in *pValue:
1.331 + * 1..4 the number of bytes in *pValue
1.332 + * 0 unassigned (*pValue undefined)
1.333 + * -1 illegal (currently not used, *pValue undefined)
1.334 + *
1.335 + * *pValue will contain the resulting bytes with the last byte in bits 7..0,
1.336 + * the second to last byte in bits 15..8, etc.
1.337 + * Currently, the function assumes but does not check that 0<=c<=0x10ffff.
1.338 + */
1.339 +U_CFUNC int32_t
1.340 +ucnv_MBCSFromUChar32(UConverterSharedData *sharedData,
1.341 + UChar32 c, uint32_t *pValue,
1.342 + UBool useFallback);
1.343 +
1.344 +/**
1.345 + * This version of _MBCSFromUChar32() is optimized for single-byte codepages.
1.346 + * It does not handle the EBCDIC swaplfnl option (set in UConverter).
1.347 + *
1.348 + * It returns the codepage byte for the code point, or -1 if it is unassigned.
1.349 + */
1.350 +U_CFUNC int32_t
1.351 +ucnv_MBCSSingleFromUChar32(UConverterSharedData *sharedData,
1.352 + UChar32 c,
1.353 + UBool useFallback);
1.354 +
1.355 +/**
1.356 + * SBCS, DBCS, and EBCDIC_STATEFUL are replaced by MBCS, but
1.357 + * we cheat a little about the type, returning the old types if appropriate.
1.358 + */
1.359 +U_CFUNC UConverterType
1.360 +ucnv_MBCSGetType(const UConverter* converter);
1.361 +
1.362 +U_CFUNC void
1.363 +ucnv_MBCSFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs,
1.364 + UErrorCode *pErrorCode);
1.365 +U_CFUNC void
1.366 +ucnv_MBCSToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs,
1.367 + UErrorCode *pErrorCode);
1.368 +
1.369 +/*
1.370 + * Internal function returning a UnicodeSet for toUnicode() conversion.
1.371 + * Currently only used for ISO-2022-CN, and only handles roundtrip mappings.
1.372 + * In the future, if we add support for reverse-fallback sets, this function
1.373 + * needs to be updated, and called for each initial state.
1.374 + * Does not currently handle extensions.
1.375 + * Does not empty the set first.
1.376 + */
1.377 +U_CFUNC void
1.378 +ucnv_MBCSGetUnicodeSetForBytes(const UConverterSharedData *sharedData,
1.379 + const USetAdder *sa,
1.380 + UConverterUnicodeSet which,
1.381 + uint8_t state, int32_t lowByte, int32_t highByte,
1.382 + UErrorCode *pErrorCode);
1.383 +
1.384 +/*
1.385 + * Internal function returning a UnicodeSet for toUnicode() conversion.
1.386 + * Currently only used for ISO-2022-CN, and only handles roundtrip mappings.
1.387 + * In the future, if we add support for fallback sets, this function
1.388 + * needs to be updated.
1.389 + * Handles extensions.
1.390 + * Does not empty the set first.
1.391 + */
1.392 +U_CFUNC void
1.393 +ucnv_MBCSGetUnicodeSetForUnicode(const UConverterSharedData *sharedData,
1.394 + const USetAdder *sa,
1.395 + UConverterUnicodeSet which,
1.396 + UErrorCode *pErrorCode);
1.397 +
1.398 +#endif
1.399 +
1.400 +#endif