sl@0: /* sl@0: ******************************************************************************* sl@0: * sl@0: * Copyright (C) 1999-2005, International Business Machines sl@0: * Corporation and others. All Rights Reserved. sl@0: * sl@0: ******************************************************************************* sl@0: * file name: utf8.h sl@0: * encoding: US-ASCII sl@0: * tab size: 8 (not used) sl@0: * indentation:4 sl@0: * sl@0: * created on: 1999sep13 sl@0: * created by: Markus W. Scherer sl@0: */ sl@0: sl@0: /** sl@0: * \file sl@0: * \brief C API: 8-bit Unicode handling macros sl@0: * sl@0: * This file defines macros to deal with 8-bit Unicode (UTF-8) code units (bytes) and strings. sl@0: * utf8.h is included by utf.h after unicode/umachine.h sl@0: * and some common definitions. sl@0: * sl@0: * For more information see utf.h and the ICU User Guide Strings chapter sl@0: * (http://icu.sourceforge.net/userguide/strings.html). sl@0: * sl@0: * Usage: sl@0: * ICU coding guidelines for if() statements should be followed when using these macros. sl@0: * Compound statements (curly braces {}) must be used for if-else-while... sl@0: * bodies and all macro statements should be terminated with semicolon. sl@0: */ sl@0: sl@0: #ifndef __UTF8_H__ sl@0: #define __UTF8_H__ sl@0: sl@0: /* utf.h must be included first. */ sl@0: #ifndef __UTF_H__ sl@0: # include "unicode/utf.h" sl@0: #endif sl@0: sl@0: /* internal definitions ----------------------------------------------------- */ sl@0: sl@0: /** sl@0: * \var utf8_countTrailBytes sl@0: * Internal array with numbers of trail bytes for any given byte used in sl@0: * lead byte position. sl@0: * @internal sl@0: */ sl@0: #ifdef U_UTF8_IMPL sl@0: U_INTERNAL const uint8_t sl@0: #elif defined(U_STATIC_IMPLEMENTATION) sl@0: U_CFUNC const uint8_t sl@0: #else sl@0: U_CFUNC U_IMPORT const uint8_t /* U_IMPORT2? */ /*U_IMPORT*/ sl@0: #endif sl@0: utf8_countTrailBytes[256]; sl@0: sl@0: /** sl@0: * Count the trail bytes for a UTF-8 lead byte. sl@0: * @internal sl@0: */ sl@0: #define U8_COUNT_TRAIL_BYTES(leadByte) (utf8_countTrailBytes[(uint8_t)leadByte]) sl@0: sl@0: /** sl@0: * Mask a UTF-8 lead byte, leave only the lower bits that form part of the code point value. sl@0: * @internal sl@0: */ sl@0: #define U8_MASK_LEAD_BYTE(leadByte, countTrailBytes) ((leadByte)&=(1<<(6-(countTrailBytes)))-1) sl@0: sl@0: /** sl@0: * Function for handling "next code point" with error-checking. sl@0: * @internal sl@0: */ sl@0: U_INTERNAL UChar32 U_EXPORT2 sl@0: utf8_nextCharSafeBody(const uint8_t *s, int32_t *pi, int32_t length, UChar32 c, UBool strict); sl@0: sl@0: /** sl@0: * Function for handling "append code point" with error-checking. sl@0: * @internal sl@0: */ sl@0: U_INTERNAL int32_t U_EXPORT2 sl@0: utf8_appendCharSafeBody(uint8_t *s, int32_t i, int32_t length, UChar32 c, UBool *pIsError); sl@0: sl@0: /** sl@0: * Function for handling "previous code point" with error-checking. sl@0: * @internal sl@0: */ sl@0: U_INTERNAL UChar32 U_EXPORT2 sl@0: utf8_prevCharSafeBody(const uint8_t *s, int32_t start, int32_t *pi, UChar32 c, UBool strict); sl@0: sl@0: /** sl@0: * Function for handling "skip backward one code point" with error-checking. sl@0: * @internal sl@0: */ sl@0: U_INTERNAL int32_t U_EXPORT2 sl@0: utf8_back1SafeBody(const uint8_t *s, int32_t start, int32_t i); sl@0: sl@0: /* single-code point definitions -------------------------------------------- */ sl@0: sl@0: /** sl@0: * Does this code unit (byte) encode a code point by itself (US-ASCII 0..0x7f)? sl@0: * @param c 8-bit code unit (byte) sl@0: * @return TRUE or FALSE sl@0: * @stable ICU 2.4 sl@0: */ sl@0: #define U8_IS_SINGLE(c) (((c)&0x80)==0) sl@0: sl@0: /** sl@0: * Is this code unit (byte) a UTF-8 lead byte? sl@0: * @param c 8-bit code unit (byte) sl@0: * @return TRUE or FALSE sl@0: * @stable ICU 2.4 sl@0: */ sl@0: #define U8_IS_LEAD(c) ((uint8_t)((c)-0xc0)<0x3e) sl@0: sl@0: /** sl@0: * Is this code unit (byte) a UTF-8 trail byte? sl@0: * @param c 8-bit code unit (byte) sl@0: * @return TRUE or FALSE sl@0: * @stable ICU 2.4 sl@0: */ sl@0: #define U8_IS_TRAIL(c) (((c)&0xc0)==0x80) sl@0: sl@0: /** sl@0: * How many code units (bytes) are used for the UTF-8 encoding sl@0: * of this Unicode code point? sl@0: * @param c 32-bit code point sl@0: * @return 1..4, or 0 if c is a surrogate or not a Unicode code point sl@0: * @stable ICU 2.4 sl@0: */ sl@0: #define U8_LENGTH(c) \ sl@0: ((uint32_t)(c)<=0x7f ? 1 : \ sl@0: ((uint32_t)(c)<=0x7ff ? 2 : \ sl@0: ((uint32_t)(c)<=0xd7ff ? 3 : \ sl@0: ((uint32_t)(c)<=0xdfff || (uint32_t)(c)>0x10ffff ? 0 : \ sl@0: ((uint32_t)(c)<=0xffff ? 3 : 4)\ sl@0: ) \ sl@0: ) \ sl@0: ) \ sl@0: ) sl@0: sl@0: /** sl@0: * The maximum number of UTF-8 code units (bytes) per Unicode code point (U+0000..U+10ffff). sl@0: * @return 4 sl@0: * @stable ICU 2.4 sl@0: */ sl@0: #define U8_MAX_LENGTH 4 sl@0: sl@0: /** sl@0: * Get a code point from a string at a random-access offset, sl@0: * without changing the offset. sl@0: * The offset may point to either the lead byte or one of the trail bytes sl@0: * for a code point, in which case the macro will read all of the bytes sl@0: * for the code point. sl@0: * The result is undefined if the offset points to an illegal UTF-8 sl@0: * byte sequence. sl@0: * Iteration through a string is more efficient with U8_NEXT_UNSAFE or U8_NEXT. sl@0: * sl@0: * @param s const uint8_t * string sl@0: * @param i string offset sl@0: * @param c output UChar32 variable sl@0: * @see U8_GET sl@0: * @stable ICU 2.4 sl@0: */ sl@0: #define U8_GET_UNSAFE(s, i, c) { \ sl@0: int32_t _u8_get_unsafe_index=(int32_t)(i); \ sl@0: U8_SET_CP_START_UNSAFE(s, _u8_get_unsafe_index); \ sl@0: U8_NEXT_UNSAFE(s, _u8_get_unsafe_index, c); \ sl@0: } sl@0: sl@0: /** sl@0: * Get a code point from a string at a random-access offset, sl@0: * without changing the offset. sl@0: * The offset may point to either the lead byte or one of the trail bytes sl@0: * for a code point, in which case the macro will read all of the bytes sl@0: * for the code point. sl@0: * If the offset points to an illegal UTF-8 byte sequence, then sl@0: * c is set to a negative value. sl@0: * Iteration through a string is more efficient with U8_NEXT_UNSAFE or U8_NEXT. sl@0: * sl@0: * @param s const uint8_t * string sl@0: * @param start starting string offset sl@0: * @param i string offset, start<=i=0x80) { \ sl@0: if(U8_IS_LEAD(c)) { \ sl@0: (c)=utf8_nextCharSafeBody((const uint8_t *)s, &(i), (int32_t)(length), c, -1); \ sl@0: } else { \ sl@0: (c)=U_SENTINEL; \ sl@0: } \ sl@0: } \ sl@0: } sl@0: sl@0: /** sl@0: * Append a code point to a string, overwriting 1 to 4 bytes. sl@0: * The offset points to the current end of the string contents sl@0: * and is advanced (post-increment). sl@0: * "Unsafe" macro, assumes a valid code point and sufficient space in the string. sl@0: * Otherwise, the result is undefined. sl@0: * sl@0: * @param s const uint8_t * string buffer sl@0: * @param i string offset sl@0: * @param c code point to append sl@0: * @see U8_APPEND sl@0: * @stable ICU 2.4 sl@0: */ sl@0: #define U8_APPEND_UNSAFE(s, i, c) { \ sl@0: if((uint32_t)(c)<=0x7f) { \ sl@0: (s)[(i)++]=(uint8_t)(c); \ sl@0: } else { \ sl@0: if((uint32_t)(c)<=0x7ff) { \ sl@0: (s)[(i)++]=(uint8_t)(((c)>>6)|0xc0); \ sl@0: } else { \ sl@0: if((uint32_t)(c)<=0xffff) { \ sl@0: (s)[(i)++]=(uint8_t)(((c)>>12)|0xe0); \ sl@0: } else { \ sl@0: (s)[(i)++]=(uint8_t)(((c)>>18)|0xf0); \ sl@0: (s)[(i)++]=(uint8_t)((((c)>>12)&0x3f)|0x80); \ sl@0: } \ sl@0: (s)[(i)++]=(uint8_t)((((c)>>6)&0x3f)|0x80); \ sl@0: } \ sl@0: (s)[(i)++]=(uint8_t)(((c)&0x3f)|0x80); \ sl@0: } \ sl@0: } sl@0: sl@0: /** sl@0: * Append a code point to a string, overwriting 1 or 2 code units. sl@0: * The offset points to the current end of the string contents sl@0: * and is advanced (post-increment). sl@0: * "Safe" macro, checks for a valid code point. sl@0: * If a non-ASCII code point is written, checks for sufficient space in the string. sl@0: * If the code point is not valid or trail bytes do not fit, sl@0: * then isError is set to TRUE. sl@0: * sl@0: * @param s const uint8_t * string buffer sl@0: * @param i string offset, i(length)) { \ sl@0: __count=(uint8_t)((length)-(i)); \ sl@0: } \ sl@0: while(__count>0 && U8_IS_TRAIL((s)[i])) { \ sl@0: ++(i); \ sl@0: --__count; \ sl@0: } \ sl@0: } \ sl@0: } sl@0: sl@0: /** sl@0: * Advance the string offset from one code point boundary to the n-th next one, sl@0: * i.e., move forward by n code points. sl@0: * (Post-incrementing iteration.) sl@0: * "Unsafe" macro, assumes well-formed UTF-8. sl@0: * sl@0: * @param s const uint8_t * string sl@0: * @param i string offset sl@0: * @param n number of code points to skip sl@0: * @see U8_FWD_N sl@0: * @stable ICU 2.4 sl@0: */ sl@0: #define U8_FWD_N_UNSAFE(s, i, n) { \ sl@0: int32_t __N=(n); \ sl@0: while(__N>0) { \ sl@0: U8_FWD_1_UNSAFE(s, i); \ sl@0: --__N; \ sl@0: } \ sl@0: } sl@0: sl@0: /** sl@0: * Advance the string offset from one code point boundary to the n-th next one, sl@0: * i.e., move forward by n code points. sl@0: * (Post-incrementing iteration.) sl@0: * "Safe" macro, checks for illegal sequences and for string boundaries. sl@0: * sl@0: * @param s const uint8_t * string sl@0: * @param i string offset, i0 && (i)<(length)) { \ sl@0: U8_FWD_1(s, i, length); \ sl@0: --__N; \ sl@0: } \ sl@0: } sl@0: sl@0: /** sl@0: * Adjust a random-access offset to a code point boundary sl@0: * at the start of a code point. sl@0: * If the offset points to a UTF-8 trail byte, sl@0: * then the offset is moved backward to the corresponding lead byte. sl@0: * Otherwise, it is not modified. sl@0: * "Unsafe" macro, assumes well-formed UTF-8. sl@0: * sl@0: * @param s const uint8_t * string sl@0: * @param i string offset sl@0: * @see U8_SET_CP_START sl@0: * @stable ICU 2.4 sl@0: */ sl@0: #define U8_SET_CP_START_UNSAFE(s, i) { \ sl@0: while(U8_IS_TRAIL((s)[i])) { --(i); } \ sl@0: } sl@0: sl@0: /** sl@0: * Adjust a random-access offset to a code point boundary sl@0: * at the start of a code point. sl@0: * If the offset points to a UTF-8 trail byte, sl@0: * then the offset is moved backward to the corresponding lead byte. sl@0: * Otherwise, it is not modified. sl@0: * "Safe" macro, checks for illegal sequences and for string boundaries. sl@0: * sl@0: * @param s const uint8_t * string sl@0: * @param start starting string offset (usually 0) sl@0: * @param i string offset, start<=i sl@0: * @see U8_SET_CP_START_UNSAFE sl@0: * @stable ICU 2.4 sl@0: */ sl@0: #define U8_SET_CP_START(s, start, i) { \ sl@0: if(U8_IS_TRAIL((s)[(i)])) { \ sl@0: (i)=utf8_back1SafeBody(s, start, (int32_t)(i)); \ sl@0: } \ sl@0: } sl@0: sl@0: /* definitions with backward iteration -------------------------------------- */ sl@0: sl@0: /** sl@0: * Move the string offset from one code point boundary to the previous one sl@0: * and get the code point between them. sl@0: * (Pre-decrementing backward iteration.) sl@0: * "Unsafe" macro, assumes well-formed UTF-8. sl@0: * sl@0: * The input offset may be the same as the string length. sl@0: * If the offset is behind a multi-byte sequence, then the macro will read sl@0: * the whole sequence. sl@0: * If the offset is behind a lead byte, then that itself sl@0: * will be returned as the code point. sl@0: * The result is undefined if the offset is behind an illegal UTF-8 sequence. sl@0: * sl@0: * @param s const uint8_t * string sl@0: * @param i string offset sl@0: * @param c output UChar32 variable sl@0: * @see U8_PREV sl@0: * @stable ICU 2.4 sl@0: */ sl@0: #define U8_PREV_UNSAFE(s, i, c) { \ sl@0: (c)=(s)[--(i)]; \ sl@0: if(U8_IS_TRAIL(c)) { \ sl@0: uint8_t __b, __count=1, __shift=6; \ sl@0: \ sl@0: /* c is a trail byte */ \ sl@0: (c)&=0x3f; \ sl@0: for(;;) { \ sl@0: __b=(s)[--(i)]; \ sl@0: if(__b>=0xc0) { \ sl@0: U8_MASK_LEAD_BYTE(__b, __count); \ sl@0: (c)|=(UChar32)__b<<__shift; \ sl@0: break; \ sl@0: } else { \ sl@0: (c)|=(UChar32)(__b&0x3f)<<__shift; \ sl@0: ++__count; \ sl@0: __shift+=6; \ sl@0: } \ sl@0: } \ sl@0: } \ sl@0: } sl@0: sl@0: /** sl@0: * Move the string offset from one code point boundary to the previous one sl@0: * and get the code point between them. sl@0: * (Pre-decrementing backward iteration.) sl@0: * "Safe" macro, checks for illegal sequences and for string boundaries. sl@0: * sl@0: * The input offset may be the same as the string length. sl@0: * If the offset is behind a multi-byte sequence, then the macro will read sl@0: * the whole sequence. sl@0: * If the offset is behind a lead byte, then that itself sl@0: * will be returned as the code point. sl@0: * If the offset is behind an illegal UTF-8 sequence, then c is set to a negative value. sl@0: * sl@0: * @param s const uint8_t * string sl@0: * @param start starting string offset (usually 0) sl@0: * @param i string offset, start<=i sl@0: * @param c output UChar32 variable, set to <0 in case of an error sl@0: * @see U8_PREV_UNSAFE sl@0: * @stable ICU 2.4 sl@0: */ sl@0: #define U8_PREV(s, start, i, c) { \ sl@0: (c)=(s)[--(i)]; \ sl@0: if((c)>=0x80) { \ sl@0: if((c)<=0xbf) { \ sl@0: (c)=utf8_prevCharSafeBody(s, start, &(i), c, -1); \ sl@0: } else { \ sl@0: (c)=U_SENTINEL; \ sl@0: } \ sl@0: } \ sl@0: } sl@0: sl@0: /** sl@0: * Move the string offset from one code point boundary to the previous one. sl@0: * (Pre-decrementing backward iteration.) sl@0: * The input offset may be the same as the string length. sl@0: * "Unsafe" macro, assumes well-formed UTF-8. sl@0: * sl@0: * @param s const uint8_t * string sl@0: * @param i string offset sl@0: * @see U8_BACK_1 sl@0: * @stable ICU 2.4 sl@0: */ sl@0: #define U8_BACK_1_UNSAFE(s, i) { \ sl@0: while(U8_IS_TRAIL((s)[--(i)])) {} \ sl@0: } sl@0: sl@0: /** sl@0: * Move the string offset from one code point boundary to the previous one. sl@0: * (Pre-decrementing backward iteration.) sl@0: * The input offset may be the same as the string length. sl@0: * "Safe" macro, checks for illegal sequences and for string boundaries. sl@0: * sl@0: * @param s const uint8_t * string sl@0: * @param start starting string offset (usually 0) sl@0: * @param i string offset, start<=i sl@0: * @see U8_BACK_1_UNSAFE sl@0: * @stable ICU 2.4 sl@0: */ sl@0: #define U8_BACK_1(s, start, i) { \ sl@0: if(U8_IS_TRAIL((s)[--(i)])) { \ sl@0: (i)=utf8_back1SafeBody(s, start, (int32_t)(i)); \ sl@0: } \ sl@0: } sl@0: sl@0: /** sl@0: * Move the string offset from one code point boundary to the n-th one before it, sl@0: * i.e., move backward by n code points. sl@0: * (Pre-decrementing backward iteration.) sl@0: * The input offset may be the same as the string length. sl@0: * "Unsafe" macro, assumes well-formed UTF-8. sl@0: * sl@0: * @param s const uint8_t * string sl@0: * @param i string offset sl@0: * @param n number of code points to skip sl@0: * @see U8_BACK_N sl@0: * @stable ICU 2.4 sl@0: */ sl@0: #define U8_BACK_N_UNSAFE(s, i, n) { \ sl@0: int32_t __N=(n); \ sl@0: while(__N>0) { \ sl@0: U8_BACK_1_UNSAFE(s, i); \ sl@0: --__N; \ sl@0: } \ sl@0: } sl@0: sl@0: /** sl@0: * Move the string offset from one code point boundary to the n-th one before it, sl@0: * i.e., move backward by n code points. sl@0: * (Pre-decrementing backward iteration.) sl@0: * The input offset may be the same as the string length. sl@0: * "Safe" macro, checks for illegal sequences and for string boundaries. sl@0: * sl@0: * @param s const uint8_t * string sl@0: * @param start index of the start of the string sl@0: * @param i string offset, i0 && (i)>(start)) { \ sl@0: U8_BACK_1(s, start, i); \ sl@0: --__N; \ sl@0: } \ sl@0: } sl@0: sl@0: /** sl@0: * Adjust a random-access offset to a code point boundary after a code point. sl@0: * If the offset is behind a partial multi-byte sequence, sl@0: * then the offset is incremented to behind the whole sequence. sl@0: * Otherwise, it is not modified. sl@0: * The input offset may be the same as the string length. sl@0: * "Unsafe" macro, assumes well-formed UTF-8. sl@0: * sl@0: * @param s const uint8_t * string sl@0: * @param i string offset sl@0: * @see U8_SET_CP_LIMIT sl@0: * @stable ICU 2.4 sl@0: */ sl@0: #define U8_SET_CP_LIMIT_UNSAFE(s, i) { \ sl@0: U8_BACK_1_UNSAFE(s, i); \ sl@0: U8_FWD_1_UNSAFE(s, i); \ sl@0: } sl@0: sl@0: /** sl@0: * Adjust a random-access offset to a code point boundary after a code point. sl@0: * If the offset is behind a partial multi-byte sequence, sl@0: * then the offset is incremented to behind the whole sequence. sl@0: * Otherwise, it is not modified. sl@0: * The input offset may be the same as the string length. sl@0: * "Safe" macro, checks for illegal sequences and for string boundaries. sl@0: * sl@0: * @param s const uint8_t * string sl@0: * @param start starting string offset (usually 0) sl@0: * @param i string offset, start<=i<=length sl@0: * @param length string length sl@0: * @see U8_SET_CP_LIMIT_UNSAFE sl@0: * @stable ICU 2.4 sl@0: */ sl@0: #define U8_SET_CP_LIMIT(s, start, i, length) { \ sl@0: if((start)<(i) && (i)<(length)) { \ sl@0: U8_BACK_1(s, start, i); \ sl@0: U8_FWD_1(s, i, length); \ sl@0: } \ sl@0: } sl@0: sl@0: #endif