Update contrib.
2 ********************************************************************
4 * Copyright (c) 1996-2005, International Business Machines Corporation and
5 * others. All Rights Reserved.
6 ********************************************************************
12 #include "unicode/utypes.h"
16 * \brief C++ API: Unicode Normalization
19 #if !UCONFIG_NO_NORMALIZATION
21 #include "unicode/uobject.h"
22 #include "unicode/unistr.h"
23 #include "unicode/chariter.h"
24 #include "unicode/unorm.h"
28 typedef struct UCharIterator UCharIterator; /**< C typedef for struct UCharIterator. @stable ICU 2.1 */
33 * The Normalizer class consists of two parts:
34 * - static functions that normalize strings or test if strings are normalized
35 * - a Normalizer object is an iterator that takes any kind of text and
36 * provides iteration over its normalized form
38 * The Normalizer class is not suitable for subclassing.
40 * The static functions are basically wrappers around the C implementation,
41 * using UnicodeString instead of UChar*.
42 * For basic information about normalization forms and details about the C API
43 * please see the documentation in unorm.h.
45 * The iterator API with the Normalizer constructors and the non-static functions
46 * uses a CharacterIterator as input. It is possible to pass a string which
47 * is then internally wrapped in a CharacterIterator.
48 * The input text is not normalized all at once, but incrementally where needed
49 * (providing efficient random access).
50 * This allows to pass in a large text but spend only a small amount of time
51 * normalizing a small part of that text.
52 * However, if the entire text is normalized, then the iterator will be
53 * slower than normalizing the entire text at once and iterating over the result.
54 * A possible use of the Normalizer iterator is also to report an index into the
55 * original text that is close to where the normalized characters come from.
57 * <em>Important:</em> The iterator API was cleaned up significantly for ICU 2.0.
58 * The earlier implementation reported the getIndex() inconsistently,
59 * and previous() could not be used after setIndex(), next(), first(), and current().
61 * Normalizer allows to start normalizing from anywhere in the input text by
62 * calling setIndexOnly(), first(), or last().
63 * Without calling any of these, the iterator will start at the beginning of the text.
65 * At any time, next() returns the next normalized code point (UChar32),
66 * with post-increment semantics (like CharacterIterator::next32PostInc()).
67 * previous() returns the previous normalized code point (UChar32),
68 * with pre-decrement semantics (like CharacterIterator::previous32()).
70 * current() returns the current code point
71 * (respectively the one at the newly set index) without moving
72 * the getIndex(). Note that if the text at the current position
73 * needs to be normalized, then these functions will do that.
74 * (This is why current() is not const.)
75 * It is more efficient to call setIndexOnly() instead, which does not
78 * getIndex() always refers to the position in the input text where the normalized
79 * code points are returned from. It does not always change with each returned
81 * The code point that is returned from any of the functions
82 * corresponds to text at or after getIndex(), according to the
83 * function's iteration semantics (post-increment or pre-decrement).
85 * next() returns a code point from at or after the getIndex()
86 * from before the next() call. After the next() call, the getIndex()
87 * might have moved to where the next code point will be returned from
88 * (from a next() or current() call).
89 * This is semantically equivalent to array access with array[index++]
90 * (post-increment semantics).
92 * previous() returns a code point from at or after the getIndex()
93 * from after the previous() call.
94 * This is semantically equivalent to array access with array[--index]
95 * (pre-decrement semantics).
97 * Internally, the Normalizer iterator normalizes a small piece of text
98 * starting at the getIndex() and ending at a following "safe" index.
99 * The normalized results is stored in an internal string buffer, and
100 * the code points are iterated from there.
101 * With multiple iteration calls, this is repeated until the next piece
102 * of text needs to be normalized, and the getIndex() needs to be moved.
104 * The following "safe" index, the internal buffer, and the secondary
105 * iteration index into that buffer are not exposed on the API.
106 * This also means that it is currently not practical to return to
107 * a particular, arbitrary position in the text because one would need to
108 * know, and be able to set, in addition to the getIndex(), at least also the
109 * current index into the internal buffer.
110 * It is currently only possible to observe when getIndex() changes
111 * (with careful consideration of the iteration semantics),
112 * at which time the internal index will be 0.
113 * For example, if getIndex() is different after next() than before it,
114 * then the internal index is 0 and one can return to this getIndex()
115 * later with setIndexOnly().
117 * @author Laura Werner, Mark Davis, Markus Scherer
120 class U_COMMON_API Normalizer : public UObject {
123 * If DONE is returned from an iteration function that returns a code point,
124 * then there are no more normalization results available.
134 * Creates a new <code>Normalizer</code> object for iterating over the
135 * normalized form of a given string.
137 * @param str The string to be normalized. The normalization
138 * will start at the beginning of the string.
140 * @param mode The normalization mode.
143 Normalizer(const UnicodeString& str, UNormalizationMode mode);
146 * Creates a new <code>Normalizer</code> object for iterating over the
147 * normalized form of a given string.
149 * @param str The string to be normalized. The normalization
150 * will start at the beginning of the string.
152 * @param length Length of the string, or -1 if NUL-terminated.
153 * @param mode The normalization mode.
156 Normalizer(const UChar* str, int32_t length, UNormalizationMode mode);
159 * Creates a new <code>Normalizer</code> object for iterating over the
160 * normalized form of the given text.
162 * @param iter The input text to be normalized. The normalization
163 * will start at the beginning of the string.
165 * @param mode The normalization mode.
168 Normalizer(const CharacterIterator& iter, UNormalizationMode mode);
172 * @param copy The object to be copied.
175 Normalizer(const Normalizer& copy);
181 virtual ~Normalizer();
184 //-------------------------------------------------------------------------
185 // Static utility methods
186 //-------------------------------------------------------------------------
189 * Normalizes a <code>UnicodeString</code> according to the specified normalization mode.
190 * This is a wrapper for unorm_normalize(), using UnicodeString's.
192 * The <code>options</code> parameter specifies which optional
193 * <code>Normalizer</code> features are to be enabled for this operation.
195 * @param source the input string to be normalized.
196 * @param mode the normalization mode
197 * @param options the optional features to be enabled (0 for no options)
198 * @param result The normalized string (on output).
199 * @param status The error code.
202 static void U_EXPORT2 normalize(const UnicodeString& source,
203 UNormalizationMode mode, int32_t options,
204 UnicodeString& result,
208 * Compose a <code>UnicodeString</code>.
209 * This is equivalent to normalize() with mode UNORM_NFC or UNORM_NFKC.
210 * This is a wrapper for unorm_normalize(), using UnicodeString's.
212 * The <code>options</code> parameter specifies which optional
213 * <code>Normalizer</code> features are to be enabled for this operation.
215 * @param source the string to be composed.
216 * @param compat Perform compatibility decomposition before composition.
217 * If this argument is <code>FALSE</code>, only canonical
218 * decomposition will be performed.
219 * @param options the optional features to be enabled (0 for no options)
220 * @param result The composed string (on output).
221 * @param status The error code.
224 static void U_EXPORT2 compose(const UnicodeString& source,
225 UBool compat, int32_t options,
226 UnicodeString& result,
230 * Static method to decompose a <code>UnicodeString</code>.
231 * This is equivalent to normalize() with mode UNORM_NFD or UNORM_NFKD.
232 * This is a wrapper for unorm_normalize(), using UnicodeString's.
234 * The <code>options</code> parameter specifies which optional
235 * <code>Normalizer</code> features are to be enabled for this operation.
237 * @param source the string to be decomposed.
238 * @param compat Perform compatibility decomposition.
239 * If this argument is <code>FALSE</code>, only canonical
240 * decomposition will be performed.
241 * @param options the optional features to be enabled (0 for no options)
242 * @param result The decomposed string (on output).
243 * @param status The error code.
246 static void U_EXPORT2 decompose(const UnicodeString& source,
247 UBool compat, int32_t options,
248 UnicodeString& result,
252 * Performing quick check on a string, to quickly determine if the string is
253 * in a particular normalization format.
254 * This is a wrapper for unorm_quickCheck(), using a UnicodeString.
256 * Three types of result can be returned UNORM_YES, UNORM_NO or
257 * UNORM_MAYBE. Result UNORM_YES indicates that the argument
258 * string is in the desired normalized format, UNORM_NO determines that
259 * argument string is not in the desired normalized format. A
260 * UNORM_MAYBE result indicates that a more thorough check is required,
261 * the user may have to put the string in its normalized form and compare the
263 * @param source string for determining if it is in a normalized format
264 * @param mode normalization format
265 * @param status A reference to a UErrorCode to receive any errors
266 * @return UNORM_YES, UNORM_NO or UNORM_MAYBE
271 static inline UNormalizationCheckResult
272 quickCheck(const UnicodeString &source, UNormalizationMode mode, UErrorCode &status);
275 * Performing quick check on a string; same as the other version of quickCheck
276 * but takes an extra options parameter like most normalization functions.
278 * @param source string for determining if it is in a normalized format
279 * @param mode normalization format
280 * @param options the optional features to be enabled (0 for no options)
281 * @param status A reference to a UErrorCode to receive any errors
282 * @return UNORM_YES, UNORM_NO or UNORM_MAYBE
287 static inline UNormalizationCheckResult
288 quickCheck(const UnicodeString &source, UNormalizationMode mode, int32_t options, UErrorCode &status);
291 * Test if a string is in a given normalization form.
292 * This is semantically equivalent to source.equals(normalize(source, mode)) .
294 * Unlike unorm_quickCheck(), this function returns a definitive result,
296 * For NFD, NFKD, and FCD, both functions work exactly the same.
297 * For NFC and NFKC where quickCheck may return "maybe", this function will
298 * perform further tests to arrive at a TRUE/FALSE result.
300 * @param src String that is to be tested if it is in a normalization format.
301 * @param mode Which normalization form to test for.
302 * @param errorCode ICU error code in/out parameter.
303 * Must fulfill U_SUCCESS before the function call.
304 * @return Boolean value indicating whether the source string is in the
305 * "mode" normalization form.
311 isNormalized(const UnicodeString &src, UNormalizationMode mode, UErrorCode &errorCode);
314 * Test if a string is in a given normalization form; same as the other version of isNormalized
315 * but takes an extra options parameter like most normalization functions.
317 * @param src String that is to be tested if it is in a normalization format.
318 * @param mode Which normalization form to test for.
319 * @param options the optional features to be enabled (0 for no options)
320 * @param errorCode ICU error code in/out parameter.
321 * Must fulfill U_SUCCESS before the function call.
322 * @return Boolean value indicating whether the source string is in the
323 * "mode" normalization form.
329 isNormalized(const UnicodeString &src, UNormalizationMode mode, int32_t options, UErrorCode &errorCode);
332 * Concatenate normalized strings, making sure that the result is normalized as well.
334 * If both the left and the right strings are in
335 * the normalization form according to "mode/options",
336 * then the result will be
339 * dest=normalize(left+right, mode, options)
342 * For details see unorm_concatenate in unorm.h.
344 * @param left Left source string.
345 * @param right Right source string.
346 * @param result The output string.
347 * @param mode The normalization mode.
348 * @param options A bit set of normalization options.
349 * @param errorCode ICU error code in/out parameter.
350 * Must fulfill U_SUCCESS before the function call.
353 * @see unorm_concatenate
356 * @see unorm_previous
360 static UnicodeString &
361 U_EXPORT2 concatenate(UnicodeString &left, UnicodeString &right,
362 UnicodeString &result,
363 UNormalizationMode mode, int32_t options,
364 UErrorCode &errorCode);
367 * Compare two strings for canonical equivalence.
368 * Further options include case-insensitive comparison and
369 * code point order (as opposed to code unit order).
371 * Canonical equivalence between two strings is defined as their normalized
372 * forms (NFD or NFC) being identical.
373 * This function compares strings incrementally instead of normalizing
374 * (and optionally case-folding) both strings entirely,
375 * improving performance significantly.
377 * Bulk normalization is only necessary if the strings do not fulfill the FCD
378 * conditions. Only in this case, and only if the strings are relatively long,
379 * is memory allocated temporarily.
380 * For FCD strings and short non-FCD strings there is no memory allocation.
382 * Semantically, this is equivalent to
383 * strcmp[CodePointOrder](NFD(foldCase(s1)), NFD(foldCase(s2)))
384 * where code point order and foldCase are all optional.
386 * UAX 21 2.5 Caseless Matching specifies that for a canonical caseless match
387 * the case folding must be performed first, then the normalization.
389 * @param s1 First source string.
390 * @param s2 Second source string.
392 * @param options A bit set of options:
393 * - U_FOLD_CASE_DEFAULT or 0 is used for default options:
394 * Case-sensitive comparison in code unit order, and the input strings
395 * are quick-checked for FCD.
397 * - UNORM_INPUT_IS_FCD
398 * Set if the caller knows that both s1 and s2 fulfill the FCD conditions.
399 * If not set, the function will quickCheck for FCD
400 * and normalize if necessary.
402 * - U_COMPARE_CODE_POINT_ORDER
403 * Set to choose code point order instead of code unit order
404 * (see u_strCompare for details).
406 * - U_COMPARE_IGNORE_CASE
407 * Set to compare strings case-insensitively using case folding,
408 * instead of case-sensitively.
409 * If set, then the following case folding options are used.
411 * - Options as used with case-insensitive comparisons, currently:
413 * - U_FOLD_CASE_EXCLUDE_SPECIAL_I
414 * (see u_strCaseCompare for details)
416 * - regular normalization options shifted left by UNORM_COMPARE_NORM_OPTIONS_SHIFT
418 * @param errorCode ICU error code in/out parameter.
419 * Must fulfill U_SUCCESS before the function call.
420 * @return <0 or 0 or >0 as usual for string comparisons
426 * @see u_strCaseCompare
430 static inline int32_t
431 compare(const UnicodeString &s1, const UnicodeString &s2,
433 UErrorCode &errorCode);
435 //-------------------------------------------------------------------------
437 //-------------------------------------------------------------------------
440 * Return the current character in the normalized text.
441 * current() may need to normalize some text at getIndex().
442 * The getIndex() is not changed.
444 * @return the current normalized code point
447 UChar32 current(void);
450 * Return the first character in the normalized text.
451 * This is equivalent to setIndexOnly(startIndex()) followed by next().
452 * (Post-increment semantics.)
454 * @return the first normalized code point
460 * Return the last character in the normalized text.
461 * This is equivalent to setIndexOnly(endIndex()) followed by previous().
462 * (Pre-decrement semantics.)
464 * @return the last normalized code point
470 * Return the next character in the normalized text.
471 * (Post-increment semantics.)
472 * If the end of the text has already been reached, DONE is returned.
473 * The DONE value could be confused with a U+FFFF non-character code point
474 * in the text. If this is possible, you can test getIndex()<endIndex()
475 * before calling next(), or (getIndex()<endIndex() || last()!=DONE)
476 * after calling next(). (Calling last() will change the iterator state!)
478 * The C API unorm_next() is more efficient and does not have this ambiguity.
480 * @return the next normalized code point
486 * Return the previous character in the normalized text and decrement.
487 * (Pre-decrement semantics.)
488 * If the beginning of the text has already been reached, DONE is returned.
489 * The DONE value could be confused with a U+FFFF non-character code point
490 * in the text. If this is possible, you can test
491 * (getIndex()>startIndex() || first()!=DONE). (Calling first() will change
492 * the iterator state!)
494 * The C API unorm_previous() is more efficient and does not have this ambiguity.
496 * @return the previous normalized code point
499 UChar32 previous(void);
502 * Set the iteration position in the input text that is being normalized,
503 * without any immediate normalization.
504 * After setIndexOnly(), getIndex() will return the same index that is
507 * @param index the desired index in the input text.
510 void setIndexOnly(int32_t index);
513 * Reset the index to the beginning of the text.
514 * This is equivalent to setIndexOnly(startIndex)).
520 * Retrieve the current iteration position in the input text that is
523 * A following call to next() will return a normalized code point from
524 * the input text at or after this index.
526 * After a call to previous(), getIndex() will point at or before the
527 * position in the input text where the normalized code point
528 * was returned from with previous().
530 * @return the current index in the input text
533 int32_t getIndex(void) const;
536 * Retrieve the index of the start of the input text. This is the begin index
537 * of the <code>CharacterIterator</code> or the start (i.e. index 0) of the string
538 * over which this <code>Normalizer</code> is iterating.
540 * @return the smallest index in the input text where the Normalizer operates
543 int32_t startIndex(void) const;
546 * Retrieve the index of the end of the input text. This is the end index
547 * of the <code>CharacterIterator</code> or the length of the string
548 * over which this <code>Normalizer</code> is iterating.
549 * This end index is exclusive, i.e., the Normalizer operates only on characters
552 * @return the first index in the input text where the Normalizer does not operate
555 int32_t endIndex(void) const;
558 * Returns TRUE when both iterators refer to the same character in the same
561 * @param that a Normalizer object to compare this one to
562 * @return comparison result
565 UBool operator==(const Normalizer& that) const;
568 * Returns FALSE when both iterators refer to the same character in the same
571 * @param that a Normalizer object to compare this one to
572 * @return comparison result
575 inline UBool operator!=(const Normalizer& that) const;
578 * Returns a pointer to a new Normalizer that is a clone of this one.
579 * The caller is responsible for deleting the new clone.
580 * @return a pointer to a new Normalizer
583 Normalizer* clone(void) const;
586 * Generates a hash code for this iterator.
588 * @return the hash code
591 int32_t hashCode(void) const;
593 //-------------------------------------------------------------------------
594 // Property access methods
595 //-------------------------------------------------------------------------
598 * Set the normalization mode for this object.
600 * <b>Note:</b>If the normalization mode is changed while iterating
601 * over a string, calls to {@link #next() } and {@link #previous() } may
602 * return previously buffers characters in the old normalization mode
603 * until the iteration is able to re-sync at the next base character.
604 * It is safest to call {@link #setIndexOnly }, {@link #reset() },
605 * {@link #setText }, {@link #first() },
606 * {@link #last() }, etc. after calling <code>setMode</code>.
608 * @param newMode the new mode for this <code>Normalizer</code>.
612 void setMode(UNormalizationMode newMode);
615 * Return the normalization mode for this object.
617 * This is an unusual name because there used to be a getMode() that
618 * returned a different type.
620 * @return the mode for this <code>Normalizer</code>
624 UNormalizationMode getUMode(void) const;
627 * Set options that affect this <code>Normalizer</code>'s operation.
628 * Options do not change the basic composition or decomposition operation
629 * that is being performed, but they control whether
630 * certain optional portions of the operation are done.
631 * Currently the only available option is obsolete.
633 * It is possible to specify multiple options that are all turned on or off.
635 * @param option the option(s) whose value is/are to be set.
636 * @param value the new setting for the option. Use <code>TRUE</code> to
637 * turn the option(s) on and <code>FALSE</code> to turn it/them off.
642 void setOption(int32_t option,
646 * Determine whether an option is turned on or off.
647 * If multiple options are specified, then the result is TRUE if any
650 * @param option the option(s) that are to be checked
651 * @return TRUE if any of the option(s) are set
655 UBool getOption(int32_t option) const;
658 * Set the input text over which this <code>Normalizer</code> will iterate.
659 * The iteration position is set to the beginning.
661 * @param newText a string that replaces the current input text
662 * @param status a UErrorCode
665 void setText(const UnicodeString& newText,
669 * Set the input text over which this <code>Normalizer</code> will iterate.
670 * The iteration position is set to the beginning.
672 * @param newText a CharacterIterator object that replaces the current input text
673 * @param status a UErrorCode
676 void setText(const CharacterIterator& newText,
680 * Set the input text over which this <code>Normalizer</code> will iterate.
681 * The iteration position is set to the beginning.
683 * @param newText a string that replaces the current input text
684 * @param length the length of the string, or -1 if NUL-terminated
685 * @param status a UErrorCode
688 void setText(const UChar* newText,
692 * Copies the input text into the UnicodeString argument.
694 * @param result Receives a copy of the text under iteration.
697 void getText(UnicodeString& result);
700 * ICU "poor man's RTTI", returns a UClassID for this class.
701 * @returns a UClassID for this class.
704 static UClassID U_EXPORT2 getStaticClassID();
707 * ICU "poor man's RTTI", returns a UClassID for the actual class.
708 * @return a UClassID for the actual class.
711 virtual UClassID getDynamicClassID() const;
714 //-------------------------------------------------------------------------
716 //-------------------------------------------------------------------------
718 Normalizer(); // default constructor not implemented
719 Normalizer &operator=(const Normalizer &that); // assignment operator not implemented
721 // Private utility methods for iteration
722 // For documentation, see the source code
723 UBool nextNormalize();
724 UBool previousNormalize();
726 void init(CharacterIterator *iter);
727 void clearBuffer(void);
729 //-------------------------------------------------------------------------
731 //-------------------------------------------------------------------------
733 UNormalizationMode fUMode;
736 // The input text and our position in it
739 // The normalization buffer is the result of normalization
740 // of the source in [currentIndex..nextIndex[ .
741 int32_t currentIndex, nextIndex;
743 // A buffer for holding intermediate results
744 UnicodeString buffer;
749 //-------------------------------------------------------------------------
750 // Inline implementations
751 //-------------------------------------------------------------------------
754 Normalizer::operator!= (const Normalizer& other) const
755 { return ! operator==(other); }
757 inline UNormalizationCheckResult
758 Normalizer::quickCheck(const UnicodeString& source,
759 UNormalizationMode mode,
760 UErrorCode &status) {
761 if(U_FAILURE(status)) {
765 return unorm_quickCheck(source.getBuffer(), source.length(),
769 inline UNormalizationCheckResult
770 Normalizer::quickCheck(const UnicodeString& source,
771 UNormalizationMode mode, int32_t options,
772 UErrorCode &status) {
773 if(U_FAILURE(status)) {
777 return unorm_quickCheckWithOptions(source.getBuffer(), source.length(),
778 mode, options, &status);
782 Normalizer::isNormalized(const UnicodeString& source,
783 UNormalizationMode mode,
784 UErrorCode &status) {
785 if(U_FAILURE(status)) {
789 return unorm_isNormalized(source.getBuffer(), source.length(),
794 Normalizer::isNormalized(const UnicodeString& source,
795 UNormalizationMode mode, int32_t options,
796 UErrorCode &status) {
797 if(U_FAILURE(status)) {
801 return unorm_isNormalizedWithOptions(source.getBuffer(), source.length(),
802 mode, options, &status);
806 Normalizer::compare(const UnicodeString &s1, const UnicodeString &s2,
808 UErrorCode &errorCode) {
809 // all argument checking is done in unorm_compare
810 return unorm_compare(s1.getBuffer(), s1.length(),
811 s2.getBuffer(), s2.length(),
818 #endif /* #if !UCONFIG_NO_NORMALIZATION */