| /* |
| ******************************************************************************* |
| * Copyright (C) 2012-2014, International Business Machines |
| * Corporation and others. All Rights Reserved. |
| ******************************************************************************* |
| * utf8collationiterator.cpp |
| * |
| * created on: 2012nov12 (from utf16collationiterator.cpp & uitercollationiterator.cpp) |
| * created by: Markus W. Scherer |
| */ |
| |
| #include "unicode/utypes.h" |
| |
| #if !UCONFIG_NO_COLLATION |
| |
| #include "unicode/utf8.h" |
| #include "charstr.h" |
| #include "cmemory.h" |
| #include "collation.h" |
| #include "collationdata.h" |
| #include "collationfcd.h" |
| #include "collationiterator.h" |
| #include "normalizer2impl.h" |
| #include "uassert.h" |
| #include "utf8collationiterator.h" |
| |
| U_NAMESPACE_BEGIN |
| |
| UTF8CollationIterator::~UTF8CollationIterator() {} |
| |
| void |
| UTF8CollationIterator::resetToOffset(int32_t newOffset) { |
| reset(); |
| pos = newOffset; |
| } |
| |
| int32_t |
| UTF8CollationIterator::getOffset() const { |
| return pos; |
| } |
| |
| uint32_t |
| UTF8CollationIterator::handleNextCE32(UChar32 &c, UErrorCode & /*errorCode*/) { |
| if(pos == length) { |
| c = U_SENTINEL; |
| return Collation::FALLBACK_CE32; |
| } |
| // Optimized combination of U8_NEXT_OR_FFFD() and UTRIE2_U8_NEXT32(). |
| c = u8[pos++]; |
| if(c < 0xc0) { |
| // ASCII 00..7F; trail bytes 80..BF map to error values. |
| return trie->data32[c]; |
| } |
| uint8_t t1, t2; |
| if(c < 0xe0 && pos != length && (t1 = (u8[pos] - 0x80)) <= 0x3f) { |
| // U+0080..U+07FF; 00..7F map to error values. |
| uint32_t ce32 = trie->data32[trie->index[(UTRIE2_UTF8_2B_INDEX_2_OFFSET - 0xc0) + c] + t1]; |
| c = ((c & 0x1f) << 6) | t1; |
| ++pos; |
| return ce32; |
| } else if(c <= 0xef && |
| ((pos + 1) < length || length < 0) && |
| (t1 = (u8[pos] - 0x80)) <= 0x3f && (c != 0xe0 || t1 >= 0x20) && |
| (t2 = (u8[pos + 1] - 0x80)) <= 0x3f |
| ) { |
| // U+0800..U+FFFF; caller maps surrogates to error values. |
| c = (UChar)((c << 12) | (t1 << 6) | t2); |
| pos += 2; |
| return UTRIE2_GET32_FROM_U16_SINGLE_LEAD(trie, c); |
| } else { |
| // Function call for supplementary code points and error cases. |
| // Illegal byte sequences yield U+FFFD. |
| c = utf8_nextCharSafeBody(u8, &pos, length, c, -3); |
| return data->getCE32(c); |
| } |
| } |
| |
| UBool |
| UTF8CollationIterator::foundNULTerminator() { |
| if(length < 0) { |
| length = --pos; |
| return TRUE; |
| } else { |
| return FALSE; |
| } |
| } |
| |
| UBool |
| UTF8CollationIterator::forbidSurrogateCodePoints() const { |
| return TRUE; |
| } |
| |
| UChar32 |
| UTF8CollationIterator::nextCodePoint(UErrorCode & /*errorCode*/) { |
| if(pos == length) { |
| return U_SENTINEL; |
| } |
| if(u8[pos] == 0 && length < 0) { |
| length = pos; |
| return U_SENTINEL; |
| } |
| UChar32 c; |
| U8_NEXT_OR_FFFD(u8, pos, length, c); |
| return c; |
| } |
| |
| UChar32 |
| UTF8CollationIterator::previousCodePoint(UErrorCode & /*errorCode*/) { |
| if(pos == 0) { |
| return U_SENTINEL; |
| } |
| UChar32 c; |
| U8_PREV_OR_FFFD(u8, 0, pos, c); |
| return c; |
| } |
| |
| void |
| UTF8CollationIterator::forwardNumCodePoints(int32_t num, UErrorCode & /*errorCode*/) { |
| U8_FWD_N(u8, pos, length, num); |
| } |
| |
| void |
| UTF8CollationIterator::backwardNumCodePoints(int32_t num, UErrorCode & /*errorCode*/) { |
| U8_BACK_N(u8, 0, pos, num); |
| } |
| |
| // FCDUTF8CollationIterator ------------------------------------------------ *** |
| |
| FCDUTF8CollationIterator::~FCDUTF8CollationIterator() {} |
| |
| void |
| FCDUTF8CollationIterator::resetToOffset(int32_t newOffset) { |
| reset(); |
| start = pos = newOffset; |
| state = CHECK_FWD; |
| } |
| |
| int32_t |
| FCDUTF8CollationIterator::getOffset() const { |
| if(state != IN_NORMALIZED) { |
| return pos; |
| } else if(pos == 0) { |
| return start; |
| } else { |
| return limit; |
| } |
| } |
| |
| uint32_t |
| FCDUTF8CollationIterator::handleNextCE32(UChar32 &c, UErrorCode &errorCode) { |
| for(;;) { |
| if(state == CHECK_FWD) { |
| // Combination of UTF8CollationIterator::handleNextCE32() with FCD check fastpath. |
| if(pos == length) { |
| c = U_SENTINEL; |
| return Collation::FALLBACK_CE32; |
| } |
| c = u8[pos++]; |
| if(c < 0xc0) { |
| // ASCII 00..7F; trail bytes 80..BF map to error values. |
| return trie->data32[c]; |
| } |
| uint8_t t1, t2; |
| if(c < 0xe0 && pos != length && (t1 = (u8[pos] - 0x80)) <= 0x3f) { |
| // U+0080..U+07FF; 00..7F map to error values. |
| uint32_t ce32 = trie->data32[trie->index[(UTRIE2_UTF8_2B_INDEX_2_OFFSET - 0xc0) + c] + t1]; |
| c = ((c & 0x1f) << 6) | t1; |
| ++pos; |
| if(CollationFCD::hasTccc(c) && pos != length && nextHasLccc()) { |
| pos -= 2; |
| } else { |
| return ce32; |
| } |
| } else if(c <= 0xef && |
| ((pos + 1) < length || length < 0) && |
| (t1 = (u8[pos] - 0x80)) <= 0x3f && (c != 0xe0 || t1 >= 0x20) && |
| (t2 = (u8[pos + 1] - 0x80)) <= 0x3f |
| ) { |
| // U+0800..U+FFFF; caller maps surrogates to error values. |
| c = (UChar)((c << 12) | (t1 << 6) | t2); |
| pos += 2; |
| if(CollationFCD::hasTccc(c) && |
| (CollationFCD::maybeTibetanCompositeVowel(c) || |
| (pos != length && nextHasLccc()))) { |
| pos -= 3; |
| } else { |
| break; // return CE32(BMP) |
| } |
| } else { |
| // Function call for supplementary code points and error cases. |
| // Illegal byte sequences yield U+FFFD. |
| c = utf8_nextCharSafeBody(u8, &pos, length, c, -3); |
| if(c == 0xfffd) { |
| return Collation::FFFD_CE32; |
| } else { |
| U_ASSERT(c > 0xffff); |
| if(CollationFCD::hasTccc(U16_LEAD(c)) && pos != length && nextHasLccc()) { |
| pos -= 4; |
| } else { |
| return data->getCE32FromSupplementary(c); |
| } |
| } |
| } |
| if(!nextSegment(errorCode)) { |
| c = U_SENTINEL; |
| return Collation::FALLBACK_CE32; |
| } |
| continue; |
| } else if(state == IN_FCD_SEGMENT && pos != limit) { |
| return UTF8CollationIterator::handleNextCE32(c, errorCode); |
| } else if(state == IN_NORMALIZED && pos != normalized.length()) { |
| c = normalized[pos++]; |
| break; |
| } else { |
| switchToForward(); |
| } |
| } |
| return UTRIE2_GET32_FROM_U16_SINGLE_LEAD(trie, c); |
| } |
| |
| UBool |
| FCDUTF8CollationIterator::nextHasLccc() const { |
| U_ASSERT(state == CHECK_FWD && pos != length); |
| // The lowest code point with ccc!=0 is U+0300 which is CC 80 in UTF-8. |
| // CJK U+4000..U+DFFF except U+Axxx are also FCD-inert. (Lead bytes E4..ED except EA.) |
| UChar32 c = u8[pos]; |
| if(c < 0xcc || (0xe4 <= c && c <= 0xed && c != 0xea)) { return FALSE; } |
| int32_t i = pos; |
| U8_NEXT_OR_FFFD(u8, i, length, c); |
| if(c > 0xffff) { c = U16_LEAD(c); } |
| return CollationFCD::hasLccc(c); |
| } |
| |
| UBool |
| FCDUTF8CollationIterator::previousHasTccc() const { |
| U_ASSERT(state == CHECK_BWD && pos != 0); |
| UChar32 c = u8[pos - 1]; |
| if(c < 0x80) { return FALSE; } |
| int32_t i = pos; |
| U8_PREV_OR_FFFD(u8, 0, i, c); |
| if(c > 0xffff) { c = U16_LEAD(c); } |
| return CollationFCD::hasTccc(c); |
| } |
| |
| UChar |
| FCDUTF8CollationIterator::handleGetTrailSurrogate() { |
| if(state != IN_NORMALIZED) { return 0; } |
| U_ASSERT(pos < normalized.length()); |
| UChar trail; |
| if(U16_IS_TRAIL(trail = normalized[pos])) { ++pos; } |
| return trail; |
| } |
| |
| UBool |
| FCDUTF8CollationIterator::foundNULTerminator() { |
| if(state == CHECK_FWD && length < 0) { |
| length = --pos; |
| return TRUE; |
| } else { |
| return FALSE; |
| } |
| } |
| |
| UChar32 |
| FCDUTF8CollationIterator::nextCodePoint(UErrorCode &errorCode) { |
| UChar32 c; |
| for(;;) { |
| if(state == CHECK_FWD) { |
| if(pos == length || ((c = u8[pos]) == 0 && length < 0)) { |
| return U_SENTINEL; |
| } |
| if(c < 0x80) { |
| ++pos; |
| return c; |
| } |
| U8_NEXT_OR_FFFD(u8, pos, length, c); |
| if(CollationFCD::hasTccc(c <= 0xffff ? c : U16_LEAD(c)) && |
| (CollationFCD::maybeTibetanCompositeVowel(c) || |
| (pos != length && nextHasLccc()))) { |
| // c is not FCD-inert, therefore it is not U+FFFD and it has a valid byte sequence |
| // and we can use U8_LENGTH() rather than a previous-position variable. |
| pos -= U8_LENGTH(c); |
| if(!nextSegment(errorCode)) { |
| return U_SENTINEL; |
| } |
| continue; |
| } |
| return c; |
| } else if(state == IN_FCD_SEGMENT && pos != limit) { |
| U8_NEXT_OR_FFFD(u8, pos, length, c); |
| return c; |
| } else if(state == IN_NORMALIZED && pos != normalized.length()) { |
| c = normalized.char32At(pos); |
| pos += U16_LENGTH(c); |
| return c; |
| } else { |
| switchToForward(); |
| } |
| } |
| } |
| |
| UChar32 |
| FCDUTF8CollationIterator::previousCodePoint(UErrorCode &errorCode) { |
| UChar32 c; |
| for(;;) { |
| if(state == CHECK_BWD) { |
| if(pos == 0) { |
| return U_SENTINEL; |
| } |
| if((c = u8[pos - 1]) < 0x80) { |
| --pos; |
| return c; |
| } |
| U8_PREV_OR_FFFD(u8, 0, pos, c); |
| if(CollationFCD::hasLccc(c <= 0xffff ? c : U16_LEAD(c)) && |
| (CollationFCD::maybeTibetanCompositeVowel(c) || |
| (pos != 0 && previousHasTccc()))) { |
| // c is not FCD-inert, therefore it is not U+FFFD and it has a valid byte sequence |
| // and we can use U8_LENGTH() rather than a previous-position variable. |
| pos += U8_LENGTH(c); |
| if(!previousSegment(errorCode)) { |
| return U_SENTINEL; |
| } |
| continue; |
| } |
| return c; |
| } else if(state == IN_FCD_SEGMENT && pos != start) { |
| U8_PREV_OR_FFFD(u8, 0, pos, c); |
| return c; |
| } else if(state >= IN_NORMALIZED && pos != 0) { |
| c = normalized.char32At(pos - 1); |
| pos -= U16_LENGTH(c); |
| return c; |
| } else { |
| switchToBackward(); |
| } |
| } |
| } |
| |
| void |
| FCDUTF8CollationIterator::forwardNumCodePoints(int32_t num, UErrorCode &errorCode) { |
| // Specify the class to avoid a virtual-function indirection. |
| // In Java, we would declare this class final. |
| while(num > 0 && FCDUTF8CollationIterator::nextCodePoint(errorCode) >= 0) { |
| --num; |
| } |
| } |
| |
| void |
| FCDUTF8CollationIterator::backwardNumCodePoints(int32_t num, UErrorCode &errorCode) { |
| // Specify the class to avoid a virtual-function indirection. |
| // In Java, we would declare this class final. |
| while(num > 0 && FCDUTF8CollationIterator::previousCodePoint(errorCode) >= 0) { |
| --num; |
| } |
| } |
| |
| void |
| FCDUTF8CollationIterator::switchToForward() { |
| U_ASSERT(state == CHECK_BWD || |
| (state == IN_FCD_SEGMENT && pos == limit) || |
| (state == IN_NORMALIZED && pos == normalized.length())); |
| if(state == CHECK_BWD) { |
| // Turn around from backward checking. |
| start = pos; |
| if(pos == limit) { |
| state = CHECK_FWD; // Check forward. |
| } else { // pos < limit |
| state = IN_FCD_SEGMENT; // Stay in FCD segment. |
| } |
| } else { |
| // Reached the end of the FCD segment. |
| if(state == IN_FCD_SEGMENT) { |
| // The input text segment is FCD, extend it forward. |
| } else { |
| // The input text segment needed to be normalized. |
| // Switch to checking forward from it. |
| start = pos = limit; |
| } |
| state = CHECK_FWD; |
| } |
| } |
| |
| UBool |
| FCDUTF8CollationIterator::nextSegment(UErrorCode &errorCode) { |
| if(U_FAILURE(errorCode)) { return FALSE; } |
| U_ASSERT(state == CHECK_FWD && pos != length); |
| // The input text [start..pos[ passes the FCD check. |
| int32_t segmentStart = pos; |
| // Collect the characters being checked, in case they need to be normalized. |
| UnicodeString s; |
| uint8_t prevCC = 0; |
| for(;;) { |
| // Fetch the next character and its fcd16 value. |
| int32_t cpStart = pos; |
| UChar32 c; |
| U8_NEXT_OR_FFFD(u8, pos, length, c); |
| uint16_t fcd16 = nfcImpl.getFCD16(c); |
| uint8_t leadCC = (uint8_t)(fcd16 >> 8); |
| if(leadCC == 0 && cpStart != segmentStart) { |
| // FCD boundary before this character. |
| pos = cpStart; |
| break; |
| } |
| s.append(c); |
| if(leadCC != 0 && (prevCC > leadCC || CollationFCD::isFCD16OfTibetanCompositeVowel(fcd16))) { |
| // Fails FCD check. Find the next FCD boundary and normalize. |
| while(pos != length) { |
| cpStart = pos; |
| U8_NEXT_OR_FFFD(u8, pos, length, c); |
| if(nfcImpl.getFCD16(c) <= 0xff) { |
| pos = cpStart; |
| break; |
| } |
| s.append(c); |
| } |
| if(!normalize(s, errorCode)) { return FALSE; } |
| start = segmentStart; |
| limit = pos; |
| state = IN_NORMALIZED; |
| pos = 0; |
| return TRUE; |
| } |
| prevCC = (uint8_t)fcd16; |
| if(pos == length || prevCC == 0) { |
| // FCD boundary after the last character. |
| break; |
| } |
| } |
| limit = pos; |
| pos = segmentStart; |
| U_ASSERT(pos != limit); |
| state = IN_FCD_SEGMENT; |
| return TRUE; |
| } |
| |
| void |
| FCDUTF8CollationIterator::switchToBackward() { |
| U_ASSERT(state == CHECK_FWD || |
| (state == IN_FCD_SEGMENT && pos == start) || |
| (state >= IN_NORMALIZED && pos == 0)); |
| if(state == CHECK_FWD) { |
| // Turn around from forward checking. |
| limit = pos; |
| if(pos == start) { |
| state = CHECK_BWD; // Check backward. |
| } else { // pos > start |
| state = IN_FCD_SEGMENT; // Stay in FCD segment. |
| } |
| } else { |
| // Reached the start of the FCD segment. |
| if(state == IN_FCD_SEGMENT) { |
| // The input text segment is FCD, extend it backward. |
| } else { |
| // The input text segment needed to be normalized. |
| // Switch to checking backward from it. |
| limit = pos = start; |
| } |
| state = CHECK_BWD; |
| } |
| } |
| |
| UBool |
| FCDUTF8CollationIterator::previousSegment(UErrorCode &errorCode) { |
| if(U_FAILURE(errorCode)) { return FALSE; } |
| U_ASSERT(state == CHECK_BWD && pos != 0); |
| // The input text [pos..limit[ passes the FCD check. |
| int32_t segmentLimit = pos; |
| // Collect the characters being checked, in case they need to be normalized. |
| UnicodeString s; |
| uint8_t nextCC = 0; |
| for(;;) { |
| // Fetch the previous character and its fcd16 value. |
| int32_t cpLimit = pos; |
| UChar32 c; |
| U8_PREV_OR_FFFD(u8, 0, pos, c); |
| uint16_t fcd16 = nfcImpl.getFCD16(c); |
| uint8_t trailCC = (uint8_t)fcd16; |
| if(trailCC == 0 && cpLimit != segmentLimit) { |
| // FCD boundary after this character. |
| pos = cpLimit; |
| break; |
| } |
| s.append(c); |
| if(trailCC != 0 && ((nextCC != 0 && trailCC > nextCC) || |
| CollationFCD::isFCD16OfTibetanCompositeVowel(fcd16))) { |
| // Fails FCD check. Find the previous FCD boundary and normalize. |
| while(fcd16 > 0xff && pos != 0) { |
| cpLimit = pos; |
| U8_PREV_OR_FFFD(u8, 0, pos, c); |
| fcd16 = nfcImpl.getFCD16(c); |
| if(fcd16 == 0) { |
| pos = cpLimit; |
| break; |
| } |
| s.append(c); |
| } |
| s.reverse(); |
| if(!normalize(s, errorCode)) { return FALSE; } |
| limit = segmentLimit; |
| start = pos; |
| state = IN_NORMALIZED; |
| pos = normalized.length(); |
| return TRUE; |
| } |
| nextCC = (uint8_t)(fcd16 >> 8); |
| if(pos == 0 || nextCC == 0) { |
| // FCD boundary before the following character. |
| break; |
| } |
| } |
| start = pos; |
| pos = segmentLimit; |
| U_ASSERT(pos != start); |
| state = IN_FCD_SEGMENT; |
| return TRUE; |
| } |
| |
| UBool |
| FCDUTF8CollationIterator::normalize(const UnicodeString &s, UErrorCode &errorCode) { |
| // NFD without argument checking. |
| U_ASSERT(U_SUCCESS(errorCode)); |
| nfcImpl.decompose(s, normalized, errorCode); |
| return U_SUCCESS(errorCode); |
| } |
| |
| U_NAMESPACE_END |
| |
| #endif // !UCONFIG_NO_COLLATION |
