| /* |
| ****************************************************************************** |
| * Copyright (c) 1996-2007, International Business Machines |
| * Corporation and others. All Rights Reserved. |
| ****************************************************************************** |
| * File unorm.cpp |
| * |
| * Created by: Vladimir Weinstein 12052000 |
| * |
| * Modification history : |
| * |
| * Date Name Description |
| * 02/01/01 synwee Added normalization quickcheck enum and method. |
| * 02/12/01 synwee Commented out quickcheck util api has been approved |
| * Added private method for doing FCD checks |
| * 02/23/01 synwee Modified quickcheck and checkFCE to run through |
| * string for codepoints < 0x300 for the normalization |
| * mode NFC. |
| * 05/25/01+ Markus Scherer total rewrite, implement all normalization here |
| * instead of just wrappers around normlzr.cpp, |
| * load unorm.dat, support Unicode 3.1 with |
| * supplementary code points, etc. |
| */ |
| |
| #include "unicode/utypes.h" |
| |
| #if !UCONFIG_NO_NORMALIZATION |
| |
| #include "unicode/udata.h" |
| #include "unicode/uchar.h" |
| #include "unicode/ustring.h" |
| #include "unicode/uiter.h" |
| #include "unicode/uniset.h" |
| #include "unicode/usetiter.h" |
| #include "unicode/unorm.h" |
| #include "ucln_cmn.h" |
| #include "unormimp.h" |
| #include "ucase.h" |
| #include "cmemory.h" |
| #include "umutex.h" |
| #include "utrie.h" |
| #include "unicode/uset.h" |
| #include "udataswp.h" |
| #include "putilimp.h" |
| |
| /* |
| * Status of tailored normalization |
| * |
| * This was done initially for investigation on Unicode public review issue 7 |
| * (http://www.unicode.org/review/). See Jitterbug 2481. |
| * While the UTC at meeting #94 (2003mar) did not take up the issue, this is |
| * a permanent feature in ICU 2.6 in support of IDNA which requires true |
| * Unicode 3.2 normalization. |
| * (NormalizationCorrections are rolled into IDNA mapping tables.) |
| * |
| * Tailored normalization as implemented here allows to "normalize less" |
| * than full Unicode normalization would. |
| * Based internally on a UnicodeSet of code points that are |
| * "excluded from normalization", the normalization functions leave those |
| * code points alone ("inert"). This means that tailored normalization |
| * still transforms text into a canonically equivalent form. |
| * It does not add decompositions to code points that do not have any or |
| * change decomposition results. |
| * |
| * Any function that searches for a safe boundary has not been touched, |
| * which means that these functions will be over-pessimistic when |
| * exclusions are applied. |
| * This should not matter because subsequent checks and normalizations |
| * do apply the exclusions; only a little more of the text may be processed |
| * than necessary under exclusions. |
| * |
| * Normalization exclusions have the following effect on excluded code points c: |
| * - c is not decomposed |
| * - c is not a composition target |
| * - c does not combine forward or backward for composition |
| * except that this is not implemented for Jamo |
| * - c is treated as having a combining class of 0 |
| */ |
| #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) |
| |
| U_NAMESPACE_USE |
| |
| /* |
| * This new implementation of the normalization code loads its data from |
| * unorm.dat, which is generated with the gennorm tool. |
| * The format of that file is described in unormimp.h . |
| */ |
| |
| /* -------------------------------------------------------------------------- */ |
| |
| enum { |
| _STACK_BUFFER_CAPACITY=100 |
| }; |
| |
| /* |
| * Constants for the bit fields in the options bit set parameter. |
| * These need not be public. |
| * A user only needs to know the currently assigned values. |
| * The number and positions of reserved bits per field can remain private |
| * and may change in future implementations. |
| */ |
| enum { |
| _NORM_OPTIONS_NX_MASK=0x1f, |
| _NORM_OPTIONS_UNICODE_MASK=0x60, |
| _NORM_OPTIONS_SETS_MASK=0x7f, |
| |
| _NORM_OPTIONS_UNICODE_SHIFT=5, |
| |
| /* |
| * The following options are used only in some composition functions. |
| * They use bits 12 and up to preserve lower bits for the available options |
| * space in unorm_compare() - |
| * see documentation for UNORM_COMPARE_NORM_OPTIONS_SHIFT. |
| */ |
| |
| /** Options bit 12, for compatibility vs. canonical decomposition. */ |
| _NORM_OPTIONS_COMPAT=0x1000, |
| /** Options bit 13, no discontiguous composition (FCC vs. NFC). */ |
| _NORM_OPTIONS_COMPOSE_CONTIGUOUS=0x2000 |
| }; |
| |
| U_CDECL_BEGIN |
| static inline UBool |
| isHangulWithoutJamoT(UChar c) { |
| c-=HANGUL_BASE; |
| return c<HANGUL_COUNT && c%JAMO_T_COUNT==0; |
| } |
| |
| /* norm32 helpers */ |
| |
| /* is this a norm32 with a regular index? */ |
| static inline UBool |
| isNorm32Regular(uint32_t norm32) { |
| return norm32<_NORM_MIN_SPECIAL; |
| } |
| |
| /* is this a norm32 with a special index for a lead surrogate? */ |
| static inline UBool |
| isNorm32LeadSurrogate(uint32_t norm32) { |
| return _NORM_MIN_SPECIAL<=norm32 && norm32<_NORM_SURROGATES_TOP; |
| } |
| |
| /* is this a norm32 with a special index for a Hangul syllable or a Jamo? */ |
| static inline UBool |
| isNorm32HangulOrJamo(uint32_t norm32) { |
| return norm32>=_NORM_MIN_HANGUL; |
| } |
| |
| /* |
| * Given isNorm32HangulOrJamo(), |
| * is this a Hangul syllable or a Jamo? |
| */ |
| /*static inline UBool |
| isHangulJamoNorm32HangulOrJamoL(uint32_t norm32) { |
| return norm32<_NORM_MIN_JAMO_V; |
| }*/ |
| |
| /* |
| * Given norm32 for Jamo V or T, |
| * is this a Jamo V? |
| */ |
| static inline UBool |
| isJamoVTNorm32JamoV(uint32_t norm32) { |
| return norm32<_NORM_JAMO_V_TOP; |
| } |
| |
| /* load unorm.dat ----------------------------------------------------------- */ |
| |
| /* normTrie: 32-bit trie result may contain a special extraData index with the folding offset */ |
| static int32_t U_CALLCONV |
| getFoldingNormOffset(uint32_t norm32) { |
| if(isNorm32LeadSurrogate(norm32)) { |
| return |
| UTRIE_BMP_INDEX_LENGTH+ |
| (((int32_t)norm32>>(_NORM_EXTRA_SHIFT-UTRIE_SURROGATE_BLOCK_BITS))& |
| (0x3ff<<UTRIE_SURROGATE_BLOCK_BITS)); |
| } else { |
| return 0; |
| } |
| } |
| |
| /* auxTrie: the folding offset is in bits 9..0 of the 16-bit trie result */ |
| static int32_t U_CALLCONV |
| getFoldingAuxOffset(uint32_t data) { |
| return (int32_t)(data&_NORM_AUX_FNC_MASK)<<UTRIE_SURROGATE_BLOCK_BITS; |
| } |
| U_CDECL_END |
| |
| #define UNORM_HARDCODE_DATA 1 |
| |
| #if UNORM_HARDCODE_DATA |
| |
| /* unorm_props_data.c is machine-generated by gennorm --csource */ |
| #include "unorm_props_data.c" |
| |
| static const UBool formatVersion_2_2=TRUE; |
| |
| #else |
| |
| #define DATA_NAME "unorm" |
| #define DATA_TYPE "icu" |
| |
| static UDataMemory *normData=NULL; |
| static UErrorCode dataErrorCode=U_ZERO_ERROR; |
| static int8_t haveNormData=0; |
| |
| static int32_t indexes[_NORM_INDEX_TOP]={ 0 }; |
| static UTrie normTrie={ 0,0,0,0,0,0,0 }, fcdTrie={ 0,0,0,0,0,0,0 }, auxTrie={ 0,0,0,0,0,0,0 }; |
| |
| /* |
| * pointers into the memory-mapped unorm.icu |
| */ |
| static const uint16_t *extraData=NULL, |
| *combiningTable=NULL, |
| *canonStartSets=NULL; |
| |
| static uint8_t formatVersion[4]={ 0, 0, 0, 0 }; |
| static UBool formatVersion_2_1=FALSE, formatVersion_2_2=FALSE; |
| |
| /* the Unicode version of the normalization data */ |
| static UVersionInfo dataVersion={ 0, 0, 0, 0 }; |
| |
| #endif |
| |
| /* cache UnicodeSets for each combination of exclusion flags */ |
| static UnicodeSet *nxCache[_NORM_OPTIONS_SETS_MASK+1]={ NULL }; |
| |
| U_CDECL_BEGIN |
| |
| static UBool U_CALLCONV |
| unorm_cleanup(void) { |
| int32_t i; |
| |
| #if !UNORM_HARDCODE_DATA |
| if(normData!=NULL) { |
| udata_close(normData); |
| normData=NULL; |
| } |
| dataErrorCode=U_ZERO_ERROR; |
| haveNormData=0; |
| #endif |
| |
| for(i=0; i<(int32_t)LENGTHOF(nxCache); ++i) { |
| if (nxCache[i]) { |
| delete nxCache[i]; |
| nxCache[i] = 0; |
| } |
| } |
| |
| return TRUE; |
| } |
| |
| #if !UNORM_HARDCODE_DATA |
| |
| static UBool U_CALLCONV |
| isAcceptable(void * /* context */, |
| const char * /* type */, const char * /* name */, |
| const UDataInfo *pInfo) { |
| if( |
| pInfo->size>=20 && |
| pInfo->isBigEndian==U_IS_BIG_ENDIAN && |
| pInfo->charsetFamily==U_CHARSET_FAMILY && |
| pInfo->dataFormat[0]==0x4e && /* dataFormat="Norm" */ |
| pInfo->dataFormat[1]==0x6f && |
| pInfo->dataFormat[2]==0x72 && |
| pInfo->dataFormat[3]==0x6d && |
| pInfo->formatVersion[0]==2 && |
| pInfo->formatVersion[2]==UTRIE_SHIFT && |
| pInfo->formatVersion[3]==UTRIE_INDEX_SHIFT |
| ) { |
| uprv_memcpy(formatVersion, pInfo->formatVersion, 4); |
| uprv_memcpy(dataVersion, pInfo->dataVersion, 4); |
| return TRUE; |
| } else { |
| return FALSE; |
| } |
| } |
| |
| #endif |
| |
| static UBool U_CALLCONV |
| _enumPropertyStartsRange(const void *context, UChar32 start, UChar32 /*limit*/, uint32_t /*value*/) { |
| /* add the start code point to the USet */ |
| const USetAdder *sa=(const USetAdder *)context; |
| sa->add(sa->set, start); |
| return TRUE; |
| } |
| |
| U_CDECL_END |
| |
| #if !UNORM_HARDCODE_DATA |
| |
| static int8_t |
| loadNormData(UErrorCode &errorCode) { |
| /* load Unicode normalization data from file */ |
| |
| /* |
| * This lazy intialization with double-checked locking (without mutex protection for |
| * haveNormData==0) is transiently unsafe under certain circumstances. |
| * Check the readme and use u_init() if necessary. |
| * |
| * While u_init() initializes the main normalization data via this functions, |
| * it does not do so for exclusion sets (which are fully mutexed). |
| * This is because |
| * - there can be many exclusion sets |
| * - they are rarely used |
| * - they are not usually used in execution paths that are |
| * as performance-sensitive as others |
| * (e.g., IDNA takes more time than unorm_quickCheck() anyway) |
| */ |
| if(haveNormData==0) { |
| UTrie _normTrie={ 0,0,0,0,0,0,0 }, _fcdTrie={ 0,0,0,0,0,0,0 }, _auxTrie={ 0,0,0,0,0,0,0 }; |
| UDataMemory *data; |
| |
| const int32_t *p=NULL; |
| const uint8_t *pb; |
| |
| if(&errorCode==NULL || U_FAILURE(errorCode)) { |
| return 0; |
| } |
| |
| /* open the data outside the mutex block */ |
| data=udata_openChoice(NULL, DATA_TYPE, DATA_NAME, isAcceptable, NULL, &errorCode); |
| dataErrorCode=errorCode; |
| if(U_FAILURE(errorCode)) { |
| return haveNormData=-1; |
| } |
| |
| p=(const int32_t *)udata_getMemory(data); |
| pb=(const uint8_t *)(p+_NORM_INDEX_TOP); |
| utrie_unserialize(&_normTrie, pb, p[_NORM_INDEX_TRIE_SIZE], &errorCode); |
| _normTrie.getFoldingOffset=getFoldingNormOffset; |
| |
| pb+=p[_NORM_INDEX_TRIE_SIZE]+p[_NORM_INDEX_UCHAR_COUNT]*2+p[_NORM_INDEX_COMBINE_DATA_COUNT]*2; |
| if(p[_NORM_INDEX_FCD_TRIE_SIZE]!=0) { |
| utrie_unserialize(&_fcdTrie, pb, p[_NORM_INDEX_FCD_TRIE_SIZE], &errorCode); |
| } |
| pb+=p[_NORM_INDEX_FCD_TRIE_SIZE]; |
| |
| if(p[_NORM_INDEX_AUX_TRIE_SIZE]!=0) { |
| utrie_unserialize(&_auxTrie, pb, p[_NORM_INDEX_AUX_TRIE_SIZE], &errorCode); |
| _auxTrie.getFoldingOffset=getFoldingAuxOffset; |
| } |
| |
| if(U_FAILURE(errorCode)) { |
| dataErrorCode=errorCode; |
| udata_close(data); |
| return haveNormData=-1; |
| } |
| |
| /* in the mutex block, set the data for this process */ |
| umtx_lock(NULL); |
| if(normData==NULL) { |
| normData=data; |
| data=NULL; |
| |
| uprv_memcpy(&indexes, p, sizeof(indexes)); |
| uprv_memcpy(&normTrie, &_normTrie, sizeof(UTrie)); |
| uprv_memcpy(&fcdTrie, &_fcdTrie, sizeof(UTrie)); |
| uprv_memcpy(&auxTrie, &_auxTrie, sizeof(UTrie)); |
| } else { |
| p=(const int32_t *)udata_getMemory(normData); |
| } |
| |
| /* initialize some variables */ |
| extraData=(uint16_t *)((uint8_t *)(p+_NORM_INDEX_TOP)+indexes[_NORM_INDEX_TRIE_SIZE]); |
| combiningTable=extraData+indexes[_NORM_INDEX_UCHAR_COUNT]; |
| formatVersion_2_1=formatVersion[0]>2 || (formatVersion[0]==2 && formatVersion[1]>=1); |
| formatVersion_2_2=formatVersion[0]>2 || (formatVersion[0]==2 && formatVersion[1]>=2); |
| if(formatVersion_2_1) { |
| canonStartSets=combiningTable+ |
| indexes[_NORM_INDEX_COMBINE_DATA_COUNT]+ |
| (indexes[_NORM_INDEX_FCD_TRIE_SIZE]+indexes[_NORM_INDEX_AUX_TRIE_SIZE])/2; |
| } |
| haveNormData=1; |
| ucln_common_registerCleanup(UCLN_COMMON_UNORM, unorm_cleanup); |
| umtx_unlock(NULL); |
| |
| /* if a different thread set it first, then close the extra data */ |
| if(data!=NULL) { |
| udata_close(data); /* NULL if it was set correctly */ |
| } |
| } |
| |
| return haveNormData; |
| } |
| |
| #endif |
| |
| static inline UBool |
| _haveData(UErrorCode &errorCode) { |
| #if UNORM_HARDCODE_DATA |
| return U_SUCCESS(errorCode); |
| #else |
| if(U_FAILURE(errorCode)) { |
| return FALSE; |
| } else if(haveNormData>0) { |
| return TRUE; |
| } else if(haveNormData<0) { |
| errorCode=dataErrorCode; |
| return FALSE; |
| } else /* haveNormData==0 */ { |
| return (UBool)(loadNormData(errorCode)>0); |
| } |
| #endif |
| } |
| |
| U_CAPI UBool U_EXPORT2 |
| unorm_haveData(UErrorCode *pErrorCode) { |
| return _haveData(*pErrorCode); |
| } |
| |
| U_CAPI const uint16_t * U_EXPORT2 |
| unorm_getFCDTrie(UErrorCode *pErrorCode) { |
| if(_haveData(*pErrorCode)) { |
| return fcdTrie.index; |
| } else { |
| return NULL; |
| } |
| } |
| |
| /* data access primitives --------------------------------------------------- */ |
| |
| static inline uint32_t |
| _getNorm32(UChar c) { |
| return UTRIE_GET32_FROM_LEAD(&normTrie, c); |
| } |
| |
| static inline uint32_t |
| _getNorm32FromSurrogatePair(uint32_t norm32, UChar c2) { |
| /* |
| * the surrogate index in norm32 stores only the number of the surrogate index block |
| * see gennorm/store.c/getFoldedNormValue() |
| */ |
| norm32= |
| UTRIE_BMP_INDEX_LENGTH+ |
| ((norm32>>(_NORM_EXTRA_SHIFT-UTRIE_SURROGATE_BLOCK_BITS))& |
| (0x3ff<<UTRIE_SURROGATE_BLOCK_BITS)); |
| return UTRIE_GET32_FROM_OFFSET_TRAIL(&normTrie, norm32, c2); |
| } |
| |
| /* |
| * get a norm32 from text with complete code points |
| * (like from decompositions) |
| */ |
| static inline uint32_t |
| _getNorm32(const UChar *p, uint32_t mask) { |
| uint32_t norm32=_getNorm32(*p); |
| if((norm32&mask) && isNorm32LeadSurrogate(norm32)) { |
| /* *p is a lead surrogate, get the real norm32 */ |
| norm32=_getNorm32FromSurrogatePair(norm32, *(p+1)); |
| } |
| return norm32; |
| } |
| |
| static inline uint16_t |
| _getFCD16(UChar c) { |
| return UTRIE_GET16_FROM_LEAD(&fcdTrie, c); |
| } |
| |
| static inline uint16_t |
| _getFCD16FromSurrogatePair(uint16_t fcd16, UChar c2) { |
| /* the surrogate index in fcd16 is an absolute offset over the start of stage 1 */ |
| return UTRIE_GET16_FROM_OFFSET_TRAIL(&fcdTrie, fcd16, c2); |
| } |
| |
| static inline const uint16_t * |
| _getExtraData(uint32_t norm32) { |
| return extraData+(norm32>>_NORM_EXTRA_SHIFT); |
| } |
| |
| #if 0 |
| /* |
| * It is possible to get the FCD data from the main trie if unorm.icu |
| * was built without the FCD trie, although it is slower. |
| * This is not implemented because it is hard to test, and because it seems |
| * unusual to want to use FCD and not build the data file for it. |
| * |
| * Untested sample code: |
| */ |
| static inline uint16_t |
| _getFCD16FromNormData(UChar32 c) { |
| uint32_t norm32, fcd; |
| |
| norm32=_getNorm32(c); |
| if((norm32&_NORM_QC_NFD) && isNorm32Regular(norm32)) { |
| /* get the lead/trail cc from the decomposition data */ |
| const uint16_t *nfd=_getExtraData(norm32); |
| if(*nfd&_NORM_DECOMP_FLAG_LENGTH_HAS_CC) { |
| fcd=nfd[1]; |
| } |
| } else { |
| fcd=norm32&_NORM_CC_MASK; |
| if(fcd!=0) { |
| /* use the code point cc value for both lead and trail cc's */ |
| fcd|=fcd>>_NORM_CC_SHIFT; /* assume that the cc is in bits 15..8 */ |
| } |
| } |
| |
| return (uint16_t)fcd; |
| } |
| #endif |
| |
| /* normalization exclusion sets --------------------------------------------- */ |
| |
| /* |
| * Normalization exclusion UnicodeSets are used for tailored normalization; |
| * see the comment near the beginning of this file. |
| * |
| * By specifying one or several sets of code points, |
| * those code points become inert for normalization. |
| */ |
| |
| static const UnicodeSet * |
| internalGetNXHangul(UErrorCode &errorCode) { |
| /* internal function, does not check for incoming U_FAILURE */ |
| UBool isCached; |
| |
| UMTX_CHECK(NULL, (UBool)(nxCache[UNORM_NX_HANGUL]!=NULL), isCached); |
| |
| if(!isCached) { |
| UnicodeSet *set=new UnicodeSet(0xac00, 0xd7a3); |
| if(set==NULL) { |
| errorCode=U_MEMORY_ALLOCATION_ERROR; |
| return NULL; |
| } |
| // Compact the set for caching. |
| set->compact(); |
| |
| umtx_lock(NULL); |
| if(nxCache[UNORM_NX_HANGUL]==NULL) { |
| nxCache[UNORM_NX_HANGUL]=set; |
| set=NULL; |
| ucln_common_registerCleanup(UCLN_COMMON_UNORM, unorm_cleanup); |
| } |
| umtx_unlock(NULL); |
| |
| delete set; |
| } |
| |
| return nxCache[UNORM_NX_HANGUL]; |
| } |
| |
| /* unorm.cpp 1.116 had and used |
| static const UnicodeSet * |
| internalGetNXFromPattern(int32_t options, const char *pattern, UErrorCode &errorCode) { |
| ... |
| } |
| */ |
| |
| /* get and set an exclusion set from a serialized UnicodeSet */ |
| static const UnicodeSet * |
| internalGetSerializedNX(int32_t options, int32_t nxIndex, UErrorCode &errorCode) { |
| /* internal function, does not check for incoming U_FAILURE */ |
| UBool isCached; |
| |
| UMTX_CHECK(NULL, (UBool)(nxCache[options]!=NULL), isCached); |
| |
| if( !isCached && |
| canonStartSets!=NULL && |
| canonStartSets[nxIndex]!=0 && canonStartSets[nxIndex+1]>canonStartSets[nxIndex] |
| ) { |
| USerializedSet sset; |
| UnicodeSet *set; |
| UChar32 start, end; |
| int32_t i; |
| |
| if( !uset_getSerializedSet( |
| &sset, |
| canonStartSets+canonStartSets[nxIndex], |
| canonStartSets[nxIndex+1]-canonStartSets[nxIndex]) |
| ) { |
| errorCode=U_INVALID_FORMAT_ERROR; |
| return NULL; |
| } |
| |
| /* turn the serialized set into a UnicodeSet */ |
| set=new UnicodeSet(); |
| if(set==NULL) { |
| errorCode=U_MEMORY_ALLOCATION_ERROR; |
| return NULL; |
| } |
| for(i=0; uset_getSerializedRange(&sset, i, &start, &end); ++i) { |
| set->add(start, end); |
| } |
| // Compact the set for caching. |
| set->compact(); |
| |
| umtx_lock(NULL); |
| if(nxCache[options]==NULL) { |
| nxCache[options]=set; |
| set=NULL; |
| ucln_common_registerCleanup(UCLN_COMMON_UNORM, unorm_cleanup); |
| } |
| umtx_unlock(NULL); |
| |
| delete set; |
| } |
| |
| return nxCache[options]; |
| } |
| |
| static const UnicodeSet * |
| internalGetNXCJKCompat(UErrorCode &errorCode) { |
| /* build a set from [[:Ideographic:]&[:NFD_QC=No:]]=[CJK Ideographs]&[has canonical decomposition] */ |
| return internalGetSerializedNX( |
| UNORM_NX_CJK_COMPAT, |
| _NORM_SET_INDEX_NX_CJK_COMPAT_OFFSET, |
| errorCode); |
| } |
| |
| static const UnicodeSet * |
| internalGetNXUnicode(uint32_t options, UErrorCode &errorCode) { |
| /* internal function, does not check for incoming U_FAILURE */ |
| int32_t nxIndex; |
| |
| options&=_NORM_OPTIONS_UNICODE_MASK; |
| switch(options) { |
| case 0: |
| return NULL; |
| case UNORM_UNICODE_3_2: |
| /* [:^Age=3.2:] */ |
| nxIndex=_NORM_SET_INDEX_NX_UNICODE32_OFFSET; |
| break; |
| default: |
| errorCode=U_ILLEGAL_ARGUMENT_ERROR; |
| return NULL; |
| } |
| |
| /* build a set with all code points that were not designated by the specified Unicode version */ |
| return internalGetSerializedNX(options, nxIndex, errorCode); |
| } |
| |
| /* Get a decomposition exclusion set. The data must be loaded. */ |
| static const UnicodeSet * |
| internalGetNX(int32_t options, UErrorCode &errorCode) { |
| options&=_NORM_OPTIONS_SETS_MASK; |
| |
| UBool isCached; |
| |
| UMTX_CHECK(NULL, (UBool)(nxCache[options]!=NULL), isCached); |
| |
| if(!isCached) { |
| /* return basic sets */ |
| if(options==UNORM_NX_HANGUL) { |
| return internalGetNXHangul(errorCode); |
| } |
| if(options==UNORM_NX_CJK_COMPAT) { |
| return internalGetNXCJKCompat(errorCode); |
| } |
| if((options&_NORM_OPTIONS_UNICODE_MASK)!=0 && (options&_NORM_OPTIONS_NX_MASK)==0) { |
| return internalGetNXUnicode(options, errorCode); |
| } |
| |
| /* build a set from multiple subsets */ |
| UnicodeSet *set; |
| const UnicodeSet *other; |
| |
| set=new UnicodeSet(); |
| if(set==NULL) { |
| errorCode=U_MEMORY_ALLOCATION_ERROR; |
| return NULL; |
| } |
| |
| if((options&UNORM_NX_HANGUL)!=0 && NULL!=(other=internalGetNXHangul(errorCode))) { |
| set->addAll(*other); |
| } |
| if((options&UNORM_NX_CJK_COMPAT)!=0 && NULL!=(other=internalGetNXCJKCompat(errorCode))) { |
| set->addAll(*other); |
| } |
| if((options&_NORM_OPTIONS_UNICODE_MASK)!=0 && NULL!=(other=internalGetNXUnicode(options, errorCode))) { |
| set->addAll(*other); |
| } |
| |
| if(U_FAILURE(errorCode)) { |
| delete set; |
| return NULL; |
| } |
| // Compact the set for caching. |
| set->compact(); |
| |
| umtx_lock(NULL); |
| if(nxCache[options]==NULL) { |
| nxCache[options]=set; |
| set=NULL; |
| ucln_common_registerCleanup(UCLN_COMMON_UNORM, unorm_cleanup); |
| } |
| umtx_unlock(NULL); |
| |
| delete set; |
| } |
| |
| return nxCache[options]; |
| } |
| |
| static inline const UnicodeSet * |
| getNX(int32_t options, UErrorCode &errorCode) { |
| if(U_FAILURE(errorCode) || (options&=_NORM_OPTIONS_SETS_MASK)==0) { |
| /* incoming failure, or no decomposition exclusions requested */ |
| return NULL; |
| } else { |
| return internalGetNX(options, errorCode); |
| } |
| } |
| |
| U_CFUNC const UnicodeSet * |
| unorm_getNX(int32_t options, UErrorCode *pErrorCode) { |
| return getNX(options, *pErrorCode); |
| } |
| |
| static inline UBool |
| nx_contains(const UnicodeSet *nx, UChar32 c) { |
| return nx!=NULL && nx->contains(c); |
| } |
| |
| static inline UBool |
| nx_contains(const UnicodeSet *nx, UChar c, UChar c2) { |
| return nx!=NULL && nx->contains(c2==0 ? c : U16_GET_SUPPLEMENTARY(c, c2)); |
| } |
| |
| /* other normalization primitives ------------------------------------------- */ |
| |
| /* get the canonical or compatibility decomposition for one character */ |
| static inline const UChar * |
| _decompose(uint32_t norm32, uint32_t qcMask, int32_t &length, |
| uint8_t &cc, uint8_t &trailCC) { |
| const UChar *p=(const UChar *)_getExtraData(norm32); |
| length=*p++; |
| |
| if((norm32&qcMask&_NORM_QC_NFKD)!=0 && length>=0x100) { |
| /* use compatibility decomposition, skip canonical data */ |
| p+=((length>>7)&1)+(length&_NORM_DECOMP_LENGTH_MASK); |
| length>>=8; |
| } |
| |
| if(length&_NORM_DECOMP_FLAG_LENGTH_HAS_CC) { |
| /* get the lead and trail cc's */ |
| UChar bothCCs=*p++; |
| cc=(uint8_t)(bothCCs>>8); |
| trailCC=(uint8_t)bothCCs; |
| } else { |
| /* lead and trail cc's are both 0 */ |
| cc=trailCC=0; |
| } |
| |
| length&=_NORM_DECOMP_LENGTH_MASK; |
| return p; |
| } |
| |
| /* get the canonical decomposition for one character */ |
| static inline const UChar * |
| _decompose(uint32_t norm32, int32_t &length, |
| uint8_t &cc, uint8_t &trailCC) { |
| const UChar *p=(const UChar *)_getExtraData(norm32); |
| length=*p++; |
| |
| if(length&_NORM_DECOMP_FLAG_LENGTH_HAS_CC) { |
| /* get the lead and trail cc's */ |
| UChar bothCCs=*p++; |
| cc=(uint8_t)(bothCCs>>8); |
| trailCC=(uint8_t)bothCCs; |
| } else { |
| /* lead and trail cc's are both 0 */ |
| cc=trailCC=0; |
| } |
| |
| length&=_NORM_DECOMP_LENGTH_MASK; |
| return p; |
| } |
| |
| /** |
| * Get the canonical decomposition for one code point. |
| * @param c code point |
| * @param buffer out-only buffer for algorithmic decompositions of Hangul |
| * @param length out-only, takes the length of the decomposition, if any |
| * @return pointer to decomposition, or 0 if none |
| * @internal |
| */ |
| U_CFUNC const UChar * |
| unorm_getCanonicalDecomposition(UChar32 c, UChar buffer[4], int32_t *pLength) { |
| uint32_t norm32; |
| |
| if(c<indexes[_NORM_INDEX_MIN_NFD_NO_MAYBE]) { |
| /* trivial case */ |
| return NULL; |
| } |
| |
| UTRIE_GET32(&normTrie, c, norm32); |
| if(norm32&_NORM_QC_NFD) { |
| if(isNorm32HangulOrJamo(norm32)) { |
| /* Hangul syllable: decompose algorithmically */ |
| UChar c2; |
| |
| c-=HANGUL_BASE; |
| |
| c2=(UChar)(c%JAMO_T_COUNT); |
| c/=JAMO_T_COUNT; |
| if(c2>0) { |
| buffer[2]=(UChar)(JAMO_T_BASE+c2); |
| *pLength=3; |
| } else { |
| *pLength=2; |
| } |
| |
| buffer[1]=(UChar)(JAMO_V_BASE+c%JAMO_V_COUNT); |
| buffer[0]=(UChar)(JAMO_L_BASE+c/JAMO_V_COUNT); |
| return buffer; |
| } else { |
| /* normal decomposition */ |
| uint8_t cc, trailCC; |
| return _decompose(norm32, *pLength, cc, trailCC); |
| } |
| } else { |
| return 0; |
| } |
| } |
| |
| /* |
| * get the combining class of (c, c2)=*p++ |
| * before: p<limit after: p<=limit |
| * if only one code unit is used, then c2==0 |
| */ |
| static inline uint8_t |
| _getNextCC(const UChar *&p, const UChar *limit, UChar &c, UChar &c2) { |
| uint32_t norm32; |
| |
| c=*p++; |
| norm32=_getNorm32(c); |
| if((norm32&_NORM_CC_MASK)==0) { |
| c2=0; |
| return 0; |
| } else { |
| if(!isNorm32LeadSurrogate(norm32)) { |
| c2=0; |
| } else { |
| /* c is a lead surrogate, get the real norm32 */ |
| if(p!=limit && UTF_IS_SECOND_SURROGATE(c2=*p)) { |
| ++p; |
| norm32=_getNorm32FromSurrogatePair(norm32, c2); |
| } else { |
| c2=0; |
| return 0; |
| } |
| } |
| |
| return (uint8_t)(norm32>>_NORM_CC_SHIFT); |
| } |
| } |
| |
| /* |
| * read backwards and get norm32 |
| * return 0 if the character is <minC |
| * if c2!=0 then (c2, c) is a surrogate pair (reversed - c2 is first surrogate but read second!) |
| */ |
| static inline uint32_t |
| _getPrevNorm32(const UChar *start, const UChar *&src, |
| uint32_t minC, uint32_t mask, |
| UChar &c, UChar &c2) { |
| uint32_t norm32; |
| |
| c=*--src; |
| c2=0; |
| |
| /* check for a surrogate before getting norm32 to see if we need to predecrement further */ |
| if(c<minC) { |
| return 0; |
| } else if(!UTF_IS_SURROGATE(c)) { |
| return _getNorm32(c); |
| } else if(UTF_IS_SURROGATE_FIRST(c)) { |
| /* unpaired first surrogate */ |
| return 0; |
| } else if(src!=start && UTF_IS_FIRST_SURROGATE(c2=*(src-1))) { |
| --src; |
| norm32=_getNorm32(c2); |
| |
| if((norm32&mask)==0) { |
| /* all surrogate pairs with this lead surrogate have only irrelevant data */ |
| return 0; |
| } else { |
| /* norm32 must be a surrogate special */ |
| return _getNorm32FromSurrogatePair(norm32, c); |
| } |
| } else { |
| /* unpaired second surrogate */ |
| c2=0; |
| return 0; |
| } |
| } |
| |
| /* |
| * get the combining class of (c, c2)=*--p |
| * before: start<p after: start<=p |
| */ |
| static inline uint8_t |
| _getPrevCC(const UChar *start, const UChar *&p) { |
| UChar c, c2; |
| |
| return (uint8_t)(_getPrevNorm32(start, p, _NORM_MIN_WITH_LEAD_CC, _NORM_CC_MASK, c, c2)>>_NORM_CC_SHIFT); |
| } |
| |
| /* |
| * is this a safe boundary character for NF*D? |
| * (lead cc==0) |
| */ |
| static inline UBool |
| _isNFDSafe(uint32_t norm32, uint32_t ccOrQCMask, uint32_t decompQCMask) { |
| if((norm32&ccOrQCMask)==0) { |
| return TRUE; /* cc==0 and no decomposition: this is NF*D safe */ |
| } |
| |
| /* inspect its decomposition - maybe a Hangul but not a surrogate here */ |
| if(isNorm32Regular(norm32) && (norm32&decompQCMask)!=0) { |
| int32_t length; |
| uint8_t cc, trailCC; |
| |
| /* decomposes, get everything from the variable-length extra data */ |
| _decompose(norm32, decompQCMask, length, cc, trailCC); |
| return cc==0; |
| } else { |
| /* no decomposition (or Hangul), test the cc directly */ |
| return (norm32&_NORM_CC_MASK)==0; |
| } |
| } |
| |
| /* |
| * is this (or does its decomposition begin with) a "true starter"? |
| * (cc==0 and NF*C_YES) |
| */ |
| static inline UBool |
| _isTrueStarter(uint32_t norm32, uint32_t ccOrQCMask, uint32_t decompQCMask) { |
| if((norm32&ccOrQCMask)==0) { |
| return TRUE; /* this is a true starter (could be Hangul or Jamo L) */ |
| } |
| |
| /* inspect its decomposition - not a Hangul or a surrogate here */ |
| if((norm32&decompQCMask)!=0) { |
| const UChar *p; |
| int32_t length; |
| uint8_t cc, trailCC; |
| |
| /* decomposes, get everything from the variable-length extra data */ |
| p=_decompose(norm32, decompQCMask, length, cc, trailCC); |
| if(cc==0) { |
| uint32_t qcMask=ccOrQCMask&_NORM_QC_MASK; |
| |
| /* does it begin with NFC_YES? */ |
| if((_getNorm32(p, qcMask)&qcMask)==0) { |
| /* yes, the decomposition begins with a true starter */ |
| return TRUE; |
| } |
| } |
| } |
| return FALSE; |
| } |
| |
| /* uchar.h */ |
| U_CAPI uint8_t U_EXPORT2 |
| u_getCombiningClass(UChar32 c) { |
| #if !UNORM_HARDCODE_DATA |
| UErrorCode errorCode=U_ZERO_ERROR; |
| if(_haveData(errorCode)) { |
| #endif |
| uint32_t norm32; |
| |
| UTRIE_GET32(&normTrie, c, norm32); |
| return (uint8_t)(norm32>>_NORM_CC_SHIFT); |
| #if !UNORM_HARDCODE_DATA |
| } else { |
| return 0; |
| } |
| #endif |
| } |
| |
| U_CFUNC UBool U_EXPORT2 |
| unorm_internalIsFullCompositionExclusion(UChar32 c) { |
| #if UNORM_HARDCODE_DATA |
| if(auxTrie.index!=NULL) { |
| #else |
| UErrorCode errorCode=U_ZERO_ERROR; |
| if(_haveData(errorCode) && auxTrie.index!=NULL) { |
| #endif |
| uint16_t aux; |
| |
| UTRIE_GET16(&auxTrie, c, aux); |
| return (UBool)((aux&_NORM_AUX_COMP_EX_MASK)!=0); |
| } else { |
| return FALSE; |
| } |
| } |
| |
| U_CFUNC UBool U_EXPORT2 |
| unorm_isCanonSafeStart(UChar32 c) { |
| #if UNORM_HARDCODE_DATA |
| if(auxTrie.index!=NULL) { |
| #else |
| UErrorCode errorCode=U_ZERO_ERROR; |
| if(_haveData(errorCode) && auxTrie.index!=NULL) { |
| #endif |
| uint16_t aux; |
| |
| UTRIE_GET16(&auxTrie, c, aux); |
| return (UBool)((aux&_NORM_AUX_UNSAFE_MASK)==0); |
| } else { |
| return FALSE; |
| } |
| } |
| |
| U_CAPI void U_EXPORT2 |
| unorm_getUnicodeVersion(UVersionInfo *versionInfo, UErrorCode *pErrorCode){ |
| if(unorm_haveData(pErrorCode)){ |
| uprv_memcpy(*versionInfo, dataVersion, 4); |
| } |
| } |
| |
| |
| U_CAPI UBool U_EXPORT2 |
| unorm_getCanonStartSet(UChar32 c, USerializedSet *fillSet) { |
| #if !UNORM_HARDCODE_DATA |
| UErrorCode errorCode=U_ZERO_ERROR; |
| #endif |
| if( fillSet!=NULL && (uint32_t)c<=0x10ffff && |
| #if !UNORM_HARDCODE_DATA |
| _haveData(errorCode) && |
| #endif |
| canonStartSets!=NULL |
| ) { |
| const uint16_t *table; |
| int32_t i, start, limit; |
| |
| /* |
| * binary search for c |
| * |
| * There are two search tables, |
| * one for BMP code points and one for supplementary ones. |
| * See unormimp.h for details. |
| */ |
| if(c<=0xffff) { |
| table=canonStartSets+canonStartSets[_NORM_SET_INDEX_CANON_SETS_LENGTH]; |
| start=0; |
| limit=canonStartSets[_NORM_SET_INDEX_CANON_BMP_TABLE_LENGTH]; |
| |
| /* each entry is a pair { c, result } */ |
| while(start<limit-2) { |
| i=(uint16_t)(((start+limit)/4)*2); /* (start+limit)/2 and address pairs */ |
| if(c<table[i]) { |
| limit=i; |
| } else { |
| start=i; |
| } |
| } |
| |
| /* found? */ |
| if(c==table[start]) { |
| i=table[start+1]; |
| if((i&_NORM_CANON_SET_BMP_MASK)==_NORM_CANON_SET_BMP_IS_INDEX) { |
| /* result 01xxxxxx xxxxxx contains index x to a USerializedSet */ |
| i&=(_NORM_MAX_CANON_SETS-1); |
| return uset_getSerializedSet(fillSet, |
| canonStartSets+i, |
| canonStartSets[_NORM_SET_INDEX_CANON_SETS_LENGTH]-i); |
| } else { |
| /* other result values are BMP code points for single-code point sets */ |
| uset_setSerializedToOne(fillSet, (UChar32)i); |
| return TRUE; |
| } |
| } |
| } else { |
| uint16_t high, low, h; |
| |
| table=canonStartSets+canonStartSets[_NORM_SET_INDEX_CANON_SETS_LENGTH]+ |
| canonStartSets[_NORM_SET_INDEX_CANON_BMP_TABLE_LENGTH]; |
| start=0; |
| limit=canonStartSets[_NORM_SET_INDEX_CANON_SUPP_TABLE_LENGTH]; |
| |
| high=(uint16_t)(c>>16); |
| low=(uint16_t)c; |
| |
| /* each entry is a triplet { high(c), low(c), result } */ |
| while(start<limit-3) { |
| i=(uint16_t)(((start+limit)/6)*3); /* (start+limit)/2 and address triplets */ |
| h=table[i]&0x1f; /* high word */ |
| if(high<h || (high==h && low<table[i+1])) { |
| limit=i; |
| } else { |
| start=i; |
| } |
| } |
| |
| /* found? */ |
| h=table[start]; |
| if(high==(h&0x1f) && low==table[start+1]) { |
| i=table[start+2]; |
| if((h&0x8000)==0) { |
| /* the result is an index to a USerializedSet */ |
| return uset_getSerializedSet(fillSet, |
| canonStartSets+i, |
| canonStartSets[_NORM_SET_INDEX_CANON_SETS_LENGTH]-i); |
| } else { |
| /* |
| * single-code point set {x} in |
| * triplet { 100xxxxx 000hhhhh llllllll llllllll xxxxxxxx xxxxxxxx } |
| */ |
| i|=((int32_t)h&0x1f00)<<8; /* add high bits from high(c) */ |
| uset_setSerializedToOne(fillSet, (UChar32)i); |
| return TRUE; |
| } |
| } |
| } |
| } |
| |
| return FALSE; /* not found */ |
| } |
| |
| U_CAPI int32_t U_EXPORT2 |
| u_getFC_NFKC_Closure(UChar32 c, UChar *dest, int32_t destCapacity, UErrorCode *pErrorCode) { |
| uint16_t aux; |
| |
| if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { |
| return 0; |
| } |
| if(destCapacity<0 || (dest==NULL && destCapacity>0)) { |
| *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; |
| return 0; |
| } |
| if(!_haveData(*pErrorCode) || auxTrie.index==NULL) { |
| return 0; |
| } |
| |
| UTRIE_GET16(&auxTrie, c, aux); |
| aux&=_NORM_AUX_FNC_MASK; |
| if(aux!=0) { |
| const UChar *s; |
| int32_t length; |
| |
| s=(const UChar *)(extraData+aux); |
| if(*s<0xff00) { |
| /* s points to the single-unit string */ |
| length=1; |
| } else { |
| length=*s&0xff; |
| ++s; |
| } |
| if(0<length && length<=destCapacity) { |
| uprv_memcpy(dest, s, length*U_SIZEOF_UCHAR); |
| } |
| return u_terminateUChars(dest, destCapacity, length, pErrorCode); |
| } else { |
| return u_terminateUChars(dest, destCapacity, 0, pErrorCode); |
| } |
| } |
| |
| /* Is c an NF<mode>-skippable code point? See unormimp.h. */ |
| U_CAPI UBool U_EXPORT2 |
| unorm_isNFSkippable(UChar32 c, UNormalizationMode mode) { |
| uint32_t norm32, mask; |
| uint16_t aux, fcd; |
| |
| #if !UNORM_HARDCODE_DATA |
| UErrorCode errorCode=U_ZERO_ERROR; |
| if(!_haveData(errorCode)) { |
| return FALSE; |
| } |
| #endif |
| |
| /* handle trivial cases; set the comparison mask for the normal ones */ |
| switch(mode) { |
| case UNORM_NONE: |
| return TRUE; |
| case UNORM_NFD: |
| mask=_NORM_CC_MASK|_NORM_QC_NFD; |
| break; |
| case UNORM_NFKD: |
| mask=_NORM_CC_MASK|_NORM_QC_NFKD; |
| break; |
| case UNORM_NFC: |
| /* case UNORM_FCC: */ |
| mask=_NORM_CC_MASK|_NORM_COMBINES_ANY|(_NORM_QC_NFC&_NORM_QC_ANY_NO); |
| break; |
| case UNORM_NFKC: |
| mask=_NORM_CC_MASK|_NORM_COMBINES_ANY|(_NORM_QC_NFKC&_NORM_QC_ANY_NO); |
| break; |
| case UNORM_FCD: |
| /* FCD: skippable if lead cc==0 and trail cc<=1 */ |
| if(fcdTrie.index!=NULL) { |
| UTRIE_GET16(&fcdTrie, c, fcd); |
| return fcd<=1; |
| } else { |
| return FALSE; |
| } |
| default: |
| return FALSE; |
| } |
| |
| /* check conditions (a)..(e), see unormimp.h */ |
| UTRIE_GET32(&normTrie, c, norm32); |
| if((norm32&mask)!=0) { |
| return FALSE; /* fails (a)..(e), not skippable */ |
| } |
| |
| if(mode<UNORM_NFC) { |
| return TRUE; /* NF*D, passed (a)..(c), is skippable */ |
| } |
| |
| /* NF*C/FCC, passed (a)..(e) */ |
| if((norm32&_NORM_QC_NFD)==0) { |
| return TRUE; /* no canonical decomposition, is skippable */ |
| } |
| |
| /* check Hangul syllables algorithmically */ |
| if(isNorm32HangulOrJamo(norm32)) { |
| /* Jamo passed (a)..(e) above, must be Hangul */ |
| return !isHangulWithoutJamoT((UChar)c); /* LVT are skippable, LV are not */ |
| } |
| |
| /* if(mode<=UNORM_NFKC) { -- enable when implementing FCC */ |
| /* NF*C, test (f) flag */ |
| if(!formatVersion_2_2 || auxTrie.index==NULL) { |
| return FALSE; /* no (f) data, say not skippable to be safe */ |
| } |
| |
| UTRIE_GET16(&auxTrie, c, aux); |
| return (aux&_NORM_AUX_NFC_SKIP_F_MASK)==0; /* TRUE=skippable if the (f) flag is not set */ |
| |
| /* } else { FCC, test fcd<=1 instead of the above } */ |
| } |
| |
| U_CAPI void U_EXPORT2 |
| unorm_addPropertyStarts(const USetAdder *sa, UErrorCode *pErrorCode) { |
| UChar c; |
| |
| if(!_haveData(*pErrorCode)) { |
| return; |
| } |
| |
| /* add the start code point of each same-value range of each trie */ |
| utrie_enum(&normTrie, NULL, _enumPropertyStartsRange, sa); |
| if(fcdTrie.index!=NULL) { |
| utrie_enum(&fcdTrie, NULL, _enumPropertyStartsRange, sa); |
| } |
| if(auxTrie.index!=NULL) { |
| utrie_enum(&auxTrie, NULL, _enumPropertyStartsRange, sa); |
| } |
| |
| /* add Hangul LV syllables and LV+1 because of skippables */ |
| for(c=HANGUL_BASE; c<HANGUL_BASE+HANGUL_COUNT; c+=JAMO_T_COUNT) { |
| sa->add(sa->set, c); |
| sa->add(sa->set, c+1); |
| } |
| sa->add(sa->set, HANGUL_BASE+HANGUL_COUNT); /* add Hangul+1 to continue with other properties */ |
| } |
| |
| U_CFUNC UNormalizationCheckResult U_EXPORT2 |
| unorm_getQuickCheck(UChar32 c, UNormalizationMode mode) { |
| static const uint32_t qcMask[UNORM_MODE_COUNT]={ |
| 0, 0, _NORM_QC_NFD, _NORM_QC_NFKD, _NORM_QC_NFC, _NORM_QC_NFKC |
| }; |
| |
| uint32_t norm32; |
| |
| #if !UNORM_HARDCODE_DATA |
| UErrorCode errorCode=U_ZERO_ERROR; |
| if(!_haveData(errorCode)) { |
| return UNORM_YES; |
| } |
| #endif |
| |
| UTRIE_GET32(&normTrie, c, norm32); |
| norm32&=qcMask[mode]; |
| |
| if(norm32==0) { |
| return UNORM_YES; |
| } else if(norm32&_NORM_QC_ANY_NO) { |
| return UNORM_NO; |
| } else /* _NORM_QC_ANY_MAYBE */ { |
| return UNORM_MAYBE; |
| } |
| } |
| |
| U_CFUNC uint16_t U_EXPORT2 |
| unorm_getFCD16FromCodePoint(UChar32 c) { |
| uint16_t fcd; |
| #if !UNORM_HARDCODE_DATA |
| UErrorCode errorCode; |
| errorCode=U_ZERO_ERROR; |
| #endif |
| |
| if( |
| #if !UNORM_HARDCODE_DATA |
| !_haveData(errorCode) || |
| #endif |
| fcdTrie.index==NULL |
| ) { |
| return 0; |
| } |
| |
| UTRIE_GET16(&fcdTrie, c, fcd); |
| return fcd; |
| } |
| |
| /* reorder UTF-16 in-place -------------------------------------------------- */ |
| |
| /* |
| * simpler, single-character version of _mergeOrdered() - |
| * bubble-insert one single code point into the preceding string |
| * which is already canonically ordered |
| * (c, c2) may or may not yet have been inserted at [current..p[ |
| * |
| * it must be p=current+lengthof(c, c2) i.e. p=current+(c2==0 ? 1 : 2) |
| * |
| * before: [start..current[ is already ordered, and |
| * [current..p[ may or may not hold (c, c2) but |
| * must be exactly the same length as (c, c2) |
| * after: [start..p[ is ordered |
| * |
| * returns the trailing combining class |
| */ |
| static uint8_t |
| _insertOrdered(const UChar *start, UChar *current, UChar *p, |
| UChar c, UChar c2, uint8_t cc) { |
| const UChar *pBack, *pPreBack; |
| UChar *r; |
| uint8_t prevCC, trailCC=cc; |
| |
| if(start<current && cc!=0) { |
| /* search for the insertion point where cc>=prevCC */ |
| pPreBack=pBack=current; |
| prevCC=_getPrevCC(start, pPreBack); |
| if(cc<prevCC) { |
| /* this will be the last code point, so keep its cc */ |
| trailCC=prevCC; |
| pBack=pPreBack; |
| while(start<pPreBack) { |
| prevCC=_getPrevCC(start, pPreBack); |
| if(cc>=prevCC) { |
| break; |
| } |
| pBack=pPreBack; |
| } |
| |
| /* |
| * this is where we are right now with all these pointers: |
| * [start..pPreBack[ 0..? code points that we can ignore |
| * [pPreBack..pBack[ 0..1 code points with prevCC<=cc |
| * [pBack..current[ 0..n code points with >cc, move up to insert (c, c2) |
| * [current..p[ 1 code point (c, c2) with cc |
| */ |
| |
| /* move the code units in between up */ |
| r=p; |
| do { |
| *--r=*--current; |
| } while(pBack!=current); |
| } |
| } |
| |
| /* insert (c, c2) */ |
| *current=c; |
| if(c2!=0) { |
| *(current+1)=c2; |
| } |
| |
| /* we know the cc of the last code point */ |
| return trailCC; |
| } |
| |
| /* |
| * merge two UTF-16 string parts together |
| * to canonically order (order by combining classes) their concatenation |
| * |
| * the two strings may already be adjacent, so that the merging is done in-place |
| * if the two strings are not adjacent, then the buffer holding the first one |
| * must be large enough |
| * the second string may or may not be ordered in itself |
| * |
| * before: [start..current[ is already ordered, and |
| * [next..limit[ may be ordered in itself, but |
| * is not in relation to [start..current[ |
| * after: [start..current+(limit-next)[ is ordered |
| * |
| * the algorithm is a simple bubble-sort that takes the characters from *next++ |
| * and inserts them in correct combining class order into the preceding part |
| * of the string |
| * |
| * since this function is called much less often than the single-code point |
| * _insertOrdered(), it just uses that for easier maintenance |
| * (see file version from before 2001aug31 for a more optimized version) |
| * |
| * returns the trailing combining class |
| */ |
| static uint8_t |
| _mergeOrdered(UChar *start, UChar *current, |
| const UChar *next, const UChar *limit, UBool isOrdered=TRUE) { |
| UChar *r; |
| UChar c, c2; |
| uint8_t cc, trailCC=0; |
| UBool adjacent; |
| |
| adjacent= current==next; |
| |
| if(start!=current || !isOrdered) { |
| while(next<limit) { |
| cc=_getNextCC(next, limit, c, c2); |
| if(cc==0) { |
| /* does not bubble back */ |
| trailCC=0; |
| if(adjacent) { |
| current=(UChar *)next; |
| } else { |
| *current++=c; |
| if(c2!=0) { |
| *current++=c2; |
| } |
| } |
| if(isOrdered) { |
| break; |
| } else { |
| start=current; |
| } |
| } else { |
| r=current+(c2==0 ? 1 : 2); |
| trailCC=_insertOrdered(start, current, r, c, c2, cc); |
| current=r; |
| } |
| } |
| } |
| |
| if(next==limit) { |
| /* we know the cc of the last code point */ |
| return trailCC; |
| } else { |
| if(!adjacent) { |
| /* copy the second string part */ |
| do { |
| *current++=*next++; |
| } while(next!=limit); |
| limit=current; |
| } |
| return _getPrevCC(start, limit); |
| } |
| } |
| |
| /* find the last true starter in [start..src[ and return the pointer to it */ |
| static const UChar * |
| _findPreviousStarter(const UChar *start, const UChar *src, |
| uint32_t ccOrQCMask, uint32_t decompQCMask, UChar minNoMaybe) { |
| uint32_t norm32; |
| UChar c, c2; |
| |
| while(start<src) { |
| norm32=_getPrevNorm32(start, src, minNoMaybe, ccOrQCMask|decompQCMask, c, c2); |
| if(_isTrueStarter(norm32, ccOrQCMask, decompQCMask)) { |
| break; |
| } |
| } |
| return src; |
| } |
| |
| /* find the first true starter in [src..limit[ and return the pointer to it */ |
| static const UChar * |
| _findNextStarter(const UChar *src, const UChar *limit, |
| uint32_t qcMask, uint32_t decompQCMask, UChar minNoMaybe) { |
| const UChar *p; |
| uint32_t norm32, ccOrQCMask; |
| int32_t length; |
| UChar c, c2; |
| uint8_t cc, trailCC; |
| |
| ccOrQCMask=_NORM_CC_MASK|qcMask; |
| |
| for(;;) { |
| if(src==limit) { |
| break; /* end of string */ |
| } |
| c=*src; |
| if(c<minNoMaybe) { |
| break; /* catches NUL terminater, too */ |
| } |
| |
| norm32=_getNorm32(c); |
| if((norm32&ccOrQCMask)==0) { |
| break; /* true starter */ |
| } |
| |
| if(isNorm32LeadSurrogate(norm32)) { |
| /* c is a lead surrogate, get the real norm32 */ |
| if((src+1)==limit || !UTF_IS_SECOND_SURROGATE(c2=*(src+1))) { |
| break; /* unmatched first surrogate: counts as a true starter */ |
| } |
| norm32=_getNorm32FromSurrogatePair(norm32, c2); |
| |
| if((norm32&ccOrQCMask)==0) { |
| break; /* true starter */ |
| } |
| } else { |
| c2=0; |
| } |
| |
| /* (c, c2) is not a true starter but its decomposition may be */ |
| if(norm32&decompQCMask) { |
| /* (c, c2) decomposes, get everything from the variable-length extra data */ |
| p=_decompose(norm32, decompQCMask, length, cc, trailCC); |
| |
| /* get the first character's norm32 to check if it is a true starter */ |
| if(cc==0 && (_getNorm32(p, qcMask)&qcMask)==0) { |
| break; /* true starter */ |
| } |
| } |
| |
| src+= c2==0 ? 1 : 2; /* not a true starter, continue */ |
| } |
| |
| return src; |
| } |
| |
| /* make NFD & NFKD ---------------------------------------------------------- */ |
| |
| U_CAPI int32_t U_EXPORT2 |
| unorm_getDecomposition(UChar32 c, UBool compat, |
| UChar *dest, int32_t destCapacity) { |
| #if !UNORM_HARDCODE_DATA |
| UErrorCode errorCode=U_ZERO_ERROR; |
| #endif |
| if( (uint32_t)c<=0x10ffff && |
| #if !UNORM_HARDCODE_DATA |
| _haveData(errorCode) && |
| #endif |
| ((dest!=NULL && destCapacity>0) || destCapacity==0) |
| ) { |
| uint32_t norm32, qcMask; |
| UChar32 minNoMaybe; |
| int32_t length; |
| |
| /* initialize */ |
| if(!compat) { |
| minNoMaybe=(UChar32)indexes[_NORM_INDEX_MIN_NFD_NO_MAYBE]; |
| qcMask=_NORM_QC_NFD; |
| } else { |
| minNoMaybe=(UChar32)indexes[_NORM_INDEX_MIN_NFKD_NO_MAYBE]; |
| qcMask=_NORM_QC_NFKD; |
| } |
| |
| if(c<minNoMaybe) { |
| /* trivial case */ |
| if(destCapacity>0) { |
| dest[0]=(UChar)c; |
| } |
| return -1; |
| } |
| |
| /* data lookup */ |
| UTRIE_GET32(&normTrie, c, norm32); |
| if((norm32&qcMask)==0) { |
| /* simple case: no decomposition */ |
| if(c<=0xffff) { |
| if(destCapacity>0) { |
| dest[0]=(UChar)c; |
| } |
| return -1; |
| } else { |
| if(destCapacity>=2) { |
| dest[0]=UTF16_LEAD(c); |
| dest[1]=UTF16_TRAIL(c); |
| } |
| return -2; |
| } |
| } else if(isNorm32HangulOrJamo(norm32)) { |
| /* Hangul syllable: decompose algorithmically */ |
| UChar c2; |
| |
| c-=HANGUL_BASE; |
| |
| c2=(UChar)(c%JAMO_T_COUNT); |
| c/=JAMO_T_COUNT; |
| if(c2>0) { |
| if(destCapacity>=3) { |
| dest[2]=(UChar)(JAMO_T_BASE+c2); |
| } |
| length=3; |
| } else { |
| length=2; |
| } |
| |
| if(destCapacity>=2) { |
| dest[1]=(UChar)(JAMO_V_BASE+c%JAMO_V_COUNT); |
| dest[0]=(UChar)(JAMO_L_BASE+c/JAMO_V_COUNT); |
| } |
| return length; |
| } else { |
| /* c decomposes, get everything from the variable-length extra data */ |
| const UChar *p, *limit; |
| uint8_t cc, trailCC; |
| |
| p=_decompose(norm32, qcMask, length, cc, trailCC); |
| if(length<=destCapacity) { |
| limit=p+length; |
| do { |
| *dest++=*p++; |
| } while(p<limit); |
| } |
| return length; |
| } |
| } else { |
| return 0; |
| } |
| } |
| |
| static int32_t |
| _decompose(UChar *dest, int32_t destCapacity, |
| const UChar *src, int32_t srcLength, |
| UBool compat, const UnicodeSet *nx, |
| uint8_t &outTrailCC) { |
| UChar buffer[3]; |
| const UChar *limit, *prevSrc, *p; |
| uint32_t norm32, ccOrQCMask, qcMask; |
| int32_t destIndex, reorderStartIndex, length; |
| UChar c, c2, minNoMaybe; |
| uint8_t cc, prevCC, trailCC; |
| |
| if(!compat) { |
| minNoMaybe=(UChar)indexes[_NORM_INDEX_MIN_NFD_NO_MAYBE]; |
| qcMask=_NORM_QC_NFD; |
| } else { |
| minNoMaybe=(UChar)indexes[_NORM_INDEX_MIN_NFKD_NO_MAYBE]; |
| qcMask=_NORM_QC_NFKD; |
| } |
| |
| /* initialize */ |
| ccOrQCMask=_NORM_CC_MASK|qcMask; |
| destIndex=reorderStartIndex=0; |
| prevCC=0; |
| |
| /* avoid compiler warnings */ |
| norm32=0; |
| c=0; |
| cc=0; |
| trailCC=0; |
| |
| if(srcLength>=0) { |
| /* string with length */ |
| limit=src+srcLength; |
| } else /* srcLength==-1 */ { |
| /* zero-terminated string */ |
| limit=NULL; |
| } |
| |
| U_ALIGN_CODE(16); |
| |
| for(;;) { |
| /* count code units below the minimum or with irrelevant data for the quick check */ |
| prevSrc=src; |
| if(limit==NULL) { |
| while((c=*src)<minNoMaybe ? c!=0 : ((norm32=_getNorm32(c))&ccOrQCMask)==0) { |
| prevCC=0; |
| ++src; |
| } |
| } else { |
| while(src!=limit && ((c=*src)<minNoMaybe || ((norm32=_getNorm32(c))&ccOrQCMask)==0)) { |
| prevCC=0; |
| ++src; |
| } |
| } |
| |
| /* copy these code units all at once */ |
| if(src!=prevSrc) { |
| length=(int32_t)(src-prevSrc); |
| if((destIndex+length)<=destCapacity) { |
| uprv_memcpy(dest+destIndex, prevSrc, length*U_SIZEOF_UCHAR); |
| } |
| destIndex+=length; |
| reorderStartIndex=destIndex; |
| } |
| |
| /* end of source reached? */ |
| if(limit==NULL ? c==0 : src==limit) { |
| break; |
| } |
| |
| /* c already contains *src and norm32 is set for it, increment src */ |
| ++src; |
| |
| /* check one above-minimum, relevant code unit */ |
| /* |
| * generally, set p and length to the decomposition string |
| * in simple cases, p==NULL and (c, c2) will hold the length code units to append |
| * in all cases, set cc to the lead and trailCC to the trail combining class |
| * |
| * the following merge-sort of the current character into the preceding, |
| * canonically ordered result text will use the optimized _insertOrdered() |
| * if there is only one single code point to process; |
| * this is indicated with p==NULL, and (c, c2) is the character to insert |
| * ((c, 0) for a BMP character and (lead surrogate, trail surrogate) |
| * for a supplementary character) |
| * otherwise, p[length] is merged in with _mergeOrdered() |
| */ |
| if(isNorm32HangulOrJamo(norm32)) { |
| if(nx_contains(nx, c)) { |
| c2=0; |
| p=NULL; |
| length=1; |
| } else { |
| /* Hangul syllable: decompose algorithmically */ |
| p=buffer; |
| cc=trailCC=0; |
| |
| c-=HANGUL_BASE; |
| |
| c2=(UChar)(c%JAMO_T_COUNT); |
| c/=JAMO_T_COUNT; |
| if(c2>0) { |
| buffer[2]=(UChar)(JAMO_T_BASE+c2); |
| length=3; |
| } else { |
| length=2; |
| } |
| |
| buffer[1]=(UChar)(JAMO_V_BASE+c%JAMO_V_COUNT); |
| buffer[0]=(UChar)(JAMO_L_BASE+c/JAMO_V_COUNT); |
| } |
| } else { |
| if(isNorm32Regular(norm32)) { |
| c2=0; |
| length=1; |
| } else { |
| /* c is a lead surrogate, get the real norm32 */ |
| if(src!=limit && UTF_IS_SECOND_SURROGATE(c2=*src)) { |
| ++src; |
| length=2; |
| norm32=_getNorm32FromSurrogatePair(norm32, c2); |
| } else { |
| c2=0; |
| length=1; |
| norm32=0; |
| } |
| } |
| |
| /* get the decomposition and the lead and trail cc's */ |
| if(nx_contains(nx, c, c2)) { |
| /* excluded: norm32==0 */ |
| cc=trailCC=0; |
| p=NULL; |
| } else if((norm32&qcMask)==0) { |
| /* c does not decompose */ |
| cc=trailCC=(uint8_t)(norm32>>_NORM_CC_SHIFT); |
| p=NULL; |
| } else { |
| /* c decomposes, get everything from the variable-length extra data */ |
| p=_decompose(norm32, qcMask, length, cc, trailCC); |
| if(length==1) { |
| /* fastpath a single code unit from decomposition */ |
| c=*p; |
| c2=0; |
| p=NULL; |
| } |
| } |
| } |
| |
| /* append the decomposition to the destination buffer, assume length>0 */ |
| if((destIndex+length)<=destCapacity) { |
| UChar *reorderSplit=dest+destIndex; |
| if(p==NULL) { |
| /* fastpath: single code point */ |
| if(cc!=0 && cc<prevCC) { |
| /* (c, c2) is out of order with respect to the preceding text */ |
| destIndex+=length; |
| trailCC=_insertOrdered(dest+reorderStartIndex, reorderSplit, dest+destIndex, c, c2, cc); |
| } else { |
| /* just append (c, c2) */ |
| dest[destIndex++]=c; |
| if(c2!=0) { |
| dest[destIndex++]=c2; |
| } |
| } |
| } else { |
| /* general: multiple code points (ordered by themselves) from decomposition */ |
| if(cc!=0 && cc<prevCC) { |
| /* the decomposition is out of order with respect to the preceding text */ |
| destIndex+=length; |
| trailCC=_mergeOrdered(dest+reorderStartIndex, reorderSplit, p, p+length); |
| } else { |
| /* just append the decomposition */ |
| do { |
| dest[destIndex++]=*p++; |
| } while(--length>0); |
| } |
| } |
| } else { |
| /* buffer overflow */ |
| /* keep incrementing the destIndex for preflighting */ |
| destIndex+=length; |
| } |
| |
| prevCC=trailCC; |
| if(prevCC==0) { |
| reorderStartIndex=destIndex; |
| } |
| } |
| |
| outTrailCC=prevCC; |
| return destIndex; |
| } |
| |
| U_CAPI int32_t U_EXPORT2 |
| unorm_decompose(UChar *dest, int32_t destCapacity, |
| const UChar *src, int32_t srcLength, |
| UBool compat, int32_t options, |
| UErrorCode *pErrorCode) { |
| const UnicodeSet *nx; |
| int32_t destIndex; |
| uint8_t trailCC; |
| |
| if(!_haveData(*pErrorCode)) { |
| return 0; |
| } |
| |
| nx=getNX(options, *pErrorCode); |
| if(U_FAILURE(*pErrorCode)) { |
| return 0; |
| } |
| |
| destIndex=_decompose(dest, destCapacity, |
| src, srcLength, |
| compat, nx, |
| trailCC); |
| |
| return u_terminateUChars(dest, destCapacity, destIndex, pErrorCode); |
| } |
| |
| /* make NFC & NFKC ---------------------------------------------------------- */ |
| |
| /* get the composition properties of the next character */ |
| static inline uint32_t |
| _getNextCombining(UChar *&p, const UChar *limit, |
| UChar &c, UChar &c2, |
| uint16_t &combiningIndex, uint8_t &cc, |
| const UnicodeSet *nx) { |
| uint32_t norm32, combineFlags; |
| |
| /* get properties */ |
| c=*p++; |
| norm32=_getNorm32(c); |
| |
| /* preset output values for most characters */ |
| c2=0; |
| combiningIndex=0; |
| cc=0; |
| |
| if((norm32&(_NORM_CC_MASK|_NORM_COMBINES_ANY))==0) { |
| return 0; |
| } else { |
| if(isNorm32Regular(norm32)) { |
| /* set cc etc. below */ |
| } else if(isNorm32HangulOrJamo(norm32)) { |
| /* a compatibility decomposition contained Jamos */ |
| combiningIndex=(uint16_t)(0xfff0|(norm32>>_NORM_EXTRA_SHIFT)); |
| return norm32&_NORM_COMBINES_ANY; |
| } else { |
| /* c is a lead surrogate, get the real norm32 */ |
| if(p!=limit && UTF_IS_SECOND_SURROGATE(c2=*p)) { |
| ++p; |
| norm32=_getNorm32FromSurrogatePair(norm32, c2); |
| } else { |
| c2=0; |
| return 0; |
| } |
| } |
| |
| if(nx_contains(nx, c, c2)) { |
| return 0; /* excluded: norm32==0 */ |
| } |
| |
| cc=(uint8_t)(norm32>>_NORM_CC_SHIFT); |
| |
| combineFlags=norm32&_NORM_COMBINES_ANY; |
| if(combineFlags!=0) { |
| combiningIndex=*(_getExtraData(norm32)-1); |
| } |
| return combineFlags; |
| } |
| } |
| |
| /* |
| * given a composition-result starter (c, c2) - which means its cc==0, |
| * it combines forward, it has extra data, its norm32!=0, |
| * it is not a Hangul or Jamo, |
| * get just its combineFwdIndex |
| * |
| * norm32(c) is special if and only if c2!=0 |
| */ |
| static inline uint16_t |
| _getCombiningIndexFromStarter(UChar c, UChar c2) { |
| uint32_t norm32; |
| |
| norm32=_getNorm32(c); |
| if(c2!=0) { |
| norm32=_getNorm32FromSurrogatePair(norm32, c2); |
| } |
| return *(_getExtraData(norm32)-1); |
| } |
| |
| /* |
| * Find the recomposition result for |
| * a forward-combining character |
| * (specified with a pointer to its part of the combiningTable[]) |
| * and a backward-combining character |
| * (specified with its combineBackIndex). |
| * |
| * If these two characters combine, then set (value, value2) |
| * with the code unit(s) of the composition character. |
| * |
| * Return value: |
| * 0 do not combine |
| * 1 combine |
| * >1 combine, and the composition is a forward-combining starter |
| * |
| * See unormimp.h for a description of the composition table format. |
| */ |
| static inline uint16_t |
| _combine(const uint16_t *table, uint16_t combineBackIndex, |
| uint16_t &value, uint16_t &value2) { |
| uint16_t key; |
| |
| /* search in the starter's composition table */ |
| for(;;) { |
| key=*table++; |
| if(key>=combineBackIndex) { |
| break; |
| } |
| table+= *table&0x8000 ? 2 : 1; |
| } |
| |
| /* mask off bit 15, the last-entry-in-the-list flag */ |
| if((key&0x7fff)==combineBackIndex) { |
| /* found! combine! */ |
| value=*table; |
| |
| /* is the composition a starter that combines forward? */ |
| key=(uint16_t)((value&0x2000)+1); |
| |
| /* get the composition result code point from the variable-length result value */ |
| if(value&0x8000) { |
| if(value&0x4000) { |
| /* surrogate pair composition result */ |
| value=(uint16_t)((value&0x3ff)|0xd800); |
| value2=*(table+1); |
| } else { |
| /* BMP composition result U+2000..U+ffff */ |
| value=*(table+1); |
| value2=0; |
| } |
| } else { |
| /* BMP composition result U+0000..U+1fff */ |
| value&=0x1fff; |
| value2=0; |
| } |
| |
| return key; |
| } else { |
| /* not found */ |
| return 0; |
| } |
| } |
| |
| static inline UBool |
| _composeHangul(UChar prev, UChar c, uint32_t norm32, const UChar *&src, const UChar *limit, |
| UBool compat, UChar *dest, const UnicodeSet *nx) { |
| if(isJamoVTNorm32JamoV(norm32)) { |
| /* c is a Jamo V, compose with previous Jamo L and following Jamo T */ |
| prev=(UChar)(prev-JAMO_L_BASE); |
| if(prev<JAMO_L_COUNT) { |
| c=(UChar)(HANGUL_BASE+(prev*JAMO_V_COUNT+(c-JAMO_V_BASE))*JAMO_T_COUNT); |
| |
| /* check if the next character is a Jamo T (normal or compatibility) */ |
| if(src!=limit) { |
| UChar next, t; |
| |
| next=*src; |
| if((t=(UChar)(next-JAMO_T_BASE))<JAMO_T_COUNT) { |
| /* normal Jamo T */ |
| ++src; |
| c+=t; |
| } else if(compat) { |
| /* if NFKC, then check for compatibility Jamo T (BMP only) */ |
| norm32=_getNorm32(next); |
| if(isNorm32Regular(norm32) && (norm32&_NORM_QC_NFKD)) { |
| const UChar *p; |
| int32_t length; |
| uint8_t cc, trailCC; |
| |
| p=_decompose(norm32, _NORM_QC_NFKD, length, cc, trailCC); |
| if(length==1 && (t=(UChar)(*p-JAMO_T_BASE))<JAMO_T_COUNT) { |
| /* compatibility Jamo T */ |
| ++src; |
| c+=t; |
| } |
| } |
| } |
| } |
| if(nx_contains(nx, c)) { |
| if(!isHangulWithoutJamoT(c)) { |
| --src; /* undo ++src from reading the Jamo T */ |
| } |
| return FALSE; |
| } |
| if(dest!=0) { |
| *dest=c; |
| } |
| return TRUE; |
| } |
| } else if(isHangulWithoutJamoT(prev)) { |
| /* c is a Jamo T, compose with previous Hangul LV that does not contain a Jamo T */ |
| c=(UChar)(prev+(c-JAMO_T_BASE)); |
| if(nx_contains(nx, c)) { |
| return FALSE; |
| } |
| if(dest!=0) { |
| *dest=c; |
| } |
| return TRUE; |
| } |
| return FALSE; |
| } |
| |
| /* |
| * recompose the characters in [p..limit[ |
| * (which is in NFD - decomposed and canonically ordered), |
| * adjust limit, and return the trailing cc |
| * |
| * since for NFKC we may get Jamos in decompositions, we need to |
| * recompose those too |
| * |
| * note that recomposition never lengthens the text: |
| * any character consists of either one or two code units; |
| * a composition may contain at most one more code unit than the original starter, |
| * while the combining mark that is removed has at least one code unit |
| */ |
| static uint8_t |
| _recompose(UChar *p, UChar *&limit, int32_t options, const UnicodeSet *nx) { |
| UChar *starter, *pRemove, *q, *r; |
| uint32_t combineFlags; |
| UChar c, c2; |
| uint16_t combineFwdIndex, combineBackIndex; |
| uint16_t result, value, value2; |
| uint8_t cc, prevCC; |
| UBool starterIsSupplementary; |
| |
| starter=NULL; /* no starter */ |
| combineFwdIndex=0; /* will not be used until starter!=NULL - avoid compiler warnings */ |
| combineBackIndex=0; /* will always be set if combineFlags!=0 - avoid compiler warnings */ |
| value=value2=0; /* always set by _combine() before used - avoid compiler warnings */ |
| starterIsSupplementary=FALSE; /* will not be used until starter!=NULL - avoid compiler warnings */ |
| prevCC=0; |
| |
| for(;;) { |
| combineFlags=_getNextCombining(p, limit, c, c2, combineBackIndex, cc, nx); |
| if((combineFlags&_NORM_COMBINES_BACK) && starter!=NULL) { |
| if(combineBackIndex&0x8000) { |
| /* c is a Jamo V/T, see if we can compose it with the previous character */ |
| /* for the PRI #29 fix, check that there is no intervening combining mark */ |
| if((options&UNORM_BEFORE_PRI_29) || prevCC==0) { |
| pRemove=NULL; /* NULL while no Hangul composition */ |
| combineFlags=0; |
| c2=*starter; |
| if(combineBackIndex==0xfff2) { |
| /* Jamo V, compose with previous Jamo L and following Jamo T */ |
| c2=(UChar)(c2-JAMO_L_BASE); |
| if(c2<JAMO_L_COUNT) { |
| pRemove=p-1; |
| c=(UChar)(HANGUL_BASE+(c2*JAMO_V_COUNT+(c-JAMO_V_BASE))*JAMO_T_COUNT); |
| if(p!=limit && (c2=(UChar)(*p-JAMO_T_BASE))<JAMO_T_COUNT) { |
| ++p; |
| c+=c2; |
| } else { |
| /* the result is an LV syllable, which is a starter (unlike LVT) */ |
| combineFlags=_NORM_COMBINES_FWD; |
| } |
| if(!nx_contains(nx, c)) { |
| *starter=c; |
| } else { |
| /* excluded */ |
| if(!isHangulWithoutJamoT(c)) { |
| --p; /* undo the ++p from reading the Jamo T */ |
| } |
| /* c is modified but not used any more -- c=*(p-1); -- re-read the Jamo V/T */ |
| pRemove=NULL; |
| } |
| } |
| |
| /* |
| * Normally, the following can not occur: |
| * Since the input is in NFD, there are no Hangul LV syllables that |
| * a Jamo T could combine with. |
| * All Jamo Ts are combined above when handling Jamo Vs. |
| * |
| * However, before the PRI #29 fix, this can occur due to |
| * an intervening combining mark between the Hangul LV and the Jamo T. |
| */ |
| } else { |
| /* Jamo T, compose with previous Hangul that does not have a Jamo T */ |
| if(isHangulWithoutJamoT(c2)) { |
| c2+=(UChar)(c-JAMO_T_BASE); |
| if(!nx_contains(nx, c2)) { |
| pRemove=p-1; |
| *starter=c2; |
| } |
| } |
| } |
| |
| if(pRemove!=NULL) { |
| /* remove the Jamo(s) */ |
| q=pRemove; |
| r=p; |
| while(r<limit) { |
| *q++=*r++; |
| } |
| p=pRemove; |
| limit=q; |
| } |
| |
| c2=0; /* c2 held *starter temporarily */ |
| |
| if(combineFlags!=0) { |
| /* |
| * not starter=NULL because the composition is a Hangul LV syllable |
| * and might combine once more (but only before the PRI #29 fix) |
| */ |
| |
| /* done? */ |
| if(p==limit) { |
| return prevCC; |
| } |
| |
| /* the composition is a Hangul LV syllable which is a starter that combines forward */ |
| combineFwdIndex=0xfff0; |
| |
| /* we combined; continue with looking for compositions */ |
| continue; |
| } |
| } |
| |
| /* |
| * now: cc==0 and the combining index does not include "forward" -> |
| * the rest of the loop body will reset starter to NULL; |
| * technically, a composed Hangul syllable is a starter, but it |
| * does not combine forward now that we have consumed all eligible Jamos; |
| * for Jamo V/T, combineFlags does not contain _NORM_COMBINES_FWD |
| */ |
| |
| } else if( |
| /* the starter is not a Hangul LV or Jamo V/T and */ |
| !(combineFwdIndex&0x8000) && |
| /* the combining mark is not blocked and */ |
| ((options&UNORM_BEFORE_PRI_29) ? |
| (prevCC!=cc || prevCC==0) : |
| (prevCC<cc || prevCC==0)) && |
| /* the starter and the combining mark (c, c2) do combine and */ |
| 0!=(result=_combine(combiningTable+combineFwdIndex, combineBackIndex, value, value2)) && |
| /* the composition result is not excluded */ |
| !nx_contains(nx, value, value2) |
| ) { |
| /* replace the starter with the composition, remove the combining mark */ |
| pRemove= c2==0 ? p-1 : p-2; /* pointer to the combining mark */ |
| |
| /* replace the starter with the composition */ |
| *starter=(UChar)value; |
| if(starterIsSupplementary) { |
| if(value2!=0) { |
| /* both are supplementary */ |
| *(starter+1)=(UChar)value2; |
| } else { |
| /* the composition is shorter than the starter, move the intermediate characters forward one */ |
| starterIsSupplementary=FALSE; |
| q=starter+1; |
| r=q+1; |
| while(r<pRemove) { |
| *q++=*r++; |
| } |
| --pRemove; |
| } |
| } else if(value2!=0) { |
| /* the composition is longer than the starter, move the intermediate characters back one */ |
| starterIsSupplementary=TRUE; |
| ++starter; /* temporarily increment for the loop boundary */ |
| q=pRemove; |
| r=++pRemove; |
| while(starter<q) { |
| *--r=*--q; |
| } |
| *starter=(UChar)value2; |
| --starter; /* undo the temporary increment */ |
| /* } else { both are on the BMP, nothing more to do */ |
| } |
| |
| /* remove the combining mark by moving the following text over it */ |
| if(pRemove<p) { |
| q=pRemove; |
| r=p; |
| while(r<limit) { |
| *q++=*r++; |
| } |
| p=pRemove; |
| limit=q; |
| } |
| |
| /* keep prevCC because we removed the combining mark */ |
| |
| /* done? */ |
| if(p==limit) { |
| return prevCC; |
| } |
| |
| /* is the composition a starter that combines forward? */ |
| if(result>1) { |
| combineFwdIndex=_getCombiningIndexFromStarter((UChar)value, (UChar)value2); |
| } else { |
| starter=NULL; |
| } |
| |
| /* we combined; continue with looking for compositions */ |
| continue; |
| } |
| } |
| |
| /* no combination this time */ |
| prevCC=cc; |
| if(p==limit) { |
| return prevCC; |
| } |
| |
| /* if (c, c2) did not combine, then check if it is a starter */ |
| if(cc==0) { |
| /* found a new starter; combineFlags==0 if (c, c2) is excluded */ |
| if(combineFlags&_NORM_COMBINES_FWD) { |
| /* it may combine with something, prepare for it */ |
| if(c2==0) { |
| starterIsSupplementary=FALSE; |
| starter=p-1; |
| } else { |
| starterIsSupplementary=TRUE; |
| starter=p-2; |
| } |
| combineFwdIndex=combineBackIndex; |
| } else { |
| /* it will not combine with anything */ |
| starter=NULL; |
| } |
| } else if(options&_NORM_OPTIONS_COMPOSE_CONTIGUOUS) { |
| /* FCC: no discontiguous compositions; any intervening character blocks */ |
| starter=NULL; |
| } |
| } |
| } |
| |
| /* decompose and recompose [prevStarter..src[ */ |
| static const UChar * |
| _composePart(UChar *stackBuffer, UChar *&buffer, int32_t &bufferCapacity, int32_t &length, |
| const UChar *prevStarter, const UChar *src, |
| uint8_t &prevCC, |
| int32_t options, const UnicodeSet *nx, |
| UErrorCode *pErrorCode) { |
| UChar *recomposeLimit; |
| uint8_t trailCC; |
| UBool compat; |
| |
| compat=(UBool)((options&_NORM_OPTIONS_COMPAT)!=0); |
| |
| /* decompose [prevStarter..src[ */ |
| length=_decompose(buffer, bufferCapacity, |
| prevStarter, (int32_t)(src-prevStarter), |
| compat, nx, |
| trailCC); |
| if(length>bufferCapacity) { |
| if(!u_growBufferFromStatic(stackBuffer, &buffer, &bufferCapacity, 2*length, 0)) { |
| *pErrorCode=U_MEMORY_ALLOCATION_ERROR; |
| return NULL; |
| } |
| length=_decompose(buffer, bufferCapacity, |
| prevStarter, (int32_t)(src-prevStarter), |
| compat, nx, |
| trailCC); |
| } |
| |
| /* recompose the decomposition */ |
| recomposeLimit=buffer+length; |
| if(length>=2) { |
| prevCC=_recompose(buffer, recomposeLimit, options, nx); |
| } |
| |
| /* return with a pointer to the recomposition and its length */ |
| length=(int32_t)(recomposeLimit-buffer); |
| return buffer; |
| } |
| |
| static int32_t |
| _compose(UChar *dest, int32_t destCapacity, |
| const UChar *src, int32_t srcLength, |
| int32_t options, const UnicodeSet *nx, |
| UErrorCode *pErrorCode) { |
| UChar stackBuffer[_STACK_BUFFER_CAPACITY]; |
| UChar *buffer; |
| int32_t bufferCapacity; |
| |
| const UChar *limit, *prevSrc, *prevStarter; |
| uint32_t norm32, ccOrQCMask, qcMask; |
| int32_t destIndex, reorderStartIndex, length; |
| UChar c, c2, minNoMaybe; |
| uint8_t cc, prevCC; |
| |
| if(options&_NORM_OPTIONS_COMPAT) { |
| minNoMaybe=(UChar)indexes[_NORM_INDEX_MIN_NFKC_NO_MAYBE]; |
| qcMask=_NORM_QC_NFKC; |
| } else { |
| minNoMaybe=(UChar)indexes[_NORM_INDEX_MIN_NFC_NO_MAYBE]; |
| qcMask=_NORM_QC_NFC; |
| } |
| |
| /* initialize */ |
| buffer=stackBuffer; |
| bufferCapacity=_STACK_BUFFER_CAPACITY; |
| |
| /* |
| * prevStarter points to the last character before the current one |
| * that is a "true" starter with cc==0 and quick check "yes". |
| * |
| * prevStarter will be used instead of looking for a true starter |
| * while incrementally decomposing [prevStarter..prevSrc[ |
| * in _composePart(). Having a good prevStarter allows to just decompose |
| * the entire [prevStarter..prevSrc[. |
| * |
| * When _composePart() backs out from prevSrc back to prevStarter, |
| * then it also backs out destIndex by the same amount. |
| * Therefore, at all times, the (prevSrc-prevStarter) source units |
| * must correspond 1:1 to destination units counted with destIndex, |
| * except for reordering. |
| * This is true for the qc "yes" characters copied in the fast loop, |
| * and for pure reordering. |
| * prevStarter must be set forward to src when this is not true: |
| * In _composePart() and after composing a Hangul syllable. |
| * |
| * This mechanism relies on the assumption that the decomposition of a true starter |
| * also begins with a true starter. gennorm/store.c checks for this. |
| */ |
| prevStarter=src; |
| |
| ccOrQCMask=_NORM_CC_MASK|qcMask; |
| destIndex=reorderStartIndex=0; |
| prevCC=0; |
| |
| /* avoid compiler warnings */ |
| norm32=0; |
| c=0; |
| |
| if(srcLength>=0) { |
| /* string with length */ |
| limit=src+srcLength; |
| } else /* srcLength==-1 */ { |
| /* zero-terminated string */ |
| limit=NULL; |
| } |
| |
| U_ALIGN_CODE(16); |
| |
| for(;;) { |
| /* count code units below the minimum or with irrelevant data for the quick check */ |
| prevSrc=src; |
| if(limit==NULL) { |
| while((c=*src)<minNoMaybe ? c!=0 : ((norm32=_getNorm32(c))&ccOrQCMask)==0) { |
| prevCC=0; |
| ++src; |
| } |
| } else { |
| while(src!=limit && ((c=*src)<minNoMaybe || ((norm32=_getNorm32(c))&ccOrQCMask)==0)) { |
| prevCC=0; |
| ++src; |
| } |
| } |
| |
| /* copy these code units all at once */ |
| if(src!=prevSrc) { |
| length=(int32_t)(src-prevSrc); |
| if((destIndex+length)<=destCapacity) { |
| uprv_memcpy(dest+destIndex, prevSrc, length*U_SIZEOF_UCHAR); |
| } |
| destIndex+=length; |
| reorderStartIndex=destIndex; |
| |
| /* set prevStarter to the last character in the quick check loop */ |
| prevStarter=src-1; |
| if(UTF_IS_SECOND_SURROGATE(*prevStarter) && prevSrc<prevStarter && UTF_IS_FIRST_SURROGATE(*(prevStarter-1))) { |
| --prevStarter; |
| } |
| |
| prevSrc=src; |
| } |
| |
| /* end of source reached? */ |
| if(limit==NULL ? c==0 : src==limit) { |
| break; |
| } |
| |
| /* c already contains *src and norm32 is set for it, increment src */ |
| ++src; |
| |
| /* |
| * source buffer pointers: |
| * |
| * all done quick check current char not yet |
| * "yes" but (c, c2) processed |
| * may combine |
| * forward |
| * [-------------[-------------[-------------[-------------[ |
| * | | | | | |
| * start prevStarter prevSrc src limit |
| * |
| * |
| * destination buffer pointers and indexes: |
| * |
| * all done might take not filled yet |
| * characters for |
| * reordering |
| * [-------------[-------------[-------------[ |
| * | | | | |
| * dest reorderStartIndex destIndex destCapacity |
| */ |
| |
| /* check one above-minimum, relevant code unit */ |
| /* |
| * norm32 is for c=*(src-1), and the quick check flag is "no" or "maybe", and/or cc!=0 |
| * check for Jamo V/T, then for surrogates and regular characters |
| * c is not a Hangul syllable or Jamo L because |
| * they are not marked with no/maybe for NFC & NFKC (and their cc==0) |
| */ |
| if(isNorm32HangulOrJamo(norm32)) { |
| /* |
| * c is a Jamo V/T: |
| * try to compose with the previous character, Jamo V also with a following Jamo T, |
| * and set values here right now in case we just continue with the main loop |
| */ |
| prevCC=cc=0; |
| reorderStartIndex=destIndex; |
| |
| if( |
| destIndex>0 && |
| _composeHangul( |
| *(prevSrc-1), c, norm32, src, limit, (UBool)((options&_NORM_OPTIONS_COMPAT)!=0), |
| destIndex<=destCapacity ? dest+(destIndex-1) : 0, |
| nx) |
| ) { |
| prevStarter=src; |
| continue; |
| } |
| |
| /* the Jamo V/T did not compose into a Hangul syllable, just append to dest */ |
| c2=0; |
| length=1; |
| prevStarter=prevSrc; |
| } else { |
| if(isNorm32Regular(norm32)) { |
| c2=0; |
| length=1; |
| } else { |
| /* c is a lead surrogate, get the real norm32 */ |
| if(src!=limit && UTF_IS_SECOND_SURROGATE(c2=*src)) { |
| ++src; |
| length=2; |
| norm32=_getNorm32FromSurrogatePair(norm32, c2); |
| } else { |
| /* c is an unpaired lead surrogate, nothing to do */ |
| c2=0; |
| length=1; |
| norm32=0; |
| } |
| } |
| |
| /* we are looking at the character (c, c2) at [prevSrc..src[ */ |
| if(nx_contains(nx, c, c2)) { |
| /* excluded: norm32==0 */ |
| cc=0; |
| } else if((norm32&qcMask)==0) { |
| cc=(uint8_t)(norm32>>_NORM_CC_SHIFT); |
| } else { |
| const UChar *p; |
| uint32_t decompQCMask; |
| |
| /* |
| * find appropriate boundaries around this character, |
| * decompose the source text from between the boundaries, |
| * and recompose it |
| * |
| * this puts the intermediate text into the side buffer because |
| * it might be longer than the recomposition end result, |
| * or the destination buffer may be too short or missing |
| * |
| * note that destIndex may be adjusted backwards to account |
| * for source text that passed the quick check but needed to |
| * take part in the recomposition |
| */ |
| decompQCMask=(qcMask<<2)&0xf; /* decomposition quick check mask */ |
| |
| /* |
| * find the last true starter in [prevStarter..src[ |
| * it is either the decomposition of the current character (at prevSrc), |
| * or prevStarter |
| */ |
| if(_isTrueStarter(norm32, ccOrQCMask, decompQCMask)) { |
| prevStarter=prevSrc; |
| } else { |
| /* adjust destIndex: back out what had been copied with qc "yes" */ |
| destIndex-=(int32_t)(prevSrc-prevStarter); |
| } |
| |
| /* find the next true starter in [src..limit[ - modifies src to point to the next starter */ |
| src=_findNextStarter(src, limit, qcMask, decompQCMask, minNoMaybe); |
| |
| /* compose [prevStarter..src[ */ |
| p=_composePart(stackBuffer, buffer, bufferCapacity, |
| length, /* output */ |
| prevStarter, src, |
| prevCC, /* output */ |
| options, nx, |
| pErrorCode); |
| |
| if(p==NULL) { |
| destIndex=0; /* an error occurred (out of memory) */ |
| break; |
| } |
| |
| /* append the recomposed buffer contents to the destination buffer */ |
| if((destIndex+length)<=destCapacity) { |
| while(length>0) { |
| dest[destIndex++]=*p++; |
| --length; |
| } |
| } else { |
| /* buffer overflow */ |
| /* keep incrementing the destIndex for preflighting */ |
| destIndex+=length; |
| } |
| |
| /* set the next starter */ |
| prevStarter=src; |
| |
| continue; |
| } |
| } |
| |
| /* append the single code point (c, c2) to the destination buffer */ |
| if((destIndex+length)<=destCapacity) { |
| if(cc!=0 && cc<prevCC) { |
| /* (c, c2) is out of order with respect to the preceding text */ |
| UChar *reorderSplit=dest+destIndex; |
| destIndex+=length; |
| prevCC=_insertOrdered(dest+reorderStartIndex, reorderSplit, dest+destIndex, c, c2, cc); |
| } else { |
| /* just append (c, c2) */ |
| dest[destIndex++]=c; |
| if(c2!=0) { |
| dest[destIndex++]=c2; |
| } |
| prevCC=cc; |
| } |
| } else { |
| /* buffer overflow */ |
| /* keep incrementing the destIndex for preflighting */ |
| destIndex+=length; |
| prevCC=cc; |
| } |
| } |
| |
| /* cleanup */ |
| if(buffer!=stackBuffer) { |
| uprv_free(buffer); |
| } |
| |
| return destIndex; |
| } |
| |
| U_CAPI int32_t U_EXPORT2 |
| unorm_compose(UChar *dest, int32_t destCapacity, |
| const UChar *src, int32_t srcLength, |
| UBool compat, int32_t options, |
| UErrorCode *pErrorCode) { |
| const UnicodeSet *nx; |
| int32_t destIndex; |
| |
| if(!_haveData(*pErrorCode)) { |
| return 0; |
| } |
| |
| nx=getNX(options, *pErrorCode); |
| if(U_FAILURE(*pErrorCode)) { |
| return 0; |
| } |
| |
| /* reset options bits that should only be set here or inside _compose() */ |
| options&=~(_NORM_OPTIONS_SETS_MASK|_NORM_OPTIONS_COMPAT|_NORM_OPTIONS_COMPOSE_CONTIGUOUS); |
| |
| if(compat) { |
| options|=_NORM_OPTIONS_COMPAT; |
| } |
| |
| destIndex=_compose(dest, destCapacity, |
| src, srcLength, |
| options, nx, |
| pErrorCode); |
| |
| return u_terminateUChars(dest, destCapacity, destIndex, pErrorCode); |
| } |
| |
| /* make FCD ----------------------------------------------------------------- */ |
| |
| static const UChar * |
| _findSafeFCD(const UChar *src, const UChar *limit, uint16_t fcd16) { |
| UChar c, c2; |
| |
| /* |
| * find the first position in [src..limit[ after some cc==0 according to FCD data |
| * |
| * at the beginning of the loop, we have fcd16 from before src |
| * |
| * stop at positions: |
| * - after trail cc==0 |
| * - at the end of the source |
| * - before lead cc==0 |
| */ |
| for(;;) { |
| /* stop if trail cc==0 for the previous character */ |
| if((fcd16&0xff)==0) { |
| break; |
| } |
| |
| /* get c=*src - stop at end of string */ |
| if(src==limit) { |
| break; |
| } |
| c=*src; |
| |
| /* stop if lead cc==0 for this character */ |
| if(c<_NORM_MIN_WITH_LEAD_CC || (fcd16=_getFCD16(c))==0) { |
| break; /* catches terminating NUL, too */ |
| } |
| |
| if(!UTF_IS_FIRST_SURROGATE(c)) { |
| if(fcd16<=0xff) { |
| break; |
| } |
| ++src; |
| } else if((src+1)!=limit && (c2=*(src+1), UTF_IS_SECOND_SURROGATE(c2))) { |
| /* c is a lead surrogate, get the real fcd16 */ |
| fcd16=_getFCD16FromSurrogatePair(fcd16, c2); |
| if(fcd16<=0xff) { |
| break; |
| } |
| src+=2; |
| } else { |
| /* c is an unpaired first surrogate, lead cc==0 */ |
| break; |
| } |
| } |
| |
| return src; |
| } |
| |
| static uint8_t |
| _decomposeFCD(const UChar *src, const UChar *decompLimit, |
| UChar *dest, int32_t &destIndex, int32_t destCapacity, |
| const UnicodeSet *nx) { |
| const UChar *p; |
| uint32_t norm32; |
| int32_t reorderStartIndex, length; |
| UChar c, c2; |
| uint8_t cc, prevCC, trailCC; |
| |
| /* |
| * canonically decompose [src..decompLimit[ |
| * |
| * all characters in this range have some non-zero cc, |
| * directly or in decomposition, |
| * so that we do not need to check in the following for quick-check limits etc. |
| * |
| * there _are_ _no_ Hangul syllables or Jamos in here because they are FCD-safe (cc==0)! |
| * |
| * we also do not need to check for c==0 because we have an established decompLimit |
| */ |
| reorderStartIndex=destIndex; |
| prevCC=0; |
| |
| while(src<decompLimit) { |
| c=*src++; |
| norm32=_getNorm32(c); |
| if(isNorm32Regular(norm32)) { |
| c2=0; |
| length=1; |
| } else { |
| /* |
| * reminder: this function is called with [src..decompLimit[ |
| * not containing any Hangul/Jamo characters, |
| * therefore the only specials are lead surrogates |
| */ |
| /* c is a lead surrogate, get the real norm32 */ |
| if(src!=decompLimit && UTF_IS_SECOND_SURROGATE(c2=*src)) { |
| ++src; |
| length=2; |
| norm32=_getNorm32FromSurrogatePair(norm32, c2); |
| } else { |
| c2=0; |
| length=1; |
| norm32=0; |
| } |
| } |
| |
| /* get the decomposition and the lead and trail cc's */ |
| if(nx_contains(nx, c, c2)) { |
| /* excluded: norm32==0 */ |
| cc=trailCC=0; |
| p=NULL; |
| } else if((norm32&_NORM_QC_NFD)==0) { |
| /* c does not decompose */ |
| cc=trailCC=(uint8_t)(norm32>>_NORM_CC_SHIFT); |
| p=NULL; |
| } else { |
| /* c decomposes, get everything from the variable-length extra data */ |
| p=_decompose(norm32, length, cc, trailCC); |
| if(length==1) { |
| /* fastpath a single code unit from decomposition */ |
| c=*p; |
| c2=0; |
| p=NULL; |
| } |
| } |
| |
| /* append the decomposition to the destination buffer, assume length>0 */ |
| if((destIndex+length)<=destCapacity) { |
| UChar *reorderSplit=dest+destIndex; |
| if(p==NULL) { |
| /* fastpath: single code point */ |
| if(cc!=0 && cc<prevCC) { |
| /* (c, c2) is out of order with respect to the preceding text */ |
| destIndex+=length; |
| trailCC=_insertOrdered(dest+reorderStartIndex, reorderSplit, dest+destIndex, c, c2, cc); |
| } else { |
| /* just append (c, c2) */ |
| dest[destIndex++]=c; |
| if(c2!=0) { |
| dest[destIndex++]=c2; |
| } |
| } |
| } else { |
| /* general: multiple code points (ordered by themselves) from decomposition */ |
| if(cc!=0 && cc<prevCC) { |
| /* the decomposition is out of order with respect to the preceding text */ |
| destIndex+=length; |
| trailCC=_mergeOrdered(dest+reorderStartIndex, reorderSplit, p, p+length); |
| } else { |
| /* just append the decomposition */ |
| do { |
| dest[destIndex++]=*p++; |
| } while(--length>0); |
| } |
| } |
| } else { |
| /* buffer overflow */ |
| /* keep incrementing the destIndex for preflighting */ |
| destIndex+=length; |
| } |
| |
| prevCC=trailCC; |
| if(prevCC==0) { |
| reorderStartIndex=destIndex; |
| } |
| } |
| |
| return prevCC; |
| } |
| |
| static int32_t |
| unorm_makeFCD(UChar *dest, int32_t destCapacity, |
| const UChar *src, int32_t srcLength, |
| const UnicodeSet *nx, |
| UErrorCode *pErrorCode) { |
| const UChar *limit, *prevSrc, *decompStart; |
| int32_t destIndex, length; |
| UChar c, c2; |
| uint16_t fcd16; |
| int16_t prevCC, cc; |
| |
| if(!_haveData(*pErrorCode)) { |
| return 0; |
| } |
| |
| /* initialize */ |
| decompStart=src; |
| destIndex=0; |
| prevCC=0; |
| |
| /* avoid compiler warnings */ |
| c=0; |
| fcd16=0; |
| |
| if(srcLength>=0) { |
| /* string with length */ |
| limit=src+srcLength; |
| } else /* srcLength==-1 */ { |
| /* zero-terminated string */ |
| limit=NULL; |
| } |
| |
| U_ALIGN_CODE(16); |
| |
| for(;;) { |
| /* skip a run of code units below the minimum or with irrelevant data for the FCD check */ |
| prevSrc=src; |
| if(limit==NULL) { |
| for(;;) { |
| c=*src; |
| if(c<_NORM_MIN_WITH_LEAD_CC) { |
| if(c==0) { |
| break; |
| } |
| prevCC=(int16_t)-c; |
| } else if((fcd16=_getFCD16(c))==0) { |
| prevCC=0; |
| } else { |
| break; |
| } |
| ++src; |
| } |
| } else { |
| for(;;) { |
| if(src==limit) { |
| break; |
| } else if((c=*src)<_NORM_MIN_WITH_LEAD_CC) { |
| prevCC=(int16_t)-c; |
| } else if((fcd16=_getFCD16(c))==0) { |
| prevCC=0; |
| } else { |
| break; |
| } |
| ++src; |
| } |
| } |
| |
| /* |
| * prevCC has values from the following ranges: |
| * 0..0xff - the previous trail combining class |
| * <0 - the negative value of the previous code unit; |
| * that code unit was <_NORM_MIN_WITH_LEAD_CC and its _getFCD16() |
| * was deferred so that average text is checked faster |
| */ |
| |
| /* copy these code units all at once */ |
| if(src!=prevSrc) { |
| length=(int32_t)(src-prevSrc); |
| if((destIndex+length)<=destCapacity) { |
| uprv_memcpy(dest+destIndex, prevSrc, length*U_SIZEOF_UCHAR); |
| } |
| destIndex+=length; |
| prevSrc=src; |
| |
| /* prevCC<0 is only possible from the above loop, i.e., only if prevSrc<src */ |
| if(prevCC<0) { |
| /* the previous character was <_NORM_MIN_WITH_LEAD_CC, we need to get its trail cc */ |
| if(!nx_contains(nx, (UChar32)-prevCC)) { |
| prevCC=(int16_t)(_getFCD16((UChar)-prevCC)&0xff); |
| } else { |
| prevCC=0; /* excluded: fcd16==0 */ |
| } |
| |
| /* |
| * set a pointer to this below-U+0300 character; |
| * if prevCC==0 then it will moved to after this character below |
| */ |
| decompStart=prevSrc-1; |
| } |
| } |
| /* |
| * now: |
| * prevSrc==src - used later to adjust destIndex before decomposition |
| * prevCC>=0 |
| */ |
| |
| /* end of source reached? */ |
| if(limit==NULL ? c==0 : src==limit) { |
| break; |
| } |
| |
| /* set a pointer to after the last source position where prevCC==0 */ |
| if(prevCC==0) { |
| decompStart=prevSrc; |
| } |
| |
| /* c already contains *src and fcd16 is set for it, increment src */ |
| ++src; |
| |
| /* check one above-minimum, relevant code unit */ |
| if(UTF_IS_FIRST_SURROGATE(c)) { |
| /* c is a lead surrogate, get the real fcd16 */ |
| if(src!=limit && UTF_IS_SECOND_SURROGATE(c2=*src)) { |
| ++src; |
| fcd16=_getFCD16FromSurrogatePair(fcd16, c2); |
| } else { |
| c2=0; |
| fcd16=0; |
| } |
| } else { |
| c2=0; |
| } |
| |
| /* we are looking at the character (c, c2) at [prevSrc..src[ */ |
| if(nx_contains(nx, c, c2)) { |
| fcd16=0; /* excluded: fcd16==0 */ |
| } |
| |
| /* check the combining order, get the lead cc */ |
| cc=(int16_t)(fcd16>>8); |
| if(cc==0 || cc>=prevCC) { |
| /* the order is ok */ |
| if(cc==0) { |
| decompStart=prevSrc; |
| } |
| prevCC=(int16_t)(fcd16&0xff); |
| |
| /* just append (c, c2) */ |
| length= c2==0 ? 1 : 2; |
| if((destIndex+length)<=destCapacity) { |
| dest[destIndex++]=c; |
| if(c2!=0) { |
| dest[destIndex++]=c2; |
| } |
| } else { |
| destIndex+=length; |
| } |
| } else { |
| /* |
| * back out the part of the source that we copied already but |
| * is now going to be decomposed; |
| * prevSrc is set to after what was copied |
| */ |
| destIndex-=(int32_t)(prevSrc-decompStart); |
| |
| /* |
| * find the part of the source that needs to be decomposed; |
| * to be safe and simple, decompose to before the next character with lead cc==0 |
| */ |
| src=_findSafeFCD(src, limit, fcd16); |
| |
| /* |
| * the source text does not fulfill the conditions for FCD; |
| * decompose and reorder a limited piece of the text |
| */ |
| prevCC=_decomposeFCD(decompStart, src, |
| dest, destIndex, destCapacity, |
| nx); |
| decompStart=src; |
| } |
| } |
| |
| return u_terminateUChars(dest, destCapacity, destIndex, pErrorCode); |
| } |
| |
| /* quick check functions ---------------------------------------------------- */ |
| |
| static UBool |
| unorm_checkFCD(const UChar *src, int32_t srcLength, const UnicodeSet *nx) { |
| const UChar *limit; |
| UChar c, c2; |
| uint16_t fcd16; |
| int16_t prevCC, cc; |
| |
| /* initialize */ |
| prevCC=0; |
| |
| if(srcLength>=0) { |
| /* string with length */ |
| limit=src+srcLength; |
| } else /* srcLength==-1 */ { |
| /* zero-terminated string */ |
| limit=NULL; |
| } |
| |
| U_ALIGN_CODE(16); |
| |
| for(;;) { |
| /* skip a run of code units below the minimum or with irrelevant data for the FCD check */ |
| if(limit==NULL) { |
| for(;;) { |
| c=*src++; |
| if(c<_NORM_MIN_WITH_LEAD_CC) { |
| if(c==0) { |
| return TRUE; |
| } |
| /* |
| * delay _getFCD16(c) for any character <_NORM_MIN_WITH_LEAD_CC |
| * because chances are good that the next one will have |
| * a leading cc of 0; |
| * _getFCD16(-prevCC) is later called when necessary - |
| * -c fits into int16_t because it is <_NORM_MIN_WITH_LEAD_CC==0x300 |
| */ |
| prevCC=(int16_t)-c; |
| } else if((fcd16=_getFCD16(c))==0) { |
| prevCC=0; |
| } else { |
| break; |
| } |
| } |
| } else { |
| for(;;) { |
| if(src==limit) { |
| return TRUE; |
| } else if((c=*src++)<_NORM_MIN_WITH_LEAD_CC) { |
| prevCC=(int16_t)-c; |
| } else if((fcd16=_getFCD16(c))==0) { |
| prevCC=0; |
| } else { |
| break; |
| } |
| } |
| } |
| |
| /* check one above-minimum, relevant code unit */ |
| if(UTF_IS_FIRST_SURROGATE(c)) { |
| /* c is a lead surrogate, get the real fcd16 */ |
| if(src!=limit && UTF_IS_SECOND_SURROGATE(c2=*src)) { |
| ++src; |
| fcd16=_getFCD16FromSurrogatePair(fcd16, c2); |
| } else { |
| c2=0; |
| fcd16=0; |
| } |
| } else { |
| c2=0; |
| } |
| |
| if(nx_contains(nx, c, c2)) { |
| prevCC=0; /* excluded: fcd16==0 */ |
| continue; |
| } |
| |
| /* |
| * prevCC has values from the following ranges: |
| * 0..0xff - the previous trail combining class |
| * <0 - the negative value of the previous code unit; |
| * that code unit was <_NORM_MIN_WITH_LEAD_CC and its _getFCD16() |
| * was deferred so that average text is checked faster |
| */ |
| |
| /* check the combining order */ |
| cc=(int16_t)(fcd16>>8); |
| if(cc!=0) { |
| if(prevCC<0) { |
| /* the previous character was <_NORM_MIN_WITH_LEAD_CC, we need to get its trail cc */ |
| if(!nx_contains(nx, (UChar32)-prevCC)) { |
| prevCC=(int16_t)(_getFCD16((UChar)-prevCC)&0xff); |
| } else { |
| prevCC=0; /* excluded: fcd16==0 */ |
| } |
| } |
| |
| if(cc<prevCC) { |
| return FALSE; |
| } |
| } |
| prevCC=(int16_t)(fcd16&0xff); |
| } |
| } |
| |
| static UNormalizationCheckResult |
| _quickCheck(const UChar *src, |
| int32_t srcLength, |
| UNormalizationMode mode, |
| UBool allowMaybe, |
| const UnicodeSet *nx, |
| UErrorCode *pErrorCode) { |
| UChar stackBuffer[_STACK_BUFFER_CAPACITY]; |
| UChar *buffer; |
| int32_t bufferCapacity; |
| |
| const UChar *start, *limit; |
| uint32_t norm32, qcNorm32, ccOrQCMask, qcMask; |
| int32_t options; |
| UChar c, c2, minNoMaybe; |
| uint8_t cc, prevCC; |
| UNormalizationCheckResult result; |
| |
| /* check arguments */ |
| if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { |
| return UNORM_MAYBE; |
| } |
| |
| if(src==NULL || srcLength<-1) { |
| *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; |
| return UNORM_MAYBE; |
| } |
| |
| if(!_haveData(*pErrorCode)) { |
| return UNORM_MAYBE; |
| } |
| |
| /* check for a valid mode and set the quick check minimum and mask */ |
| switch(mode) { |
| case UNORM_NFC: |
| minNoMaybe=(UChar)indexes[_NORM_INDEX_MIN_NFC_NO_MAYBE]; |
| qcMask=_NORM_QC_NFC; |
| options=0; |
| break; |
| case UNORM_NFKC: |
| minNoMaybe=(UChar)indexes[_NORM_INDEX_MIN_NFKC_NO_MAYBE]; |
| qcMask=_NORM_QC_NFKC; |
| options=_NORM_OPTIONS_COMPAT; |
| break; |
| case UNORM_NFD: |
| minNoMaybe=(UChar)indexes[_NORM_INDEX_MIN_NFD_NO_MAYBE]; |
| qcMask=_NORM_QC_NFD; |
| options=0; |
| break; |
| case UNORM_NFKD: |
| minNoMaybe=(UChar)indexes[_NORM_INDEX_MIN_NFKD_NO_MAYBE]; |
| qcMask=_NORM_QC_NFKD; |
| options=_NORM_OPTIONS_COMPAT; |
| break; |
| case UNORM_FCD: |
| if(fcdTrie.index==NULL) { |
| *pErrorCode=U_UNSUPPORTED_ERROR; |
| return UNORM_MAYBE; |
| } |
| return unorm_checkFCD(src, srcLength, nx) ? UNORM_YES : UNORM_NO; |
| default: |
| *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; |
| return UNORM_MAYBE; |
| } |
| |
| /* initialize */ |
| buffer=stackBuffer; |
| bufferCapacity=_STACK_BUFFER_CAPACITY; |
| |
| ccOrQCMask=_NORM_CC_MASK|qcMask; |
| result=UNORM_YES; |
| prevCC=0; |
| |
| start=src; |
| if(srcLength>=0) { |
| /* string with length */ |
| limit=src+srcLength; |
| } else /* srcLength==-1 */ { |
| /* zero-terminated string */ |
| limit=NULL; |
| } |
| |
| U_ALIGN_CODE(16); |
| |
| for(;;) { |
| /* skip a run of code units below the minimum or with irrelevant data for the quick check */ |
| if(limit==NULL) { |
| for(;;) { |
| c=*src++; |
| if(c<minNoMaybe) { |
| if(c==0) { |
| goto endloop; /* break out of outer loop */ |
| } |
| } else if(((norm32=_getNorm32(c))&ccOrQCMask)!=0) { |
| break; |
| } |
| prevCC=0; |
| } |
| } else { |
| for(;;) { |
| if(src==limit) { |
| goto endloop; /* break out of outer loop */ |
| } else if((c=*src++)>=minNoMaybe && ((norm32=_getNorm32(c))&ccOrQCMask)!=0) { |
| break; |
| } |
| prevCC=0; |
| } |
| } |
| |
| /* check one above-minimum, relevant code unit */ |
| if(isNorm32LeadSurrogate(norm32)) { |
| /* c is a lead surrogate, get the real norm32 */ |
| if(src!=limit && UTF_IS_SECOND_SURROGATE(c2=*src)) { |
| ++src; |
| norm32=_getNorm32FromSurrogatePair(norm32, c2); |
| } else { |
| c2=0; |
| norm32=0; |
| } |
| } else { |
| c2=0; |
| } |
| |
| if(nx_contains(nx, c, c2)) { |
| /* excluded: norm32==0 */ |
| norm32=0; |
| } |
| |
| /* check the combining order */ |
| cc=(uint8_t)(norm32>>_NORM_CC_SHIFT); |
| if(cc!=0 && cc<prevCC) { |
| result=UNORM_NO; |
| break; |
| } |
| prevCC=cc; |
| |
| /* check for "no" or "maybe" quick check flags */ |
| qcNorm32=norm32&qcMask; |
| if(qcNorm32&_NORM_QC_ANY_NO) { |
| result=UNORM_NO; |
| break; |
| } else if(qcNorm32!=0) { |
| /* "maybe" can only occur for NFC and NFKC */ |
| if(allowMaybe) { |
| result=UNORM_MAYBE; |
| } else { |
| /* normalize a section around here to see if it is really normalized or not */ |
| const UChar *prevStarter; |
| uint32_t decompQCMask; |
| int32_t length; |
| |
| decompQCMask=(qcMask<<2)&0xf; /* decomposition quick check mask */ |
| |
| /* find the previous starter */ |
| prevStarter=src-1; /* set prevStarter to the beginning of the current character */ |
| if(UTF_IS_TRAIL(*prevStarter)) { |
| --prevStarter; /* safe because unpaired surrogates do not result in "maybe" */ |
| } |
| prevStarter=_findPreviousStarter(start, prevStarter, ccOrQCMask, decompQCMask, minNoMaybe); |
| |
| /* find the next true starter in [src..limit[ - modifies src to point to the next starter */ |
| src=_findNextStarter(src, limit, qcMask, decompQCMask, minNoMaybe); |
| |
| /* decompose and recompose [prevStarter..src[ */ |
| _composePart(stackBuffer, buffer, bufferCapacity, |
| length, |
| prevStarter, |
| src, |
| prevCC, |
| options, nx, pErrorCode); |
| if(U_FAILURE(*pErrorCode)) { |
| result=UNORM_MAYBE; /* error (out of memory) */ |
| break; |
| } |
| |
| /* compare the normalized version with the original */ |
| if(0!=uprv_strCompare(prevStarter, (int32_t)(src-prevStarter), buffer, length, FALSE, FALSE)) { |
| result=UNORM_NO; /* normalization differs */ |
| break; |
| } |
| |
| /* continue after the next starter */ |
| } |
| } |
| } |
| endloop: |
| |
| if(buffer!=stackBuffer) { |
| uprv_free(buffer); |
| } |
| |
| return result; |
| } |
| |
| U_CAPI UNormalizationCheckResult U_EXPORT2 |
| unorm_quickCheck(const UChar *src, |
| int32_t srcLength, |
| UNormalizationMode mode, |
| UErrorCode *pErrorCode) { |
| return _quickCheck(src, srcLength, mode, TRUE, NULL, pErrorCode); |
| } |
| |
| U_CAPI UNormalizationCheckResult U_EXPORT2 |
| unorm_quickCheckWithOptions(const UChar *src, int32_t srcLength, |
| UNormalizationMode mode, int32_t options, |
| UErrorCode *pErrorCode) { |
| return _quickCheck(src, srcLength, mode, TRUE, getNX(options, *pErrorCode), pErrorCode); |
| } |
| |
| U_CFUNC UNormalizationCheckResult |
| unorm_internalQuickCheck(const UChar *src, |
| int32_t srcLength, |
| UNormalizationMode mode, |
| UBool allowMaybe, |
| const UnicodeSet *nx, |
| UErrorCode *pErrorCode) { |
| return _quickCheck(src, srcLength, mode, allowMaybe, nx, pErrorCode); |
| } |
| |
| U_CAPI UBool U_EXPORT2 |
| unorm_isNormalized(const UChar *src, int32_t srcLength, |
| UNormalizationMode mode, |
| UErrorCode *pErrorCode) { |
| return (UBool)(UNORM_YES==_quickCheck(src, srcLength, mode, FALSE, NULL, pErrorCode)); |
| } |
| |
| U_CAPI UBool U_EXPORT2 |
| unorm_isNormalizedWithOptions(const UChar *src, int32_t srcLength, |
| UNormalizationMode mode, int32_t options, |
| UErrorCode *pErrorCode) { |
| return (UBool)(UNORM_YES==_quickCheck(src, srcLength, mode, FALSE, getNX(options, *pErrorCode), pErrorCode)); |
| } |
| |
| /* normalize() API ---------------------------------------------------------- */ |
| |
| /** |
| * Internal API for normalizing. |
| * Does not check for bad input. |
| * Requires _haveData() to be true. |
| * @internal |
| */ |
| U_CFUNC int32_t |
| unorm_internalNormalizeWithNX(UChar *dest, int32_t destCapacity, |
| const UChar *src, int32_t srcLength, |
| UNormalizationMode mode, int32_t options, const UnicodeSet *nx, |
| UErrorCode *pErrorCode) { |
| int32_t destLength; |
| uint8_t trailCC; |
| |
| switch(mode) { |
| case UNORM_NFD: |
| destLength=_decompose(dest, destCapacity, |
| src, srcLength, |
| FALSE, nx, trailCC); |
| break; |
| case UNORM_NFKD: |
| destLength=_decompose(dest, destCapacity, |
| src, srcLength, |
| TRUE, nx, trailCC); |
| break; |
| case UNORM_NFC: |
| destLength=_compose(dest, destCapacity, |
| src, srcLength, |
| options, nx, pErrorCode); |
| break; |
| case UNORM_NFKC: |
| destLength=_compose(dest, destCapacity, |
| src, srcLength, |
| options|_NORM_OPTIONS_COMPAT, nx, pErrorCode); |
| break; |
| case UNORM_FCD: |
| if(fcdTrie.index==NULL) { |
| *pErrorCode=U_UNSUPPORTED_ERROR; |
| return 0; |
| } |
| return unorm_makeFCD(dest, destCapacity, |
| src, srcLength, |
| nx, |
| pErrorCode); |
| #if 0 |
| case UNORM_FCC: |
| destLength=_compose(dest, destCapacity, |
| src, srcLength, |
| options|_NORM_OPTIONS_COMPOSE_CONTIGUOUS, nx, pErrorCode); |
| break; |
| #endif |
| case UNORM_NONE: |
| /* just copy the string */ |
| if(srcLength==-1) { |
| srcLength=u_strlen(src); |
| } |
| if(srcLength>0 && srcLength<=destCapacity) { |
| uprv_memcpy(dest, src, srcLength*U_SIZEOF_UCHAR); |
| } |
| destLength=srcLength; |
| break; |
| default: |
| *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; |
| return 0; |
| } |
| |
| return u_terminateUChars(dest, destCapacity, destLength, pErrorCode); |
| } |
| |
| /** |
| * Internal API for normalizing. |
| * Does not check for bad input. |
| * @internal |
| */ |
| U_CAPI int32_t U_EXPORT2 |
| unorm_internalNormalize(UChar *dest, int32_t destCapacity, |
| const UChar *src, int32_t srcLength, |
| UNormalizationMode mode, int32_t options, |
| UErrorCode *pErrorCode) { |
| const UnicodeSet *nx; |
| |
| if(!_haveData(*pErrorCode)) { |
| return 0; |
| } |
| |
| nx=getNX(options, *pErrorCode); |
| if(U_FAILURE(*pErrorCode)) { |
| return 0; |
| } |
| |
| /* reset options bits that should only be set inside unorm_internalNormalizeWithNX() */ |
| options&=~(_NORM_OPTIONS_SETS_MASK|_NORM_OPTIONS_COMPAT|_NORM_OPTIONS_COMPOSE_CONTIGUOUS); |
| |
| return unorm_internalNormalizeWithNX(dest, destCapacity, |
| src, srcLength, |
| mode, options, nx, |
| pErrorCode); |
| } |
| |
| /** Public API for normalizing. */ |
| U_CAPI int32_t U_EXPORT2 |
| unorm_normalize(const UChar *src, int32_t srcLength, |
| UNormalizationMode mode, int32_t options, |
| UChar *dest, int32_t destCapacity, |
| UErrorCode *pErrorCode) { |
| /* check argument values */ |
| if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { |
| return 0; |
| } |
| |
| if( destCapacity<0 || (dest==NULL && destCapacity>0) || |
| src==NULL || srcLength<-1 |
| ) { |
| *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; |
| return 0; |
| } |
| |
| /* check for overlapping src and destination */ |
| if( dest!=NULL && |
| ((src>=dest && src<(dest+destCapacity)) || |
| (srcLength>0 && dest>=src && dest<(src+srcLength))) |
| ) { |
| *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; |
| return 0; |
| } |
| |
| return unorm_internalNormalize(dest, destCapacity, |
| src, srcLength, |
| mode, options, |
| pErrorCode); |
| } |
| |
| |
| /* iteration functions ------------------------------------------------------ */ |
| |
| /* |
| * These iteration functions are the core implementations of the |
| * Normalizer class iteration API. |
| * They read from a UCharIterator into their own buffer |
| * and normalize into the Normalizer iteration buffer. |
| * Normalizer itself then iterates over its buffer until that needs to be |
| * filled again. |
| */ |
| |
| /* |
| * ### TODO: |
| * Now that UCharIterator.next/previous return (int32_t)-1 not (UChar)0xffff |
| * if iteration bounds are reached, |
| * try to not call hasNext/hasPrevious and instead check for >=0. |
| */ |
| |
| /* backward iteration ------------------------------------------------------- */ |
| |
| /* |
| * read backwards and get norm32 |
| * return 0 if the character is <minC |
| * if c2!=0 then (c2, c) is a surrogate pair (reversed - c2 is first surrogate but read second!) |
| */ |
| static inline uint32_t |
| _getPrevNorm32(UCharIterator &src, uint32_t minC, uint32_t mask, UChar &c, UChar &c2) { |
| uint32_t norm32; |
| |
| /* need src.hasPrevious() */ |
| c=(UChar)src.previous(&src); |
| c2=0; |
| |
| /* check for a surrogate before getting norm32 to see if we need to predecrement further */ |
| if(c<minC) { |
| return 0; |
| } else if(!UTF_IS_SURROGATE(c)) { |
| return _getNorm32(c); |
| } else if(UTF_IS_SURROGATE_FIRST(c) || !src.hasPrevious(&src)) { |
| /* unpaired surrogate */ |
| return 0; |
| } else if(UTF_IS_FIRST_SURROGATE(c2=(UChar)src.previous(&src))) { |
| norm32=_getNorm32(c2); |
| if((norm32&mask)==0) { |
| /* all surrogate pairs with this lead surrogate have irrelevant data */ |
| return 0; |
| } else { |
| /* norm32 must be a surrogate special */ |
| return _getNorm32FromSurrogatePair(norm32, c); |
| } |
| } else { |
| /* unpaired second surrogate, undo the c2=src.previous() movement */ |
| src.move(&src, 1, UITER_CURRENT); |
| c2=0; |
| return 0; |
| } |
| } |
| |
| /* |
| * read backwards and check if the character is a previous-iteration boundary |
| * if c2!=0 then (c2, c) is a surrogate pair (reversed - c2 is first surrogate but read second!) |
| */ |
| typedef UBool |
| IsPrevBoundaryFn(UCharIterator &src, uint32_t minC, uint32_t mask, UChar &c, UChar &c2); |
| |
| /* |
| * for NF*D: |
| * read backwards and check if the lead combining class is 0 |
| * if c2!=0 then (c2, c) is a surrogate pair (reversed - c2 is first surrogate but read second!) |
| */ |
| static UBool |
| _isPrevNFDSafe(UCharIterator &src, uint32_t minC, uint32_t ccOrQCMask, UChar &c, UChar &c2) { |
| return _isNFDSafe(_getPrevNorm32(src, minC, ccOrQCMask, c, c2), ccOrQCMask, ccOrQCMask&_NORM_QC_MASK); |
| } |
| |
| /* |
| * read backwards and check if the character is (or its decomposition begins with) |
| * a "true starter" (cc==0 and NF*C_YES) |
| * if c2!=0 then (c2, c) is a surrogate pair (reversed - c2 is first surrogate but read second!) |
| */ |
| static UBool |
| _isPrevTrueStarter(UCharIterator &src, uint32_t minC, uint32_t ccOrQCMask, UChar &c, UChar &c2) { |
| uint32_t norm32, decompQCMask; |
| |
| decompQCMask=(ccOrQCMask<<2)&0xf; /* decomposition quick check mask */ |
| norm32=_getPrevNorm32(src, minC, ccOrQCMask|decompQCMask, c, c2); |
| return _isTrueStarter(norm32, ccOrQCMask, decompQCMask); |
| } |
| |
| static int32_t |
| _findPreviousIterationBoundary(UCharIterator &src, |
| IsPrevBoundaryFn *isPrevBoundary, uint32_t minC, uint32_t mask, |
| UChar *&buffer, int32_t &bufferCapacity, |
| int32_t &startIndex, |
| UErrorCode *pErrorCode) { |
| UChar *stackBuffer; |
| UChar c, c2; |
| UBool isBoundary; |
| |
| /* initialize */ |
| stackBuffer=buffer; |
| startIndex=bufferCapacity; /* fill the buffer from the end backwards */ |
| |
| while(src.hasPrevious(&src)) { |
| isBoundary=isPrevBoundary(src, minC, mask, c, c2); |
| |
| /* always write this character to the front of the buffer */ |
| /* make sure there is enough space in the buffer */ |
| if(startIndex < (c2==0 ? 1 : 2)) { |
| int32_t bufferLength=bufferCapacity; |
| |
| if(!u_growBufferFromStatic(stackBuffer, &buffer, &bufferCapacity, 2*bufferCapacity, bufferLength)) { |
| *pErrorCode=U_MEMORY_ALLOCATION_ERROR; |
| src.move(&src, 0, UITER_START); |
| return 0; |
| } |
| |
| /* move the current buffer contents up */ |
| uprv_memmove(buffer+(bufferCapacity-bufferLength), buffer, bufferLength*U_SIZEOF_UCHAR); |
| startIndex+=bufferCapacity-bufferLength; |
| } |
| |
| buffer[--startIndex]=c; |
| if(c2!=0) { |
| buffer[--startIndex]=c2; |
| } |
| |
| /* stop if this just-copied character is a boundary */ |
| if(isBoundary) { |
| break; |
| } |
| } |
| |
| /* return the length of the buffer contents */ |
| return bufferCapacity-startIndex; |
| } |
| |
| U_CAPI int32_t U_EXPORT2 |
| unorm_previous(UCharIterator *src, |
| UChar *dest, int32_t destCapacity, |
| UNormalizationMode mode, int32_t options, |
| UBool doNormalize, UBool *pNeededToNormalize, |
| UErrorCode *pErrorCode) { |
| UChar stackBuffer[100]; |
| UChar *buffer=NULL; |
| IsPrevBoundaryFn *isPreviousBoundary=NULL; |
| uint32_t mask=0; |
| int32_t startIndex=0, bufferLength=0, bufferCapacity=0, destLength=0; |
| int32_t c=0, c2=0; |
| UChar minC=0; |
| |
| /* check argument values */ |
| if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { |
| return 0; |
| } |
| |
| if( destCapacity<0 || (dest==NULL && destCapacity>0) || |
| src==NULL |
| ) { |
| *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; |
| return 0; |
| } |
| |
| if(!_haveData(*pErrorCode)) { |
| return 0; |
| } |
| |
| if(pNeededToNormalize!=NULL) { |
| *pNeededToNormalize=FALSE; |
| } |
| |
| switch(mode) { |
| case UNORM_FCD: |
| if(fcdTrie.index==NULL) { |
| *pErrorCode=U_UNSUPPORTED_ERROR; |
| return 0; |
| } |
| /* fall through to NFD */ |
| case UNORM_NFD: |
| isPreviousBoundary=_isPrevNFDSafe; |
| minC=_NORM_MIN_WITH_LEAD_CC; |
| mask=_NORM_CC_MASK|_NORM_QC_NFD; |
| break; |
| case UNORM_NFKD: |
| isPreviousBoundary=_isPrevNFDSafe; |
| minC=_NORM_MIN_WITH_LEAD_CC; |
| mask=_NORM_CC_MASK|_NORM_QC_NFKD; |
| break; |
| case UNORM_NFC: |
| isPreviousBoundary=_isPrevTrueStarter; |
| minC=(UChar)indexes[_NORM_INDEX_MIN_NFC_NO_MAYBE]; |
| mask=_NORM_CC_MASK|_NORM_QC_NFC; |
| break; |
| case UNORM_NFKC: |
| isPreviousBoundary=_isPrevTrueStarter; |
| minC=(UChar)indexes[_NORM_INDEX_MIN_NFKC_NO_MAYBE]; |
| mask=_NORM_CC_MASK|_NORM_QC_NFKC; |
| break; |
| case UNORM_NONE: |
| destLength=0; |
| if((c=src->previous(src))>=0) { |
| destLength=1; |
| if(UTF_IS_TRAIL(c) && (c2=src->previous(src))>=0) { |
| if(UTF_IS_LEAD(c2)) { |
| if(destCapacity>=2) { |
| dest[1]=(UChar)c; /* trail surrogate */ |
| destLength=2; |
| } |
| c=c2; /* lead surrogate to be written below */ |
| } else { |
| src->move(src, 1, UITER_CURRENT); |
| } |
| } |
| |
| if(destCapacity>0) { |
| dest[0]=(UChar)c; |
| } |
| } |
| return u_terminateUChars(dest, destCapacity, destLength, pErrorCode); |
| default: |
| *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; |
| return 0; |
| } |
| |
| buffer=stackBuffer; |
| bufferCapacity=(int32_t)(sizeof(stackBuffer)/U_SIZEOF_UCHAR); |
| bufferLength=_findPreviousIterationBoundary(*src, |
| isPreviousBoundary, minC, mask, |
| buffer, bufferCapacity, |
| startIndex, |
| pErrorCode); |
| if(bufferLength>0) { |
| if(doNormalize) { |
| destLength=unorm_internalNormalize(dest, destCapacity, |
| buffer+startIndex, bufferLength, |
| mode, options, |
| pErrorCode); |
| if(pNeededToNormalize!=0 && U_SUCCESS(*pErrorCode)) { |
| *pNeededToNormalize= |
| (UBool)(destLength!=bufferLength || |
| 0!=uprv_memcmp(dest, buffer+startIndex, destLength*U_SIZEOF_UCHAR)); |
| } |
| } else { |
| /* just copy the source characters */ |
| if(destCapacity>0) { |
| uprv_memcpy(dest, buffer+startIndex, uprv_min(bufferLength, destCapacity)*U_SIZEOF_UCHAR); |
| } |
| destLength=u_terminateUChars(dest, destCapacity, bufferLength, pErrorCode); |
| } |
| } else { |
| destLength=u_terminateUChars(dest, destCapacity, 0, pErrorCode); |
| } |
| |
| /* cleanup */ |
| if(buffer!=stackBuffer) { |
| uprv_free(buffer); |
| } |
| |
| return destLength; |
| } |
| |
| /* forward iteration -------------------------------------------------------- */ |
| |
| /* |
| * read forward and get norm32 |
| * return 0 if the character is <minC |
| * if c2!=0 then (c2, c) is a surrogate pair |
| * always reads complete characters |
| */ |
| static inline uint32_t |
| _getNextNorm32(UCharIterator &src, uint32_t minC, uint32_t mask, UChar &c, UChar &c2) { |
| uint32_t norm32; |
| |
| /* need src.hasNext() to be true */ |
| c=(UChar)src.next(&src); |
| c2=0; |
| |
| if(c<minC) { |
| return 0; |
| } |
| |
| norm32=_getNorm32(c); |
| if(UTF_IS_FIRST_SURROGATE(c)) { |
| if(src.hasNext(&src) && UTF_IS_SECOND_SURROGATE(c2=(UChar)src.current(&src))) { |
| src.move(&src, 1, UITER_CURRENT); /* skip the c2 surrogate */ |
| if((norm32&mask)==0) { |
| /* irrelevant data */ |
| return 0; |
| } else { |
| /* norm32 must be a surrogate special */ |
| return _getNorm32FromSurrogatePair(norm32, c2); |
| } |
| } else { |
| /* unmatched surrogate */ |
| c2=0; |
| return 0; |
| } |
| } |
| return norm32; |
| } |
| |
| /* |
| * read forward and check if the character is a next-iteration boundary |
| * if c2!=0 then (c, c2) is a surrogate pair |
| */ |
| typedef UBool |
| IsNextBoundaryFn(UCharIterator &src, uint32_t minC, uint32_t mask, UChar &c, UChar &c2); |
| |
| /* |
| * for NF*D: |
| * read forward and check if the lead combining class is 0 |
| * if c2!=0 then (c, c2) is a surrogate pair |
| */ |
| static UBool |
| _isNextNFDSafe(UCharIterator &src, uint32_t minC, uint32_t ccOrQCMask, UChar &c, UChar &c2) { |
| return _isNFDSafe(_getNextNorm32(src, minC, ccOrQCMask, c, c2), ccOrQCMask, ccOrQCMask&_NORM_QC_MASK); |
| } |
| |
| /* |
| * for NF*C: |
| * read forward and check if the character is (or its decomposition begins with) |
| * a "true starter" (cc==0 and NF*C_YES) |
| * if c2!=0 then (c, c2) is a surrogate pair |
| */ |
| static UBool |
| _isNextTrueStarter(UCharIterator &src, uint32_t minC, uint32_t ccOrQCMask, UChar &c, UChar &c2) { |
| uint32_t norm32, decompQCMask; |
| |
| decompQCMask=(ccOrQCMask<<2)&0xf; /* decomposition quick check mask */ |
| norm32=_getNextNorm32(src, minC, ccOrQCMask|decompQCMask, c, c2); |
| return _isTrueStarter(norm32, ccOrQCMask, decompQCMask); |
| } |
| |
| static int32_t |
| _findNextIterationBoundary(UCharIterator &src, |
| IsNextBoundaryFn *isNextBoundary, uint32_t minC, uint32_t mask, |
| UChar *&buffer, int32_t &bufferCapacity, |
| UErrorCode *pErrorCode) { |
| UChar *stackBuffer; |
| int32_t bufferIndex; |
| UChar c, c2; |
| |
| if(!src.hasNext(&src)) { |
| return 0; |
| } |
| |
| /* initialize */ |
| stackBuffer=buffer; |
| |
| /* get one character and ignore its properties */ |
| buffer[0]=c=(UChar)src.next(&src); |
| bufferIndex=1; |
| if(UTF_IS_FIRST_SURROGATE(c) && src.hasNext(&src)) { |
| if(UTF_IS_SECOND_SURROGATE(c2=(UChar)src.next(&src))) { |
| buffer[bufferIndex++]=c2; |
| } else { |
| src.move(&src, -1, UITER_CURRENT); /* back out the non-trail-surrogate */ |
| } |
| } |
| |
| /* get all following characters until we see a boundary */ |
| /* checking hasNext() instead of c!=DONE on the off-chance that U+ffff is part of the string */ |
| while(src.hasNext(&src)) { |
| if(isNextBoundary(src, minC, mask, c, c2)) { |
| /* back out the latest movement to stop at the boundary */ |
| src.move(&src, c2==0 ? -1 : -2, UITER_CURRENT); |
| break; |
| } else { |
| if(bufferIndex+(c2==0 ? 1 : 2)<=bufferCapacity || |
| /* attempt to grow the buffer */ |
| u_growBufferFromStatic(stackBuffer, &buffer, &bufferCapacity, |
| 2*bufferCapacity, |
| bufferIndex) |
| ) { |
| buffer[bufferIndex++]=c; |
| if(c2!=0) { |
| buffer[bufferIndex++]=c2; |
| } |
| } else { |
| *pErrorCode=U_MEMORY_ALLOCATION_ERROR; |
| src.move(&src, 0, UITER_LIMIT); |
| return 0; |
| } |
| } |
| } |
| |
| /* return the length of the buffer contents */ |
| return bufferIndex; |
| } |
| |
| U_CAPI int32_t U_EXPORT2 |
| unorm_next(UCharIterator *src, |
| UChar *dest, int32_t destCapacity, |
| UNormalizationMode mode, int32_t options, |
| UBool doNormalize, UBool *pNeededToNormalize, |
| UErrorCode *pErrorCode) { |
| UChar stackBuffer[100]; |
| UChar *buffer; |
| IsNextBoundaryFn *isNextBoundary; |
| uint32_t mask; |
| int32_t bufferLength, bufferCapacity, destLength; |
| int32_t c, c2; |
| UChar minC; |
| |
| /* check argument values */ |
| if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { |
| return 0; |
| } |
| |
| if( destCapacity<0 || (dest==NULL && destCapacity>0) || |
| src==NULL |
| ) { |
| *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; |
| return 0; |
| } |
| |
| if(!_haveData(*pErrorCode)) { |
| return 0; |
| } |
| |
| if(pNeededToNormalize!=NULL) { |
| *pNeededToNormalize=FALSE; |
| } |
| |
| switch(mode) { |
| case UNORM_FCD: |
| if(fcdTrie.index==NULL) { |
| *pErrorCode=U_UNSUPPORTED_ERROR; |
| return 0; |
| } |
| /* fall through to NFD */ |
| case UNORM_NFD: |
| isNextBoundary=_isNextNFDSafe; |
| minC=_NORM_MIN_WITH_LEAD_CC; |
| mask=_NORM_CC_MASK|_NORM_QC_NFD; |
| break; |
| case UNORM_NFKD: |
| isNextBoundary=_isNextNFDSafe; |
| minC=_NORM_MIN_WITH_LEAD_CC; |
| mask=_NORM_CC_MASK|_NORM_QC_NFKD; |
| break; |
| case UNORM_NFC: |
| isNextBoundary=_isNextTrueStarter; |
| minC=(UChar)indexes[_NORM_INDEX_MIN_NFC_NO_MAYBE]; |
| mask=_NORM_CC_MASK|_NORM_QC_NFC; |
| break; |
| case UNORM_NFKC: |
| isNextBoundary=_isNextTrueStarter; |
| minC=(UChar)indexes[_NORM_INDEX_MIN_NFKC_NO_MAYBE]; |
| mask=_NORM_CC_MASK|_NORM_QC_NFKC; |
| break; |
| case UNORM_NONE: |
| destLength=0; |
| if((c=src->next(src))>=0) { |
| destLength=1; |
| if(UTF_IS_LEAD(c) && (c2=src->next(src))>=0) { |
| if(UTF_IS_TRAIL(c2)) { |
| if(destCapacity>=2) { |
| dest[1]=(UChar)c2; /* trail surrogate */ |
| destLength=2; |
| } |
| /* lead surrogate to be written below */ |
| } else { |
| src->move(src, -1, UITER_CURRENT); |
| } |
| } |
| |
| if(destCapacity>0) { |
| dest[0]=(UChar)c; |
| } |
| } |
| return u_terminateUChars(dest, destCapacity, destLength, pErrorCode); |
| default: |
| *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; |
| return 0; |
| } |
| |
| buffer=stackBuffer; |
| bufferCapacity=(int32_t)(sizeof(stackBuffer)/U_SIZEOF_UCHAR); |
| bufferLength=_findNextIterationBoundary(*src, |
| isNextBoundary, minC, mask, |
| buffer, bufferCapacity, |
| pErrorCode); |
| if(bufferLength>0) { |
| if(doNormalize) { |
| destLength=unorm_internalNormalize(dest, destCapacity, |
| buffer, bufferLength, |
| mode, options, |
| pErrorCode); |
| if(pNeededToNormalize!=0 && U_SUCCESS(*pErrorCode)) { |
| *pNeededToNormalize= |
| (UBool)(destLength!=bufferLength || |
| 0!=uprv_memcmp(dest, buffer, destLength*U_SIZEOF_UCHAR)); |
| } |
| } else { |
| /* just copy the source characters */ |
| if(destCapacity>0) { |
| uprv_memcpy(dest, buffer, uprv_min(bufferLength, destCapacity)*U_SIZEOF_UCHAR); |
| } |
| destLength=u_terminateUChars(dest, destCapacity, bufferLength, pErrorCode); |
| } |
| } else { |
| destLength=u_terminateUChars(dest, destCapacity, 0, pErrorCode); |
| } |
| |
| /* cleanup */ |
| if(buffer!=stackBuffer) { |
| uprv_free(buffer); |
| } |
| |
| return destLength; |
| } |
| |
| /* |
| * ### TODO: check if NF*D and FCD iteration finds optimal boundaries |
| * and if not, how hard it would be to improve it. |
| * For example, see _findSafeFCD(). |
| */ |
| |
| /* Concatenation of normalized strings -------------------------------------- */ |
| |
| U_CAPI int32_t U_EXPORT2 |
| unorm_concatenate(const UChar *left, int32_t leftLength, |
| const UChar *right, int32_t rightLength, |
| UChar *dest, int32_t destCapacity, |
| UNormalizationMode mode, int32_t options, |
| UErrorCode *pErrorCode) { |
| UChar stackBuffer[100]; |
| UChar *buffer; |
| int32_t bufferLength, bufferCapacity; |
| |
| UCharIterator iter; |
| int32_t leftBoundary, rightBoundary, destLength; |
| |
| /* check argument values */ |
| if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { |
| return 0; |
| } |
| |
| if( destCapacity<0 || (dest==NULL && destCapacity>0) || |
| left==NULL || leftLength<-1 || |
| right==NULL || rightLength<-1 |
| ) { |
| *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; |
| return 0; |
| } |
| |
| /* check for overlapping right and destination */ |
| if( dest!=NULL && |
| ((right>=dest && right<(dest+destCapacity)) || |
| (rightLength>0 && dest>=right && dest<(right+rightLength))) |
| ) { |
| *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; |
| return 0; |
| } |
| |
| /* allow left==dest */ |
| |
| /* set up intermediate buffer */ |
| buffer=stackBuffer; |
| bufferCapacity=(int32_t)(sizeof(stackBuffer)/U_SIZEOF_UCHAR); |
| |
| /* |
| * Input: left[0..leftLength[ + right[0..rightLength[ |
| * |
| * Find normalization-safe boundaries leftBoundary and rightBoundary |
| * and copy the end parts together: |
| * buffer=left[leftBoundary..leftLength[ + right[0..rightBoundary[ |
| * |
| * dest=left[0..leftBoundary[ + |
| * normalize(buffer) + |
| * right[rightBoundary..rightLength[ |
| */ |
| |
| /* |
| * find a normalization boundary at the end of the left string |
| * and copy the end part into the buffer |
| */ |
| uiter_setString(&iter, left, leftLength); |
| iter.index=leftLength=iter.length; /* end of left string */ |
| |
| bufferLength=unorm_previous(&iter, buffer, bufferCapacity, |
| mode, options, |
| FALSE, NULL, |
| pErrorCode); |
| leftBoundary=iter.index; |
| if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) { |
| *pErrorCode=U_ZERO_ERROR; |
| if(!u_growBufferFromStatic(stackBuffer, &buffer, &bufferCapacity, 2*bufferLength, 0)) { |
| *pErrorCode=U_MEMORY_ALLOCATION_ERROR; |
| /* dont need to cleanup here since |
| * u_growBufferFromStatic frees buffer if(buffer!=stackBuffer) |
| */ |
| return 0; |
| } |
| |
| /* just copy from the left string: we know the boundary already */ |
| uprv_memcpy(buffer, left+leftBoundary, bufferLength*U_SIZEOF_UCHAR); |
| } |
| |
| /* |
| * find a normalization boundary at the beginning of the right string |
| * and concatenate the beginning part to the buffer |
| */ |
| uiter_setString(&iter, right, rightLength); |
| rightLength=iter.length; /* in case it was -1 */ |
| |
| rightBoundary=unorm_next(&iter, buffer+bufferLength, bufferCapacity-bufferLength, |
| mode, options, |
| FALSE, NULL, |
| pErrorCode); |
| if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) { |
| *pErrorCode=U_ZERO_ERROR; |
| if(!u_growBufferFromStatic(stackBuffer, &buffer, &bufferCapacity, bufferLength+rightBoundary, 0)) { |
| *pErrorCode=U_MEMORY_ALLOCATION_ERROR; |
| /* dont need to cleanup here since |
| * u_growBufferFromStatic frees buffer if(buffer!=stackBuffer) |
| */ |
| return 0; |
| } |
| |
| /* just copy from the right string: we know the boundary already */ |
| uprv_memcpy(buffer+bufferLength, right, rightBoundary*U_SIZEOF_UCHAR); |
| } |
| |
| bufferLength+=rightBoundary; |
| |
| /* copy left[0..leftBoundary[ to dest */ |
| if(left!=dest && leftBoundary>0 && destCapacity>0) { |
| uprv_memcpy(dest, left, uprv_min(leftBoundary, destCapacity)*U_SIZEOF_UCHAR); |
| } |
| destLength=leftBoundary; |
| |
| /* concatenate the normalization of the buffer to dest */ |
| if(destCapacity>destLength) { |
| destLength+=unorm_internalNormalize(dest+destLength, destCapacity-destLength, |
| buffer, bufferLength, |
| mode, options, |
| pErrorCode); |
| } else { |
| destLength+=unorm_internalNormalize(NULL, 0, |
| buffer, bufferLength, |
| mode, options, |
| pErrorCode); |
| } |
| /* |
| * only errorCode that is expected is a U_BUFFER_OVERFLOW_ERROR |
| * so we dont check for the error code here..just let it pass through |
| */ |
| /* concatenate right[rightBoundary..rightLength[ to dest */ |
| right+=rightBoundary; |
| rightLength-=rightBoundary; |
| if(rightLength>0 && destCapacity>destLength) { |
| uprv_memcpy(dest+destLength, right, uprv_min(rightLength, destCapacity-destLength)*U_SIZEOF_UCHAR); |
| } |
| destLength+=rightLength; |
| |
| /* cleanup */ |
| if(buffer!=stackBuffer) { |
| uprv_free(buffer); |
| } |
| |
| return u_terminateUChars(dest, destCapacity, destLength, pErrorCode); |
| } |
| |
| #endif /* #if !UCONFIG_NO_NORMALIZATION */ |