| /* |
| ******************************************************************************* |
| * |
| * Copyright (C) 2003, International Business Machines |
| * Corporation and others. All Rights Reserved. |
| * |
| ******************************************************************************* |
| * file name: testidn.cpp |
| * encoding: US-ASCII |
| * tab size: 8 (not used) |
| * indentation:4 |
| * |
| * created on: 2003-02-06 |
| * created by: Ram Viswanadha |
| * |
| * This program reads the rfc3454_*.txt files, |
| * parses them, and extracts the data for Nameprep conformance. |
| * It then preprocesses it and writes a binary file for efficient use |
| * in various IDNA conversion processes. |
| */ |
| |
| #include <stdio.h> |
| #include <stdlib.h> |
| #include "unicode/utypes.h" |
| |
| #if !UCONFIG_NO_IDNA && !UCONFIG_NO_TRANSLITERATION |
| |
| #include "unicode/uchar.h" |
| #include "unicode/putil.h" |
| #include "cmemory.h" |
| #include "cstring.h" |
| #include "unicode/udata.h" |
| #include "unewdata.h" |
| #include "uoptions.h" |
| #include "uparse.h" |
| #include "utrie.h" |
| #include "umutex.h" |
| #include "sprpimpl.h" |
| #include "testidna.h" |
| |
| UBool beVerbose=FALSE, haveCopyright=TRUE; |
| |
| /* prototypes --------------------------------------------------------------- */ |
| |
| |
| static UBool isDataLoaded = FALSE; |
| static UTrie idnTrie={ 0,0,0,0,0,0,0 }; |
| static UDataMemory *idnData=NULL; |
| static UErrorCode dataErrorCode =U_ZERO_ERROR; |
| |
| |
| static const uint16_t* mappingData = NULL; |
| static int32_t indexes[_IDNA_INDEX_TOP]={ 0 }; |
| |
| |
| static void |
| parseMappings(const char *filename, UBool withNorm, UBool reportError,TestIDNA& test, UErrorCode *pErrorCode); |
| |
| static void |
| parseTable(const char *filename, UBool isUnassigned, TestIDNA& test, UErrorCode *pErrorCode); |
| |
| static UBool loadIDNData(UErrorCode &errorCode); |
| |
| static UBool cleanup(); |
| |
| static void |
| compareMapping(uint32_t codepoint, uint32_t* mapping, int32_t mapLength, |
| UBool withNorm); |
| |
| static void |
| compareFlagsForRange(uint32_t start, uint32_t end, |
| UBool isUnassigned); |
| |
| static void |
| testAllCodepoints(TestIDNA& test); |
| |
| static TestIDNA* pTestIDNA =NULL; |
| |
| static const char* fileNames[] = { |
| "rfc3454_A_1.txt", /* contains unassigned code points */ |
| "rfc3454_C_X.txt", /* contains code points that are prohibited */ |
| "rfc3454_B_1.txt", /* contains case mappings when normalization is turned off */ |
| "rfc3454_B_2.txt", /* contains case mappings when normalization it turned on */ |
| /* "NormalizationCorrections.txt",contains NFKC case mappings whicha are not included in UTR 21 */ |
| }; |
| /* -------------------------------------------------------------------------- */ |
| |
| /* file definitions */ |
| #define DATA_NAME "uidna" |
| #define DATA_TYPE "icu" |
| |
| #define MISC_DIR "misc" |
| |
| extern int |
| testData(TestIDNA& test) { |
| char* filename = (char*) malloc(strlen(IntlTest::pathToDataDirectory())*3); |
| //TODO get the srcDir dynamically |
| const char *srcDir=IntlTest::pathToDataDirectory(); |
| char *basename=NULL; |
| UErrorCode errorCode=U_ZERO_ERROR; |
| char *saveBasename =NULL; |
| |
| loadIDNData(errorCode); |
| if(U_FAILURE(dataErrorCode)){ |
| test.errln( "Could not load data. Error: %s\n",u_errorName(dataErrorCode)); |
| return dataErrorCode; |
| } |
| |
| //initialize |
| pTestIDNA = &test; |
| |
| /* prepare the filename beginning with the source dir */ |
| if(uprv_strchr(srcDir,U_FILE_SEP_CHAR) == NULL){ |
| filename[0] = 0x2E; |
| filename[1] = U_FILE_SEP_CHAR; |
| uprv_strcpy(filename+2,srcDir); |
| }else{ |
| uprv_strcpy(filename, srcDir); |
| } |
| basename=filename+uprv_strlen(filename); |
| if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) { |
| *basename++=U_FILE_SEP_CHAR; |
| } |
| |
| /* process unassigned */ |
| basename=filename+uprv_strlen(filename); |
| if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) { |
| *basename++=U_FILE_SEP_CHAR; |
| } |
| |
| /* first copy misc directory */ |
| saveBasename = basename; |
| uprv_strcpy(basename,MISC_DIR); |
| basename = basename + uprv_strlen(MISC_DIR); |
| *basename++=U_FILE_SEP_CHAR; |
| |
| /* process unassigned */ |
| uprv_strcpy(basename,fileNames[0]); |
| parseTable(filename,TRUE, test,&errorCode); |
| if(U_FAILURE(errorCode)) { |
| test.errln( "Could not open file %s for reading \n", filename); |
| return errorCode; |
| } |
| /* process prohibited */ |
| uprv_strcpy(basename,fileNames[1]); |
| parseTable(filename,FALSE, test, &errorCode); |
| if(U_FAILURE(errorCode)) { |
| test.errln( "Could not open file %s for reading \n", filename); |
| return errorCode; |
| } |
| |
| /* process mappings */ |
| uprv_strcpy(basename,fileNames[2]); |
| parseMappings(filename, FALSE, FALSE,test, &errorCode); |
| if(U_FAILURE(errorCode)) { |
| test.errln( "Could not open file %s for reading \n", filename); |
| return errorCode; |
| } |
| uprv_strcpy(basename,fileNames[3]); |
| parseMappings(filename, TRUE, FALSE,test, &errorCode); |
| if(U_FAILURE(errorCode)) { |
| test.errln( "Could not open file %s for reading \n", filename); |
| return errorCode; |
| } |
| |
| testAllCodepoints(test); |
| |
| cleanup(); |
| pTestIDNA = NULL; |
| free(filename); |
| return errorCode; |
| } |
| U_CDECL_BEGIN |
| static void U_CALLCONV |
| caseMapLineFn(void *context, |
| char *fields[][2], int32_t /*fieldCount*/, |
| UErrorCode *pErrorCode) { |
| uint32_t mapping[40]; |
| char *end, *s; |
| uint32_t code; |
| int32_t length; |
| UBool* mapWithNorm = (UBool*) context; |
| |
| /* get the character code, field 0 */ |
| code=(uint32_t)uprv_strtoul(fields[0][0], &end, 16); |
| if(end<=fields[0][0] || end!=fields[0][1]) { |
| *pErrorCode=U_PARSE_ERROR; |
| |
| } |
| |
| s = fields[1][0]; |
| /* parse the mapping string */ |
| length=u_parseCodePoints(s, mapping, sizeof(mapping)/4, pErrorCode); |
| |
| /* store the mapping */ |
| |
| compareMapping(code,mapping, length, *mapWithNorm); |
| } |
| U_CDECL_END |
| |
| static void |
| parseMappings(const char *filename,UBool withNorm, UBool reportError, TestIDNA& test, UErrorCode *pErrorCode) { |
| char *fields[3][2]; |
| |
| if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { |
| return; |
| } |
| |
| u_parseDelimitedFile(filename, ';', fields, 3, caseMapLineFn, &withNorm, pErrorCode); |
| |
| //fprintf(stdout,"Number of code points that have mappings with length >1 : %i\n",len); |
| |
| if(U_FAILURE(*pErrorCode) && (reportError || *pErrorCode!=U_FILE_ACCESS_ERROR)) { |
| test.errln( "testidn error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode)); |
| } |
| } |
| |
| /* parser for UnicodeData.txt ----------------------------------------------- */ |
| U_CDECL_BEGIN |
| |
| static void U_CALLCONV |
| unicodeDataLineFn(void *context, |
| char *fields[][2], int32_t /*fieldCount*/, |
| UErrorCode *pErrorCode) { |
| uint32_t rangeStart=0,rangeEnd =0; |
| UBool* isUnassigned = (UBool*) context; |
| |
| u_parseCodePointRange(fields[0][0], &rangeStart,&rangeEnd, pErrorCode); |
| |
| if(U_FAILURE(*pErrorCode)){ |
| *pErrorCode = U_PARSE_ERROR; |
| return; |
| } |
| |
| |
| compareFlagsForRange(rangeStart,rangeEnd,*isUnassigned); |
| |
| } |
| |
| U_CDECL_END |
| |
| static void |
| parseTable(const char *filename,UBool isUnassigned,TestIDNA& test, UErrorCode *pErrorCode) { |
| char *fields[2][2]; |
| if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { |
| return; |
| } |
| |
| u_parseDelimitedFile(filename, ';', fields, 1, unicodeDataLineFn, &isUnassigned, pErrorCode); |
| |
| |
| if(U_FAILURE(*pErrorCode)) { |
| test.errln( "testidn error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode)); |
| } |
| } |
| |
| static void |
| testAllCodepoints(TestIDNA& test){ |
| if(isDataLoaded){ |
| uint32_t i = 0; |
| int32_t unassigned = 0; |
| int32_t prohibited = 0; |
| int32_t mappedWithNorm = 0; |
| int32_t mapped = 0; |
| int32_t noValueInTrie = 0; |
| |
| |
| for(i=0;i<=0x10FFFF;i++){ |
| uint32_t result = 0; |
| UTRIE_GET16(&idnTrie,i, result); |
| |
| if(result != UIDNA_NO_VALUE ){ |
| if((result & 0x07) == UIDNA_UNASSIGNED){ |
| unassigned++; |
| } |
| if((result & 0x07) == UIDNA_PROHIBITED){ |
| prohibited++; |
| } |
| if((result>>5) == _IDNA_MAP_TO_NOTHING){ |
| mapped++; |
| } |
| if((result & 0x07) == UIDNA_MAP_NFKC){ |
| mappedWithNorm++; |
| } |
| }else{ |
| noValueInTrie++; |
| if(result > 0){ |
| test.errln("The return value for 0x%06X is wrong. %i\n",i,result); |
| } |
| } |
| } |
| |
| test.logln("Number of Unassinged code points : %i \n",unassigned); |
| test.logln("Number of Prohibited code points : %i \n",prohibited); |
| test.logln("Number of Mapped code points : %i \n",mapped); |
| test.logln("Number of Mapped with NFKC code points : %i \n",mappedWithNorm); |
| test.logln("Number of code points that have no value in Trie: %i \n",noValueInTrie); |
| |
| } |
| } |
| |
| static inline void getValues(uint32_t result, int8_t& flag, |
| int8_t& length, int32_t& index){ |
| /* first 3 bits contain the flag */ |
| flag = (int8_t) (result & 0x07); |
| /* next 2 bits contain the length */ |
| length = (int8_t) ((result>>3) & 0x03); |
| /* next 11 bits contain the index */ |
| index = (result>> 5); |
| } |
| |
| static void |
| compareMapping(uint32_t codepoint, uint32_t* mapping,int32_t mapLength, |
| UBool withNorm){ |
| if(isDataLoaded){ |
| uint32_t result = 0; |
| UTRIE_GET16(&idnTrie,codepoint, result); |
| |
| int8_t flag, length; |
| int32_t index; |
| getValues(result,flag,length, index); |
| |
| |
| if(withNorm){ |
| if(flag != UIDNA_MAP_NFKC){ |
| pTestIDNA->errln( "Did not get the assigned flag for codepoint 0x%08X. Expected: %i Got: %i\n",codepoint, UIDNA_MAP_NFKC, flag); |
| } |
| }else{ |
| if(flag==UIDNA_NO_VALUE || flag == UIDNA_PROHIBITED){ |
| if(index != _IDNA_MAP_TO_NOTHING ){ |
| pTestIDNA->errln( "Did not get the assigned flag for codepoint 0x%08X. Expected: %i Got: %i\n", codepoint, _IDNA_MAP_TO_NOTHING, index); |
| } |
| } |
| } |
| if(length ==_IDNA_LENGTH_IN_MAPPING_TABLE){ |
| length = (int8_t)mappingData[index]; |
| index++; |
| } |
| int32_t realLength =0; |
| /* figure out the real length */ |
| for(int32_t j=0; j<mapLength; j++){ |
| if(mapping[j] > 0xFFFF){ |
| realLength +=2; |
| }else{ |
| realLength++; |
| } |
| } |
| |
| if(realLength != length){ |
| pTestIDNA->errln( "Did not get the expected length. Expected: %i Got: %i\n", mapLength, length); |
| } |
| |
| |
| for(int8_t i =0; i< mapLength; i++){ |
| if(mapping[i] <= 0xFFFF){ |
| if(mappingData[index+i] != (uint16_t)mapping[i]){ |
| pTestIDNA->errln("Did not get the expected result. Expected: 0x%04X Got: 0x%04X \n", mapping[i], mappingData[index+i]); |
| } |
| }else{ |
| UChar lead = UTF16_LEAD(mapping[i]); |
| UChar trail = UTF16_TRAIL(mapping[i]); |
| if(mappingData[index+i] != lead || |
| mappingData[index+i+1] != trail){ |
| pTestIDNA->errln( "Did not get the expected result. Expected: 0x%04X 0x%04X Got: 0x%04X 0x%04X", lead, trail, mappingData[index+i], mappingData[index+i+1]); |
| } |
| } |
| } |
| |
| } |
| |
| } |
| |
| static void |
| compareFlagsForRange(uint32_t start, uint32_t end, |
| UBool isUnassigned){ |
| if(isDataLoaded){ |
| uint32_t result =0 ; |
| while(start < end+1){ |
| UTRIE_GET16(&idnTrie,start, result); |
| if(isUnassigned){ |
| if(result != UIDNA_UNASSIGNED){ |
| pTestIDNA->errln( "UIDNA_UASSIGNED flag failed for 0x%06X. Expected: %04X Got: %04X\n",start,UIDNA_UNASSIGNED, result); |
| } |
| }else{ |
| if((result & 0x03) != UIDNA_PROHIBITED){ |
| pTestIDNA->errln( "UIDNA_PROHIBITED flag failed for 0x%06X. Expected: %04X Got: %04X\n\n",start,UIDNA_PROHIBITED, result); |
| } |
| } |
| start++; |
| } |
| } |
| } |
| |
| UBool |
| cleanup() { |
| if(idnData!=NULL) { |
| udata_close(idnData); |
| idnData=NULL; |
| } |
| dataErrorCode=U_ZERO_ERROR; |
| isDataLoaded=FALSE; |
| |
| return TRUE; |
| } |
| U_CDECL_BEGIN |
| static UBool U_CALLCONV |
| isAcceptable(void * /* context */, |
| const char * /* type */, const char * /* name */, |
| const UDataInfo *pInfo) { |
| if( |
| pInfo->size>=20 && |
| pInfo->isBigEndian==U_IS_BIG_ENDIAN && |
| pInfo->charsetFamily==U_CHARSET_FAMILY && |
| pInfo->dataFormat[0]==0x49 && /* dataFormat="IDNA" 0x49, 0x44, 0x4e, 0x41 */ |
| pInfo->dataFormat[1]==0x44 && |
| pInfo->dataFormat[2]==0x4e && |
| pInfo->dataFormat[3]==0x41 && |
| pInfo->formatVersion[0]==2 && |
| pInfo->formatVersion[2]==UTRIE_SHIFT && |
| pInfo->formatVersion[3]==UTRIE_INDEX_SHIFT |
| ) { |
| return TRUE; |
| } else { |
| return FALSE; |
| } |
| } |
| |
| /* idnTrie: the folding offset is the lead FCD value itself */ |
| static int32_t U_CALLCONV |
| getFoldingOffset(uint32_t data) { |
| if(data&0x8000) { |
| return (int32_t)(data&0x7fff); |
| } else { |
| return 0; |
| } |
| } |
| U_CDECL_END |
| |
| static UBool |
| loadIDNData(UErrorCode &errorCode) { |
| /* load Unicode normalization data from file */ |
| if(isDataLoaded==FALSE) { |
| UTrie _idnTrie={ 0,0,0,0,0,0,0 }; |
| UDataMemory *data; |
| const int32_t *p=NULL; |
| const uint8_t *pb; |
| if(&errorCode==NULL || U_FAILURE(errorCode)) { |
| return 0; |
| } |
| |
| /* open the data outside the mutex block */ |
| data=udata_openChoice(NULL, DATA_TYPE, DATA_NAME, isAcceptable, NULL, &errorCode); |
| dataErrorCode=errorCode; |
| if(U_FAILURE(errorCode)) { |
| return isDataLoaded=FALSE; |
| } |
| |
| p=(const int32_t *)udata_getMemory(data); |
| pb=(const uint8_t *)(p+_IDNA_INDEX_TOP); |
| utrie_unserialize(&_idnTrie, pb, p[_IDNA_INDEX_TRIE_SIZE], &errorCode); |
| _idnTrie.getFoldingOffset=getFoldingOffset; |
| |
| |
| if(U_FAILURE(errorCode)) { |
| dataErrorCode=errorCode; |
| udata_close(data); |
| return isDataLoaded=FALSE; |
| } |
| |
| /* in the mutex block, set the data for this process */ |
| umtx_lock(NULL); |
| if(idnData==NULL) { |
| idnData=data; |
| data=NULL; |
| uprv_memcpy(&indexes, p, sizeof(indexes)); |
| uprv_memcpy(&idnTrie, &_idnTrie, sizeof(UTrie)); |
| } else { |
| p=(const int32_t *)udata_getMemory(idnData); |
| } |
| umtx_unlock(NULL); |
| /* initialize some variables */ |
| mappingData=(uint16_t *)((uint8_t *)(p+_IDNA_INDEX_TOP)+indexes[_IDNA_INDEX_TRIE_SIZE]); |
| |
| isDataLoaded = TRUE; |
| |
| /* if a different thread set it first, then close the extra data */ |
| if(data!=NULL) { |
| udata_close(data); /* NULL if it was set correctly */ |
| } |
| } |
| |
| return isDataLoaded; |
| } |
| |
| #endif /* #if !UCONFIG_NO_IDNA */ |
| |
| /* |
| * Hey, Emacs, please set the following: |
| * |
| * Local Variables: |
| * indent-tabs-mode: nil |
| * End: |
| * |
| */ |
