| /* |
| ******************************************************************************* |
| * |
| * Copyright (C) 2002-2009, International Business Machines |
| * Corporation and others. All Rights Reserved. |
| * |
| ******************************************************************************* |
| * file name: props2.c |
| * encoding: US-ASCII |
| * tab size: 8 (not used) |
| * indentation:4 |
| * |
| * created on: 2002feb24 |
| * created by: Markus W. Scherer |
| * |
| * Parse more Unicode Character Database files and store |
| * additional Unicode character properties in bit set vectors. |
| */ |
| |
| #include <stdio.h> |
| #include "unicode/utypes.h" |
| #include "unicode/uchar.h" |
| #include "unicode/uscript.h" |
| #include "cstring.h" |
| #include "cmemory.h" |
| #include "utrie.h" |
| #include "uprops.h" |
| #include "propsvec.h" |
| #include "uparse.h" |
| #include "writesrc.h" |
| #include "genprops.h" |
| |
| #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) |
| |
| /* data --------------------------------------------------------------------- */ |
| |
| static UNewTrie *newTrie; |
| UPropsVectors *pv; |
| |
| /* miscellaneous ------------------------------------------------------------ */ |
| |
| static char * |
| trimTerminateField(char *s, char *limit) { |
| /* trim leading whitespace */ |
| s=(char *)u_skipWhitespace(s); |
| |
| /* trim trailing whitespace */ |
| while(s<limit && (*(limit-1)==' ' || *(limit-1)=='\t')) { |
| --limit; |
| } |
| *limit=0; |
| |
| return s; |
| } |
| |
| static void |
| parseTwoFieldFile(char *filename, char *basename, |
| const char *ucdFile, const char *suffix, |
| UParseLineFn *lineFn, |
| UErrorCode *pErrorCode) { |
| char *fields[2][2]; |
| |
| if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { |
| return; |
| } |
| |
| writeUCDFilename(basename, ucdFile, suffix); |
| |
| u_parseDelimitedFile(filename, ';', fields, 2, lineFn, NULL, pErrorCode); |
| if(U_FAILURE(*pErrorCode)) { |
| fprintf(stderr, "error parsing %s.txt: %s\n", ucdFile, u_errorName(*pErrorCode)); |
| } |
| } |
| |
| static void U_CALLCONV |
| ageLineFn(void *context, |
| char *fields[][2], int32_t fieldCount, |
| UErrorCode *pErrorCode); |
| |
| static void |
| parseMultiFieldFile(char *filename, char *basename, |
| const char *ucdFile, const char *suffix, |
| int32_t fieldCount, |
| UParseLineFn *lineFn, |
| UErrorCode *pErrorCode) { |
| char *fields[20][2]; |
| |
| if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { |
| return; |
| } |
| |
| writeUCDFilename(basename, ucdFile, suffix); |
| |
| u_parseDelimitedFile(filename, ';', fields, fieldCount, lineFn, NULL, pErrorCode); |
| if(U_FAILURE(*pErrorCode)) { |
| fprintf(stderr, "error parsing %s.txt: %s\n", ucdFile, u_errorName(*pErrorCode)); |
| } |
| } |
| |
| static void U_CALLCONV |
| numericLineFn(void *context, |
| char *fields[][2], int32_t fieldCount, |
| UErrorCode *pErrorCode); |
| |
| /* parse files with single enumerated properties ---------------------------- */ |
| |
| struct SingleEnum { |
| const char *ucdFile, *propName; |
| UProperty prop; |
| int32_t vecWord, vecShift; |
| uint32_t vecMask; |
| }; |
| typedef struct SingleEnum SingleEnum; |
| |
| static void |
| parseSingleEnumFile(char *filename, char *basename, const char *suffix, |
| const SingleEnum *sen, |
| UErrorCode *pErrorCode); |
| |
| static const SingleEnum scriptSingleEnum={ |
| "Scripts", "script", |
| UCHAR_SCRIPT, |
| 0, 0, UPROPS_SCRIPT_MASK |
| }; |
| |
| static const SingleEnum blockSingleEnum={ |
| "Blocks", "block", |
| UCHAR_BLOCK, |
| 0, UPROPS_BLOCK_SHIFT, UPROPS_BLOCK_MASK |
| }; |
| |
| static const SingleEnum graphemeClusterBreakSingleEnum={ |
| "GraphemeBreakProperty", "Grapheme_Cluster_Break", |
| UCHAR_GRAPHEME_CLUSTER_BREAK, |
| 2, UPROPS_GCB_SHIFT, UPROPS_GCB_MASK |
| }; |
| |
| static const SingleEnum wordBreakSingleEnum={ |
| "WordBreakProperty", "Word_Break", |
| UCHAR_WORD_BREAK, |
| 2, UPROPS_WB_SHIFT, UPROPS_WB_MASK |
| }; |
| |
| static const SingleEnum sentenceBreakSingleEnum={ |
| "SentenceBreakProperty", "Sentence_Break", |
| UCHAR_SENTENCE_BREAK, |
| 2, UPROPS_SB_SHIFT, UPROPS_SB_MASK |
| }; |
| |
| static const SingleEnum lineBreakSingleEnum={ |
| "LineBreak", "line break", |
| UCHAR_LINE_BREAK, |
| UPROPS_LB_VWORD, UPROPS_LB_SHIFT, UPROPS_LB_MASK |
| }; |
| |
| static const SingleEnum eawSingleEnum={ |
| "EastAsianWidth", "east asian width", |
| UCHAR_EAST_ASIAN_WIDTH, |
| 0, UPROPS_EA_SHIFT, UPROPS_EA_MASK |
| }; |
| |
| static void U_CALLCONV |
| singleEnumLineFn(void *context, |
| char *fields[][2], int32_t fieldCount, |
| UErrorCode *pErrorCode) { |
| const SingleEnum *sen; |
| char *s; |
| uint32_t start, end, uv; |
| int32_t value; |
| |
| sen=(const SingleEnum *)context; |
| |
| u_parseCodePointRange(fields[0][0], &start, &end, pErrorCode); |
| if(U_FAILURE(*pErrorCode)) { |
| fprintf(stderr, "genprops: syntax error in %s.txt field 0 at %s\n", sen->ucdFile, fields[0][0]); |
| exit(*pErrorCode); |
| } |
| |
| /* parse property alias */ |
| s=trimTerminateField(fields[1][0], fields[1][1]); |
| value=u_getPropertyValueEnum(sen->prop, s); |
| if(value<0) { |
| if(sen->prop==UCHAR_BLOCK) { |
| if(isToken("Greek", s)) { |
| value=UBLOCK_GREEK; /* Unicode 3.2 renames this to "Greek and Coptic" */ |
| } else if(isToken("Combining Marks for Symbols", s)) { |
| value=UBLOCK_COMBINING_MARKS_FOR_SYMBOLS; /* Unicode 3.2 renames this to "Combining Diacritical Marks for Symbols" */ |
| } else if(isToken("Private Use", s)) { |
| value=UBLOCK_PRIVATE_USE; /* Unicode 3.2 renames this to "Private Use Area" */ |
| } |
| } |
| } |
| if(value<0) { |
| fprintf(stderr, "genprops error: unknown %s name in %s.txt field 1 at %s\n", |
| sen->propName, sen->ucdFile, s); |
| exit(U_PARSE_ERROR); |
| } |
| |
| uv=(uint32_t)(value<<sen->vecShift); |
| if((uv&sen->vecMask)!=uv) { |
| fprintf(stderr, "genprops error: %s value overflow (0x%x) at %s\n", |
| sen->propName, (int)uv, s); |
| exit(U_INTERNAL_PROGRAM_ERROR); |
| } |
| |
| if(start==0 && end==0x10ffff) { |
| /* Also set bits for initialValue and errorValue. */ |
| end=UPVEC_MAX_CP; |
| } |
| upvec_setValue(pv, start, end, sen->vecWord, uv, sen->vecMask, pErrorCode); |
| if(U_FAILURE(*pErrorCode)) { |
| fprintf(stderr, "genprops error: unable to set %s code: %s\n", |
| sen->propName, u_errorName(*pErrorCode)); |
| exit(*pErrorCode); |
| } |
| } |
| |
| static void |
| parseSingleEnumFile(char *filename, char *basename, const char *suffix, |
| const SingleEnum *sen, |
| UErrorCode *pErrorCode) { |
| char *fields[2][2]; |
| |
| if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { |
| return; |
| } |
| |
| writeUCDFilename(basename, sen->ucdFile, suffix); |
| |
| u_parseDelimitedFile(filename, ';', fields, 2, singleEnumLineFn, (void *)sen, pErrorCode); |
| if(U_FAILURE(*pErrorCode)) { |
| fprintf(stderr, "error parsing %s.txt: %s\n", sen->ucdFile, u_errorName(*pErrorCode)); |
| } |
| } |
| |
| /* parse files with multiple binary properties ------------------------------ */ |
| |
| struct Binary { |
| const char *propName; |
| int32_t vecWord, vecShift; |
| }; |
| typedef struct Binary Binary; |
| |
| struct Binaries { |
| const char *ucdFile; |
| const Binary *binaries; |
| int32_t binariesCount; |
| }; |
| typedef struct Binaries Binaries; |
| |
| static const Binary |
| propListNames[]={ |
| { "White_Space", 1, UPROPS_WHITE_SPACE }, |
| { "Dash", 1, UPROPS_DASH }, |
| { "Hyphen", 1, UPROPS_HYPHEN }, |
| { "Quotation_Mark", 1, UPROPS_QUOTATION_MARK }, |
| { "Terminal_Punctuation", 1, UPROPS_TERMINAL_PUNCTUATION }, |
| { "Hex_Digit", 1, UPROPS_HEX_DIGIT }, |
| { "ASCII_Hex_Digit", 1, UPROPS_ASCII_HEX_DIGIT }, |
| { "Ideographic", 1, UPROPS_IDEOGRAPHIC }, |
| { "Diacritic", 1, UPROPS_DIACRITIC }, |
| { "Extender", 1, UPROPS_EXTENDER }, |
| { "Noncharacter_Code_Point", 1, UPROPS_NONCHARACTER_CODE_POINT }, |
| { "Grapheme_Link", 1, UPROPS_GRAPHEME_LINK }, |
| { "IDS_Binary_Operator", 1, UPROPS_IDS_BINARY_OPERATOR }, |
| { "IDS_Trinary_Operator", 1, UPROPS_IDS_TRINARY_OPERATOR }, |
| { "Radical", 1, UPROPS_RADICAL }, |
| { "Unified_Ideograph", 1, UPROPS_UNIFIED_IDEOGRAPH }, |
| { "Deprecated", 1, UPROPS_DEPRECATED }, |
| { "Logical_Order_Exception", 1, UPROPS_LOGICAL_ORDER_EXCEPTION }, |
| |
| /* new properties in Unicode 4.0.1 */ |
| { "STerm", 1, UPROPS_S_TERM }, |
| { "Variation_Selector", 1, UPROPS_VARIATION_SELECTOR }, |
| |
| /* new properties in Unicode 4.1 */ |
| { "Pattern_Syntax", 1, UPROPS_PATTERN_SYNTAX }, |
| { "Pattern_White_Space", 1, UPROPS_PATTERN_WHITE_SPACE } |
| }; |
| |
| static const Binaries |
| propListBinaries={ |
| "PropList", propListNames, LENGTHOF(propListNames) |
| }; |
| |
| static const Binary |
| derCorePropsNames[]={ |
| { "XID_Start", 1, UPROPS_XID_START }, |
| { "XID_Continue", 1, UPROPS_XID_CONTINUE }, |
| |
| /* before Unicode 4/ICU 2.6/format version 3.2, these used to be Other_XYZ from PropList.txt */ |
| { "Math", 1, UPROPS_MATH }, |
| { "Alphabetic", 1, UPROPS_ALPHABETIC }, |
| { "Grapheme_Extend", 1, UPROPS_GRAPHEME_EXTEND }, |
| { "Default_Ignorable_Code_Point", 1, UPROPS_DEFAULT_IGNORABLE_CODE_POINT }, |
| |
| /* new properties bits in ICU 2.6/format version 3.2 */ |
| { "ID_Start", 1, UPROPS_ID_START }, |
| { "ID_Continue", 1, UPROPS_ID_CONTINUE }, |
| { "Grapheme_Base", 1, UPROPS_GRAPHEME_BASE }, |
| |
| /* |
| * Unicode 5/ICU 3.6 moves Grapheme_Link from PropList.txt |
| * to DerivedCoreProperties.txt and deprecates it. |
| */ |
| { "Grapheme_Link", 1, UPROPS_GRAPHEME_LINK } |
| }; |
| |
| static const Binaries |
| derCorePropsBinaries={ |
| "DerivedCoreProperties", derCorePropsNames, LENGTHOF(derCorePropsNames) |
| }; |
| |
| static char ignoredProps[100][64]; |
| static int32_t ignoredPropsCount; |
| |
| static void |
| addIgnoredProp(char *s, char *limit) { |
| int32_t i; |
| |
| s=trimTerminateField(s, limit); |
| for(i=0; i<ignoredPropsCount; ++i) { |
| if(0==uprv_strcmp(ignoredProps[i], s)) { |
| return; |
| } |
| } |
| uprv_strcpy(ignoredProps[ignoredPropsCount++], s); |
| } |
| |
| static void U_CALLCONV |
| binariesLineFn(void *context, |
| char *fields[][2], int32_t fieldCount, |
| UErrorCode *pErrorCode) { |
| const Binaries *bin; |
| char *s; |
| uint32_t start, end, uv; |
| int32_t i; |
| |
| bin=(const Binaries *)context; |
| |
| u_parseCodePointRange(fields[0][0], &start, &end, pErrorCode); |
| if(U_FAILURE(*pErrorCode)) { |
| fprintf(stderr, "genprops: syntax error in %s.txt field 0 at %s\n", bin->ucdFile, fields[0][0]); |
| exit(*pErrorCode); |
| } |
| |
| /* parse binary property name */ |
| s=(char *)u_skipWhitespace(fields[1][0]); |
| for(i=0;; ++i) { |
| if(i==bin->binariesCount) { |
| /* ignore unrecognized properties */ |
| if(beVerbose) { |
| addIgnoredProp(s, fields[1][1]); |
| } |
| return; |
| } |
| if(isToken(bin->binaries[i].propName, s)) { |
| break; |
| } |
| } |
| |
| if(bin->binaries[i].vecShift>=32) { |
| fprintf(stderr, "genprops error: shift value %d>=32 for %s %s\n", |
| (int)bin->binaries[i].vecShift, bin->ucdFile, bin->binaries[i].propName); |
| exit(U_INTERNAL_PROGRAM_ERROR); |
| } |
| uv=U_MASK(bin->binaries[i].vecShift); |
| |
| if(start==0 && end==0x10ffff) { |
| /* Also set bits for initialValue and errorValue. */ |
| end=UPVEC_MAX_CP; |
| } |
| upvec_setValue(pv, start, end, bin->binaries[i].vecWord, uv, uv, pErrorCode); |
| if(U_FAILURE(*pErrorCode)) { |
| fprintf(stderr, "genprops error: unable to set %s code: %s\n", |
| bin->binaries[i].propName, u_errorName(*pErrorCode)); |
| exit(*pErrorCode); |
| } |
| } |
| |
| static void |
| parseBinariesFile(char *filename, char *basename, const char *suffix, |
| const Binaries *bin, |
| UErrorCode *pErrorCode) { |
| char *fields[2][2]; |
| int32_t i; |
| |
| if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { |
| return; |
| } |
| |
| writeUCDFilename(basename, bin->ucdFile, suffix); |
| |
| ignoredPropsCount=0; |
| |
| u_parseDelimitedFile(filename, ';', fields, 2, binariesLineFn, (void *)bin, pErrorCode); |
| if(U_FAILURE(*pErrorCode)) { |
| fprintf(stderr, "error parsing %s.txt: %s\n", bin->ucdFile, u_errorName(*pErrorCode)); |
| } |
| |
| if(beVerbose) { |
| for(i=0; i<ignoredPropsCount; ++i) { |
| printf("genprops: ignoring property %s in %s.txt\n", ignoredProps[i], bin->ucdFile); |
| } |
| } |
| } |
| |
| /* -------------------------------------------------------------------------- */ |
| |
| U_CFUNC void |
| initAdditionalProperties() { |
| UErrorCode errorCode=U_ZERO_ERROR; |
| pv=upvec_open(UPROPS_VECTOR_WORDS, &errorCode); |
| if(U_FAILURE(errorCode)) { |
| fprintf(stderr, "error: upvec_open() failed - %s\n", u_errorName(errorCode)); |
| exit(errorCode); |
| } |
| } |
| |
| U_CFUNC void |
| exitAdditionalProperties() { |
| utrie_close(newTrie); |
| upvec_close(pv); |
| } |
| |
| U_CFUNC void |
| generateAdditionalProperties(char *filename, const char *suffix, UErrorCode *pErrorCode) { |
| char *basename; |
| |
| basename=filename+uprv_strlen(filename); |
| |
| /* process various UCD .txt files */ |
| |
| /* add Han numeric types & values */ |
| parseMultiFieldFile(filename, basename, "DerivedNumericValues", suffix, 2, numericLineFn, pErrorCode); |
| |
| parseTwoFieldFile(filename, basename, "DerivedAge", suffix, ageLineFn, pErrorCode); |
| |
| /* |
| * UTR 24 says: |
| * Section 2: |
| * "Common - For characters that may be used |
| * within multiple scripts, |
| * or any unassigned code points." |
| * |
| * Section 4: |
| * "The value COMMON is the default value, |
| * given to all code points that are not |
| * explicitly mentioned in the data file." |
| * |
| * COMMON==USCRIPT_COMMON==0 - nothing to do |
| */ |
| parseSingleEnumFile(filename, basename, suffix, &scriptSingleEnum, pErrorCode); |
| |
| parseSingleEnumFile(filename, basename, suffix, &blockSingleEnum, pErrorCode); |
| |
| parseBinariesFile(filename, basename, suffix, &propListBinaries, pErrorCode); |
| |
| parseBinariesFile(filename, basename, suffix, &derCorePropsBinaries, pErrorCode); |
| |
| parseSingleEnumFile(filename, basename, suffix, &graphemeClusterBreakSingleEnum, pErrorCode); |
| |
| parseSingleEnumFile(filename, basename, suffix, &wordBreakSingleEnum, pErrorCode); |
| |
| parseSingleEnumFile(filename, basename, suffix, &sentenceBreakSingleEnum, pErrorCode); |
| |
| /* |
| * LineBreak-4.0.0.txt: |
| * - All code points, assigned and unassigned, that are not listed |
| * explicitly are given the value "XX". |
| * |
| * XX==U_LB_UNKNOWN==0 - nothing to do |
| */ |
| parseSingleEnumFile(filename, basename, suffix, &lineBreakSingleEnum, pErrorCode); |
| |
| /* |
| * Preset East Asian Width defaults: |
| * |
| * http://www.unicode.org/reports/tr11/#Unassigned |
| * 7.1 Unassigned and Private Use characters |
| * |
| * All unassigned characters are by default classified as non-East Asian neutral, |
| * except for the range U+20000 to U+2FFFD, |
| * since all code positions from U+20000 to U+2FFFD are intended for CJK ideographs (W). |
| * All Private use characters are by default classified as ambiguous, |
| * since their definition depends on context. |
| * |
| * N for all ==0 - nothing to do |
| * A for Private Use |
| * W for plane 2 |
| */ |
| *pErrorCode=U_ZERO_ERROR; |
| upvec_setValue(pv, 0xe000, 0xf8ff, 0, (uint32_t)(U_EA_AMBIGUOUS<<UPROPS_EA_SHIFT), UPROPS_EA_MASK, pErrorCode); |
| upvec_setValue(pv, 0xf0000, 0xffffd, 0, (uint32_t)(U_EA_AMBIGUOUS<<UPROPS_EA_SHIFT), UPROPS_EA_MASK, pErrorCode); |
| upvec_setValue(pv, 0x100000, 0x10fffd, 0, (uint32_t)(U_EA_AMBIGUOUS<<UPROPS_EA_SHIFT), UPROPS_EA_MASK, pErrorCode); |
| upvec_setValue(pv, 0x20000, 0x2fffd, 0, (uint32_t)(U_EA_WIDE<<UPROPS_EA_SHIFT), UPROPS_EA_MASK, pErrorCode); |
| if(U_FAILURE(*pErrorCode)) { |
| fprintf(stderr, "genprops: unable to set default East Asian Widths: %s\n", u_errorName(*pErrorCode)); |
| exit(*pErrorCode); |
| } |
| |
| /* parse EastAsianWidth.txt */ |
| parseSingleEnumFile(filename, basename, suffix, &eawSingleEnum, pErrorCode); |
| |
| { |
| UPVecToUTrieContext toUTrie={ NULL, 50000 /* capacity */, 0, TRUE /* latin1Linear */ }; |
| upvec_compact(pv, upvec_compactToUTrieHandler, &toUTrie, pErrorCode); |
| if(U_FAILURE(*pErrorCode)) { |
| fprintf(stderr, "genprops error: unable to build trie for additional properties: %s\n", |
| u_errorName(*pErrorCode)); |
| exit(*pErrorCode); |
| } |
| newTrie=toUTrie.newTrie; |
| } |
| } |
| |
| /* DerivedAge.txt ----------------------------------------------------------- */ |
| |
| static void U_CALLCONV |
| ageLineFn(void *context, |
| char *fields[][2], int32_t fieldCount, |
| UErrorCode *pErrorCode) { |
| char *s, *numberLimit; |
| uint32_t value, start, end, version; |
| |
| u_parseCodePointRange(fields[0][0], &start, &end, pErrorCode); |
| if(U_FAILURE(*pErrorCode)) { |
| fprintf(stderr, "genprops: syntax error in DerivedAge.txt field 0 at %s\n", fields[0][0]); |
| exit(*pErrorCode); |
| } |
| |
| /* ignore "unassigned" (the default is already set to 0.0) */ |
| s=(char *)u_skipWhitespace(fields[1][0]); |
| if(0==uprv_strncmp(s, "unassigned", 10)) { |
| return; |
| } |
| |
| /* parse version number */ |
| value=(uint32_t)uprv_strtoul(s, &numberLimit, 10); |
| if(s==numberLimit || value==0 || value>15 || (*numberLimit!='.' && *numberLimit!=' ' && *numberLimit!='\t' && *numberLimit!=0)) { |
| fprintf(stderr, "genprops: syntax error in DerivedAge.txt field 1 at %s\n", fields[1][0]); |
| *pErrorCode=U_PARSE_ERROR; |
| exit(U_PARSE_ERROR); |
| } |
| version=value<<4; |
| |
| /* parse minor version number */ |
| if(*numberLimit=='.') { |
| s=(char *)u_skipWhitespace(numberLimit+1); |
| value=(uint32_t)uprv_strtoul(s, &numberLimit, 10); |
| if(s==numberLimit || value>15 || (*numberLimit!=' ' && *numberLimit!='\t' && *numberLimit!=0)) { |
| fprintf(stderr, "genprops: syntax error in DerivedAge.txt field 1 at %s\n", fields[1][0]); |
| *pErrorCode=U_PARSE_ERROR; |
| exit(U_PARSE_ERROR); |
| } |
| version|=value; |
| } |
| |
| if(start==0 && end==0x10ffff) { |
| /* Also set bits for initialValue and errorValue. */ |
| end=UPVEC_MAX_CP; |
| } |
| upvec_setValue(pv, start, end, 0, version<<UPROPS_AGE_SHIFT, UPROPS_AGE_MASK, pErrorCode); |
| if(U_FAILURE(*pErrorCode)) { |
| fprintf(stderr, "genprops error: unable to set character age: %s\n", u_errorName(*pErrorCode)); |
| exit(*pErrorCode); |
| } |
| } |
| |
| /* DerivedNumericValues.txt ------------------------------------------------- */ |
| |
| static void U_CALLCONV |
| numericLineFn(void *context, |
| char *fields[][2], int32_t fieldCount, |
| UErrorCode *pErrorCode) { |
| Props newProps={ 0 }; |
| char *s, *numberLimit; |
| uint32_t start, end, value, oldProps32; |
| char c; |
| UBool isFraction; |
| |
| /* get the code point range */ |
| u_parseCodePointRange(fields[0][0], &start, &end, pErrorCode); |
| if(U_FAILURE(*pErrorCode)) { |
| fprintf(stderr, "genprops: syntax error in DerivedNumericValues.txt field 0 at %s\n", fields[0][0]); |
| exit(*pErrorCode); |
| } |
| |
| /* |
| * Ignore the |
| * # @missing: 0000..10FFFF; NaN |
| * line from Unicode 5.1's DerivedNumericValues.txt: |
| * The following code cannot parse "NaN", and we don't want to overwrite |
| * the numeric values for all characters after reading most |
| * from UnicodeData.txt already. |
| */ |
| if(start==0 && end==0x10ffff) { |
| return; |
| } |
| |
| /* check if the numeric value is a fraction (this code does not handle any) */ |
| isFraction=FALSE; |
| s=uprv_strchr(fields[1][0], '.'); |
| if(s!=NULL) { |
| numberLimit=s+1; |
| while('0'<=(c=*numberLimit++) && c<='9') { |
| if(c!='0') { |
| isFraction=TRUE; |
| break; |
| } |
| } |
| } |
| |
| if(isFraction) { |
| value=0; |
| } else { |
| /* parse numeric value */ |
| s=(char *)u_skipWhitespace(fields[1][0]); |
| |
| /* try large, single-significant-digit numbers, may otherwise overflow strtoul() */ |
| if('1'<=s[0] && s[0]<='9' && s[1]=='0' && s[2]=='0') { |
| /* large integers are encoded in a special way, see store.c */ |
| uint8_t exp=0; |
| |
| value=s[0]-'0'; |
| numberLimit=s; |
| while(*(++numberLimit)=='0') { |
| ++exp; |
| } |
| newProps.exponent=exp; |
| } else { |
| /* normal number parsing */ |
| value=(uint32_t)uprv_strtoul(s, &numberLimit, 10); |
| } |
| if(numberLimit<=s || (*numberLimit!='.' && u_skipWhitespace(numberLimit)!=fields[1][1]) || value>=0x80000000) { |
| fprintf(stderr, "genprops: syntax error in DerivedNumericValues.txt field 1 at %s\n", fields[0][0]); |
| exit(U_PARSE_ERROR); |
| } |
| } |
| |
| /* |
| * Unicode 4.0.1 removes the third column that used to list the numeric type. |
| * Assume that either the data is the same as in UnicodeData.txt, |
| * or else that the numeric type is "numeric". |
| * This should work because we only expect to add numeric values for |
| * Han characters; for those, UnicodeData.txt lists only ranges without |
| * specific properties for single characters. |
| */ |
| |
| /* set the new numeric value */ |
| newProps.code=start; |
| newProps.numericValue=(int32_t)value; /* newly parsed numeric value */ |
| /* the exponent may have been set above */ |
| |
| for(; start<=end; ++start) { |
| uint32_t newProps32; |
| int32_t oldNtv; |
| oldProps32=getProps(start); |
| oldNtv=(int32_t)GET_NUMERIC_TYPE_VALUE(oldProps32); |
| |
| if(isFraction) { |
| if(UPROPS_NTV_FRACTION_START<=oldNtv && oldNtv<UPROPS_NTV_LARGE_START) { |
| /* this code point was already listed with its numeric value in UnicodeData.txt */ |
| continue; |
| } else { |
| fprintf(stderr, "genprops: not prepared for new fractions in DerivedNumericValues.txt field 1 at %s\n", fields[1][0]); |
| exit(U_PARSE_ERROR); |
| } |
| } |
| |
| /* |
| * For simplicity, and because we only expect to set numeric values for Han characters, |
| * for now we only allow to set these values for Lo characters. |
| */ |
| if(oldNtv==UPROPS_NTV_NONE && GET_CATEGORY(oldProps32)!=U_OTHER_LETTER) { |
| fprintf(stderr, "genprops error: new numeric value for a character other than Lo in DerivedNumericValues.txt at %s\n", fields[0][0]); |
| exit(U_PARSE_ERROR); |
| } |
| |
| /* verify that we do not change an existing value (fractions were excluded above) */ |
| if(oldNtv!=UPROPS_NTV_NONE) { |
| /* the code point already has a value stored */ |
| newProps.numericType=UPROPS_NTV_GET_TYPE(oldNtv); |
| newProps32=makeProps(&newProps); |
| if(oldNtv!=GET_NUMERIC_TYPE_VALUE(newProps32)) { |
| fprintf(stderr, "genprops error: new numeric value differs from old one for U+%04lx\n", (long)start); |
| exit(U_PARSE_ERROR); |
| } |
| /* same value, continue */ |
| } else { |
| /* the code point is getting a new numeric value */ |
| newProps.numericType=(uint8_t)U_NT_NUMERIC; /* assumed numeric type, see Unicode 4.0.1 comment */ |
| newProps32=makeProps(&newProps); |
| if(beVerbose) { |
| printf("adding U+%04x numeric type %d encoded-numeric-type-value 0x%03x from %s\n", |
| (int)start, U_NT_NUMERIC, (int)GET_NUMERIC_TYPE_VALUE(newProps32), fields[0][0]); |
| } |
| |
| addProps(start, newProps32|GET_CATEGORY(oldProps32)); |
| } |
| } |
| } |
| |
| /* data serialization ------------------------------------------------------- */ |
| |
| U_CFUNC int32_t |
| writeAdditionalData(FILE *f, uint8_t *p, int32_t capacity, int32_t indexes[UPROPS_INDEX_COUNT]) { |
| const uint32_t *pvArray; |
| int32_t pvRows, pvCount; |
| int32_t length; |
| UErrorCode errorCode; |
| |
| pvArray=upvec_getArray(pv, &pvRows, NULL); |
| pvCount=pvRows*UPROPS_VECTOR_WORDS; |
| |
| errorCode=U_ZERO_ERROR; |
| length=utrie_serialize(newTrie, p, capacity, NULL, TRUE, &errorCode); |
| if(U_FAILURE(errorCode)) { |
| fprintf(stderr, "genprops error: unable to serialize trie for additional properties: %s\n", u_errorName(errorCode)); |
| exit(errorCode); |
| } |
| if(p!=NULL) { |
| if(beVerbose) { |
| printf("size in bytes of additional props trie:%5u\n", (int)length); |
| } |
| if(f!=NULL) { |
| UTrie trie={ NULL }; |
| UTrie2 *trie2; |
| |
| utrie_unserialize(&trie, p, length, &errorCode); |
| if(U_FAILURE(errorCode)) { |
| fprintf( |
| stderr, |
| "genprops error: failed to utrie_unserialize(trie for additional properties) - %s\n", |
| u_errorName(errorCode)); |
| exit(errorCode); |
| } |
| |
| /* use UTrie2 */ |
| trie2=utrie2_fromUTrie(&trie, trie.initialValue, &errorCode); |
| if(U_FAILURE(errorCode)) { |
| fprintf( |
| stderr, |
| "genprops error: utrie2_fromUTrie() failed - %s\n", |
| u_errorName(errorCode)); |
| exit(errorCode); |
| } |
| { |
| /* delete lead surrogate code unit values */ |
| UChar lead; |
| trie2=utrie2_cloneAsThawed(trie2, &errorCode); |
| for(lead=0xd800; lead<0xdc00; ++lead) { |
| utrie2_set32ForLeadSurrogateCodeUnit(trie2, lead, trie2->initialValue, &errorCode); |
| } |
| utrie2_freeze(trie2, UTRIE2_16_VALUE_BITS, &errorCode); |
| if(U_FAILURE(errorCode)) { |
| fprintf( |
| stderr, |
| "genbidi error: deleting lead surrogate code unit values failed - %s\n", |
| u_errorName(errorCode)); |
| exit(errorCode); |
| } |
| } |
| |
| usrc_writeUTrie2Arrays(f, |
| "static const uint16_t propsVectorsTrie_index[%ld]={\n", NULL, |
| trie2, |
| "\n};\n\n"); |
| usrc_writeUTrie2Struct(f, |
| "static const UTrie2 propsVectorsTrie={\n", |
| trie2, "propsVectorsTrie_index", NULL, |
| "};\n\n"); |
| |
| utrie2_close(trie2); |
| } |
| |
| p+=length; |
| capacity-=length; |
| |
| /* set indexes */ |
| indexes[UPROPS_ADDITIONAL_VECTORS_INDEX]= |
| indexes[UPROPS_ADDITIONAL_TRIE_INDEX]+length/4; |
| indexes[UPROPS_ADDITIONAL_VECTORS_COLUMNS_INDEX]=UPROPS_VECTOR_WORDS; |
| indexes[UPROPS_RESERVED_INDEX]= |
| indexes[UPROPS_ADDITIONAL_VECTORS_INDEX]+pvCount; |
| |
| indexes[UPROPS_MAX_VALUES_INDEX]= |
| (((int32_t)U_EA_COUNT-1)<<UPROPS_EA_SHIFT)| |
| (((int32_t)UBLOCK_COUNT-1)<<UPROPS_BLOCK_SHIFT)| |
| (((int32_t)USCRIPT_CODE_LIMIT-1)&UPROPS_SCRIPT_MASK); |
| indexes[UPROPS_MAX_VALUES_2_INDEX]= |
| (((int32_t)U_LB_COUNT-1)<<UPROPS_LB_SHIFT)| |
| (((int32_t)U_SB_COUNT-1)<<UPROPS_SB_SHIFT)| |
| (((int32_t)U_WB_COUNT-1)<<UPROPS_WB_SHIFT)| |
| (((int32_t)U_GCB_COUNT-1)<<UPROPS_GCB_SHIFT)| |
| ((int32_t)U_DT_COUNT-1); |
| } |
| |
| if(p!=NULL && (pvCount*4)<=capacity) { |
| if(f!=NULL) { |
| usrc_writeArray(f, |
| "static const uint32_t propsVectors[%ld]={\n", |
| pvArray, 32, pvCount, |
| "};\n\n"); |
| fprintf(f, "static const int32_t countPropsVectors=%ld;\n", (long)pvCount); |
| fprintf(f, "static const int32_t propsVectorsColumns=%ld;\n", (long)indexes[UPROPS_ADDITIONAL_VECTORS_COLUMNS_INDEX]); |
| } else { |
| uprv_memcpy(p, pvArray, pvCount*4); |
| } |
| if(beVerbose) { |
| printf("number of additional props vectors: %5u\n", (int)pvRows); |
| printf("number of 32-bit words per vector: %5u\n", UPROPS_VECTOR_WORDS); |
| } |
| } |
| length+=pvCount*4; |
| |
| return length; |
| } |