| /* |
| ******************************************************************************* |
| * |
| * Copyright (C) 1999-2008, International Business Machines |
| * Corporation and others. All Rights Reserved. |
| * |
| ******************************************************************************* |
| * file name: genprops.c |
| * encoding: US-ASCII |
| * tab size: 8 (not used) |
| * indentation:4 |
| * |
| * created on: 1999dec08 |
| * created by: Markus W. Scherer |
| * |
| * This program reads several of the Unicode character database text files, |
| * parses them, and extracts most of the properties for each character. |
| * It then writes a binary file containing the properties |
| * that is designed to be used directly for random-access to |
| * the properties of each Unicode character. |
| */ |
| |
| #include <stdio.h> |
| #include <stdlib.h> |
| #include "unicode/utypes.h" |
| #include "unicode/uchar.h" |
| #include "unicode/putil.h" |
| #include "unicode/uclean.h" |
| #include "cmemory.h" |
| #include "cstring.h" |
| #include "unewdata.h" |
| #include "uoptions.h" |
| #include "uparse.h" |
| #include "uprops.h" |
| #include "propsvec.h" |
| |
| U_CDECL_BEGIN |
| #include "genprops.h" |
| U_CDECL_END |
| |
| #define LENGTHOF(array) (sizeof(array)/sizeof((array)[0])) |
| |
| UBool beVerbose=FALSE, haveCopyright=TRUE; |
| |
| /* prototypes --------------------------------------------------------------- */ |
| |
| static void |
| parseDB(const char *filename, UErrorCode *pErrorCode); |
| |
| /* -------------------------------------------------------------------------- */ |
| |
| enum |
| { |
| HELP_H, |
| HELP_QUESTION_MARK, |
| VERBOSE, |
| COPYRIGHT, |
| DESTDIR, |
| SOURCEDIR, |
| UNICODE_VERSION, |
| ICUDATADIR, |
| CSOURCE |
| }; |
| |
| /* Keep these values in sync with the above enums */ |
| static UOption options[]={ |
| UOPTION_HELP_H, |
| UOPTION_HELP_QUESTION_MARK, |
| UOPTION_VERBOSE, |
| UOPTION_COPYRIGHT, |
| UOPTION_DESTDIR, |
| UOPTION_SOURCEDIR, |
| UOPTION_DEF("unicode", 'u', UOPT_REQUIRES_ARG), |
| UOPTION_ICUDATADIR, |
| UOPTION_DEF("csource", 'C', UOPT_NO_ARG) |
| }; |
| |
| extern int |
| main(int argc, char* argv[]) { |
| char filename[300]; |
| const char *srcDir=NULL, *destDir=NULL, *suffix=NULL; |
| char *basename=NULL; |
| UErrorCode errorCode=U_ZERO_ERROR; |
| |
| U_MAIN_INIT_ARGS(argc, argv); |
| |
| /* preset then read command line options */ |
| options[DESTDIR].value=u_getDataDirectory(); |
| options[SOURCEDIR].value=""; |
| options[UNICODE_VERSION].value=""; |
| options[ICUDATADIR].value=u_getDataDirectory(); |
| argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options); |
| |
| /* error handling, printing usage message */ |
| if(argc<0) { |
| fprintf(stderr, |
| "error in command line argument \"%s\"\n", |
| argv[-argc]); |
| } |
| if(argc<0 || options[HELP_H].doesOccur || options[HELP_QUESTION_MARK].doesOccur) { |
| /* |
| * Broken into chucks because the C89 standard says the minimum |
| * required supported string length is 509 bytes. |
| */ |
| fprintf(stderr, |
| "Usage: %s [-options] [suffix]\n" |
| "\n" |
| "read the UnicodeData.txt file and other Unicode properties files and\n" |
| "create a binary file " DATA_NAME "." DATA_TYPE " with the character properties\n" |
| "\n", |
| argv[0]); |
| fprintf(stderr, |
| "Options:\n" |
| "\t-h or -? or --help this usage text\n" |
| "\t-v or --verbose verbose output\n" |
| "\t-c or --copyright include a copyright notice\n" |
| "\t-u or --unicode Unicode version, followed by the version like 3.0.0\n" |
| "\t-C or --csource generate a .c source file rather than the .icu binary\n"); |
| fprintf(stderr, |
| "\t-d or --destdir destination directory, followed by the path\n" |
| "\t-s or --sourcedir source directory, followed by the path\n" |
| "\t-i or --icudatadir directory for locating any needed intermediate data files,\n" |
| "\t followed by path, defaults to %s\n" |
| "\tsuffix suffix that is to be appended with a '-'\n" |
| "\t to the source file basenames before opening;\n" |
| "\t 'genprops new' will read UnicodeData-new.txt etc.\n", |
| u_getDataDirectory()); |
| return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR; |
| } |
| |
| /* get the options values */ |
| beVerbose=options[VERBOSE].doesOccur; |
| haveCopyright=options[COPYRIGHT].doesOccur; |
| srcDir=options[SOURCEDIR].value; |
| destDir=options[DESTDIR].value; |
| |
| if(argc>=2) { |
| suffix=argv[1]; |
| } else { |
| suffix=NULL; |
| } |
| |
| if(options[UNICODE_VERSION].doesOccur) { |
| setUnicodeVersion(options[UNICODE_VERSION].value); |
| } |
| /* else use the default dataVersion in store.c */ |
| |
| if (options[ICUDATADIR].doesOccur) { |
| u_setDataDirectory(options[ICUDATADIR].value); |
| } |
| |
| /* prepare the filename beginning with the source dir */ |
| uprv_strcpy(filename, srcDir); |
| basename=filename+uprv_strlen(filename); |
| if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) { |
| *basename++=U_FILE_SEP_CHAR; |
| } |
| |
| /* initialize */ |
| initStore(); |
| |
| /* process UnicodeData.txt */ |
| writeUCDFilename(basename, "UnicodeData", suffix); |
| parseDB(filename, &errorCode); |
| |
| /* process additional properties files */ |
| *basename=0; |
| generateAdditionalProperties(filename, suffix, &errorCode); |
| |
| /* process parsed data */ |
| if(U_SUCCESS(errorCode)) { |
| /* write the properties data file */ |
| generateData(destDir, options[CSOURCE].doesOccur); |
| } |
| |
| exitStore(); |
| u_cleanup(); |
| return errorCode; |
| } |
| |
| U_CFUNC void |
| writeUCDFilename(char *basename, const char *filename, const char *suffix) { |
| int32_t length=(int32_t)uprv_strlen(filename); |
| uprv_strcpy(basename, filename); |
| if(suffix!=NULL) { |
| basename[length++]='-'; |
| uprv_strcpy(basename+length, suffix); |
| length+=(int32_t)uprv_strlen(suffix); |
| } |
| uprv_strcpy(basename+length, ".txt"); |
| } |
| |
| U_CFUNC UBool |
| isToken(const char *token, const char *s) { |
| const char *z; |
| int32_t j; |
| |
| s=u_skipWhitespace(s); |
| for(j=0;; ++j) { |
| if(token[j]!=0) { |
| if(s[j]!=token[j]) { |
| break; |
| } |
| } else { |
| z=u_skipWhitespace(s+j); |
| if(*z==';' || *z==0) { |
| return TRUE; |
| } else { |
| break; |
| } |
| } |
| } |
| |
| return FALSE; |
| } |
| |
| U_CFUNC int32_t |
| getTokenIndex(const char *const tokens[], int32_t countTokens, const char *s) { |
| const char *t, *z; |
| int32_t i, j; |
| |
| s=u_skipWhitespace(s); |
| for(i=0; i<countTokens; ++i) { |
| t=tokens[i]; |
| if(t!=NULL) { |
| for(j=0;; ++j) { |
| if(t[j]!=0) { |
| if(s[j]!=t[j]) { |
| break; |
| } |
| } else { |
| z=u_skipWhitespace(s+j); |
| if(*z==';' || *z==0 || *z=='#' || *z=='\r' || *z=='\n') { |
| return i; |
| } else { |
| break; |
| } |
| } |
| } |
| } |
| } |
| return -1; |
| } |
| |
| /* parser for UnicodeData.txt ----------------------------------------------- */ |
| |
| /* general categories */ |
| const char *const |
| genCategoryNames[U_CHAR_CATEGORY_COUNT]={ |
| "Cn", |
| "Lu", "Ll", "Lt", "Lm", "Lo", "Mn", "Me", |
| "Mc", "Nd", "Nl", "No", |
| "Zs", "Zl", "Zp", |
| "Cc", "Cf", "Co", "Cs", |
| "Pd", "Ps", "Pe", "Pc", "Po", |
| "Sm", "Sc", "Sk", "So", |
| "Pi", "Pf" |
| }; |
| |
| const char *const |
| decompositionTypeNames[U_DT_COUNT]={ |
| NULL, |
| NULL, |
| "compat", |
| "circle", |
| "final", |
| "font", |
| "fraction", |
| "initial", |
| "isolated", |
| "medial", |
| "narrow", |
| "noBreak", |
| "small", |
| "square", |
| "sub", |
| "super", |
| "vertical", |
| "wide" |
| }; |
| |
| static struct { |
| uint32_t first, last, props; |
| char name[80]; |
| } unicodeAreas[32]; |
| |
| static int32_t unicodeAreaIndex=0; |
| |
| static void U_CALLCONV |
| unicodeDataLineFn(void *context, |
| char *fields[][2], int32_t fieldCount, |
| UErrorCode *pErrorCode) { |
| Props p; |
| char *end; |
| static uint32_t prevCode=0; |
| uint32_t value; |
| int32_t i; |
| |
| /* reset the properties */ |
| uprv_memset(&p, 0, sizeof(Props)); |
| |
| /* get the character code, field 0 */ |
| p.code=(uint32_t)uprv_strtoul(fields[0][0], &end, 16); |
| if(end<=fields[0][0] || end!=fields[0][1]) { |
| fprintf(stderr, "genprops: syntax error in field 0 at %s\n", fields[0][0]); |
| *pErrorCode=U_PARSE_ERROR; |
| exit(U_PARSE_ERROR); |
| } |
| |
| /* get general category, field 2 */ |
| i=getTokenIndex(genCategoryNames, U_CHAR_CATEGORY_COUNT, fields[2][0]); |
| if(i>=0) { |
| p.generalCategory=(uint8_t)i; |
| } else { |
| fprintf(stderr, "genprops: unknown general category \"%s\" at code 0x%lx\n", |
| fields[2][0], (unsigned long)p.code); |
| *pErrorCode=U_PARSE_ERROR; |
| exit(U_PARSE_ERROR); |
| } |
| |
| /* get decomposition type, field 5 */ |
| if(fields[5][0]<fields[5][1]) { |
| /* there is some decomposition */ |
| if(*fields[5][0]!='<') { |
| /* canonical */ |
| i=U_DT_CANONICAL; |
| } else { |
| /* get compatibility type */ |
| end=fields[5][0]+1; |
| while(end<fields[5][1] && *end!='>') { |
| ++end; |
| } |
| *end='#'; |
| i=getTokenIndex(decompositionTypeNames, U_DT_COUNT, fields[5][0]+1); |
| if(i<0) { |
| fprintf(stderr, "genprops: unknown decomposition type \"%s\" at code 0x%lx\n", |
| fields[5][0], (unsigned long)p.code); |
| *pErrorCode=U_PARSE_ERROR; |
| exit(U_PARSE_ERROR); |
| } |
| } |
| upvec_setValue(pv, p.code, p.code, 2, (uint32_t)i, UPROPS_DT_MASK, pErrorCode); |
| if(U_FAILURE(*pErrorCode)) { |
| fprintf(stderr, "genprops error: unable to set decomposition type: %s\n", u_errorName(*pErrorCode)); |
| exit(*pErrorCode); |
| } |
| } |
| |
| /* decimal digit value, field 6 */ |
| if(fields[6][0]<fields[6][1]) { |
| value=(uint32_t)uprv_strtoul(fields[6][0], &end, 10); |
| if(end!=fields[6][1] || value>0x7fff) { |
| fprintf(stderr, "genprops: syntax error in field 6 at code 0x%lx\n", |
| (unsigned long)p.code); |
| *pErrorCode=U_PARSE_ERROR; |
| exit(U_PARSE_ERROR); |
| } |
| p.numericValue=(int32_t)value; |
| p.numericType=1; |
| } |
| |
| /* digit value, field 7 */ |
| if(fields[7][0]<fields[7][1]) { |
| value=(uint32_t)uprv_strtoul(fields[7][0], &end, 10); |
| if(end!=fields[7][1] || value>0x7fff) { |
| fprintf(stderr, "genprops: syntax error in field 7 at code 0x%lx\n", |
| (unsigned long)p.code); |
| *pErrorCode=U_PARSE_ERROR; |
| exit(U_PARSE_ERROR); |
| } |
| if(p.numericType==0) { |
| p.numericValue=(int32_t)value; |
| p.numericType=2; |
| } else if((int32_t)value!=p.numericValue) { |
| fprintf(stderr, "genprops error: numeric values in fields 6 & 7 different at code 0x%lx\n", |
| (unsigned long)p.code); |
| *pErrorCode=U_PARSE_ERROR; |
| exit(U_PARSE_ERROR); |
| } |
| } |
| |
| /* numeric value, field 8 */ |
| if(fields[8][0]<fields[8][1]) { |
| char *s=fields[8][0]; |
| UBool isNegative; |
| |
| /* get a possible minus sign */ |
| if(*s=='-') { |
| isNegative=TRUE; |
| ++s; |
| } else { |
| isNegative=FALSE; |
| } |
| |
| value=(uint32_t)uprv_strtoul(s, &end, 10); |
| if(value>0 && *end=='/') { |
| /* field 8 may contain a fractional value, get the denominator */ |
| if(p.numericType>0) { |
| fprintf(stderr, "genprops error: numeric values in fields 6..8 different at code 0x%lx\n", |
| (unsigned long)p.code); |
| *pErrorCode=U_PARSE_ERROR; |
| exit(U_PARSE_ERROR); |
| } |
| |
| p.denominator=(uint32_t)uprv_strtoul(end+1, &end, 10); |
| if(p.denominator==0) { |
| fprintf(stderr, "genprops: denominator is 0 in field 8 at code 0x%lx\n", |
| (unsigned long)p.code); |
| *pErrorCode=U_PARSE_ERROR; |
| exit(U_PARSE_ERROR); |
| } |
| } |
| if(end!=fields[8][1] || value>0x7fffffff) { |
| fprintf(stderr, "genprops: syntax error in field 8 at code 0x%lx\n", |
| (unsigned long)p.code); |
| *pErrorCode=U_PARSE_ERROR; |
| exit(U_PARSE_ERROR); |
| } |
| |
| if(p.numericType==0) { |
| if(isNegative) { |
| p.numericValue=-(int32_t)value; |
| } else { |
| p.numericValue=(int32_t)value; |
| } |
| p.numericType=3; |
| } else if((int32_t)value!=p.numericValue) { |
| fprintf(stderr, "genprops error: numeric values in fields 6..8 different at code 0x%lx\n", |
| (unsigned long)p.code); |
| *pErrorCode=U_PARSE_ERROR; |
| exit(U_PARSE_ERROR); |
| } |
| } |
| |
| value=makeProps(&p); |
| |
| if(*fields[1][0]=='<') { |
| /* first or last entry of a Unicode area */ |
| size_t length=fields[1][1]-fields[1][0]; |
| |
| if(length<9) { |
| /* name too short for an area name */ |
| } else if(0==uprv_memcmp(", First>", fields[1][1]-8, 8)) { |
| /* set the current area */ |
| if(unicodeAreas[unicodeAreaIndex].first==0xffffffff) { |
| length-=9; |
| unicodeAreas[unicodeAreaIndex].first=p.code; |
| unicodeAreas[unicodeAreaIndex].props=value; |
| uprv_memcpy(unicodeAreas[unicodeAreaIndex].name, fields[1][0]+1, length); |
| unicodeAreas[unicodeAreaIndex].name[length]=0; |
| } else { |
| /* error: a previous area is incomplete */ |
| fprintf(stderr, "genprops: error - area \"%s\" is incomplete\n", unicodeAreas[unicodeAreaIndex].name); |
| *pErrorCode=U_PARSE_ERROR; |
| exit(U_PARSE_ERROR); |
| } |
| return; |
| } else if(0==uprv_memcmp(", Last>", fields[1][1]-7, 7)) { |
| /* check that the current area matches, and complete it with the last code point */ |
| length-=8; |
| if( unicodeAreas[unicodeAreaIndex].props==value && |
| 0==uprv_memcmp(unicodeAreas[unicodeAreaIndex].name, fields[1][0]+1, length) && |
| unicodeAreas[unicodeAreaIndex].name[length]==0 && |
| unicodeAreas[unicodeAreaIndex].first<p.code |
| ) { |
| unicodeAreas[unicodeAreaIndex].last=p.code; |
| if(beVerbose) { |
| printf("Unicode area U+%04lx..U+%04lx \"%s\"\n", |
| (unsigned long)unicodeAreas[unicodeAreaIndex].first, |
| (unsigned long)unicodeAreas[unicodeAreaIndex].last, |
| unicodeAreas[unicodeAreaIndex].name); |
| } |
| unicodeAreas[++unicodeAreaIndex].first=0xffffffff; |
| } else { |
| /* error: different properties between first & last, different area name, first>=last */ |
| fprintf(stderr, "genprops: error - Last of area \"%s\" is incorrect\n", unicodeAreas[unicodeAreaIndex].name); |
| *pErrorCode=U_PARSE_ERROR; |
| exit(U_PARSE_ERROR); |
| } |
| return; |
| } else { |
| /* not an area name */ |
| } |
| } |
| |
| /* check for non-character code points */ |
| if((p.code&0xfffe)==0xfffe || (uint32_t)(p.code-0xfdd0)<0x20) { |
| fprintf(stderr, "genprops: error - properties for non-character code point U+%04lx\n", |
| (unsigned long)p.code); |
| *pErrorCode=U_PARSE_ERROR; |
| exit(U_PARSE_ERROR); |
| } |
| |
| /* check that the code points (p.code) are in ascending order */ |
| if(p.code<=prevCode && p.code>0) { |
| fprintf(stderr, "genprops: error - UnicodeData entries out of order, U+%04lx after U+%04lx\n", |
| (unsigned long)p.code, (unsigned long)prevCode); |
| *pErrorCode=U_PARSE_ERROR; |
| exit(U_PARSE_ERROR); |
| } |
| prevCode=p.code; |
| |
| /* properties for a single code point */ |
| addProps(p.code, value); |
| } |
| |
| /* set repeated properties for the areas */ |
| static void |
| repeatAreaProps() { |
| uint32_t puaProps; |
| int32_t i; |
| UBool hasPlane15PUA, hasPlane16PUA; |
| UErrorCode errorCode; |
| |
| /* |
| * UnicodeData.txt before 3.0.1 did not contain the PUAs on |
| * planes 15 and 16. |
| * If that is the case, then we add them here, using the properties |
| * from the BMP PUA. |
| */ |
| puaProps=0; |
| hasPlane15PUA=hasPlane16PUA=FALSE; |
| |
| for(i=0; i<unicodeAreaIndex; ++i) { |
| repeatProps(unicodeAreas[i].first, |
| unicodeAreas[i].last, |
| unicodeAreas[i].props); |
| if(unicodeAreas[i].first==0xe000) { |
| puaProps=unicodeAreas[i].props; |
| } else if(unicodeAreas[i].first==0xf0000) { |
| hasPlane15PUA=TRUE; |
| } else if(unicodeAreas[i].first==0x100000) { |
| hasPlane16PUA=TRUE; |
| } |
| } |
| |
| if(puaProps!=0) { |
| if(!hasPlane15PUA) { |
| repeatProps(0xf0000, 0xffffd, puaProps); |
| } |
| if(!hasPlane16PUA) { |
| repeatProps(0x100000, 0x10fffd, puaProps); |
| } |
| } |
| |
| /* Hangul have canonical decompositions */ |
| errorCode=U_ZERO_ERROR; |
| upvec_setValue(pv, 0xac00, 0xd7a3, 2, (uint32_t)U_DT_CANONICAL, UPROPS_DT_MASK, &errorCode); |
| if(U_FAILURE(errorCode)) { |
| fprintf(stderr, "genprops error: unable to set decomposition type: %s\n", u_errorName(errorCode)); |
| exit(errorCode); |
| } |
| } |
| |
| static void |
| parseDB(const char *filename, UErrorCode *pErrorCode) { |
| char *fields[15][2]; |
| |
| if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { |
| return; |
| } |
| |
| /* while unicodeAreas[unicodeAreaIndex] is unused, set its first to a bogus value */ |
| unicodeAreas[0].first=0xffffffff; |
| |
| u_parseDelimitedFile(filename, ';', fields, 15, unicodeDataLineFn, NULL, pErrorCode); |
| |
| if(unicodeAreas[unicodeAreaIndex].first!=0xffffffff) { |
| fprintf(stderr, "genprops: error - the last area \"%s\" from U+%04lx is incomplete\n", |
| unicodeAreas[unicodeAreaIndex].name, |
| (unsigned long)unicodeAreas[unicodeAreaIndex].first); |
| *pErrorCode=U_PARSE_ERROR; |
| exit(U_PARSE_ERROR); |
| } |
| |
| repeatAreaProps(); |
| |
| if(U_FAILURE(*pErrorCode)) { |
| return; |
| } |
| } |
| |
| /* |
| * Hey, Emacs, please set the following: |
| * |
| * Local Variables: |
| * indent-tabs-mode: nil |
| * End: |
| * |
| */ |
