| /******************************************************************** |
| * COPYRIGHT: |
| * Copyright (C) 2001 IBM, Inc. All Rights Reserved. |
| * |
| ********************************************************************/ |
| /******************************************************************************** |
| * |
| * File ubrkperf.cpp |
| * |
| * Modification History: |
| * Name Description |
| * Vladimir Weinstein First Version, based on collperf |
| * |
| ********************************************************************************* |
| */ |
| |
| // |
| // This program tests break iterator performance |
| // Currently we test only ICU APIs with the future possibility of testing *nix & win32 APIs |
| // (if any) |
| // A text file is required as input. It must be in utf-8 or utf-16 format, |
| // and include a byte order mark. Either LE or BE format is OK. |
| // |
| |
| const char gUsageString[] = |
| "usage: ubrkperf options...\n" |
| "-help Display this message.\n" |
| "-file file_name utf-16/utf-8 format file.\n" |
| "-locale name ICU locale to use. Default is en_US\n" |
| "-langid 0x1234 Windows Language ID number. Default to value for -locale option\n" |
| " see http://msdn.microsoft.com/library/psdk/winbase/nls_8xo3.htm\n" |
| "-win Run test using Windows native services. (currently not working) (ICU is default)\n" |
| "-unix Run test using Unix word breaking services. (currently not working) \n" |
| "-mac Run test using MacOSX word breaking services.\n" |
| "-uselen Use API with string lengths. Default is null-terminated strings\n" |
| "-char Use character break iterator\n" |
| "-word Use word break iterator\n" |
| "-line Use line break iterator\n" |
| "-sentence Use sentence break iterator\n" |
| "-loop nnnn Loopcount for test. Adjust for reasonable total running time.\n" |
| "-iloop n Inner Loop Count. Default = 1. Number of calls to function\n" |
| " under test at each call point. For measuring test overhead.\n" |
| "-terse Terse numbers-only output. Intended for use by scripts.\n" |
| "-dump Display stuff.\n" |
| "-capi Use C APIs instead of C++ APIs (currently not working)\n" |
| "-next Do the next test\n" |
| "-isBound Do the isBound test\n" |
| ; |
| |
| |
| #include <stdio.h> |
| #include <string.h> |
| #include <stdlib.h> |
| #include <math.h> |
| #include <locale.h> |
| #include <errno.h> |
| #include <sys/stat.h> |
| |
| #include <unicode/utypes.h> |
| #include <unicode/ucol.h> |
| #include <unicode/ucoleitr.h> |
| #include <unicode/uloc.h> |
| #include <unicode/ustring.h> |
| #include <unicode/ures.h> |
| #include <unicode/uchar.h> |
| #include <unicode/ucnv.h> |
| #include <unicode/utf8.h> |
| |
| #include <unicode/brkiter.h> |
| |
| |
| #ifdef WIN32 |
| #include <windows.h> |
| #else |
| // |
| // Stubs for Windows API functions when building on UNIXes. |
| // |
| #include <sys/time.h> |
| unsigned long timeGetTime() { |
| struct timeval t; |
| gettimeofday(&t, 0); |
| unsigned long val = t.tv_sec * 1000; // Let it overflow. Who cares. |
| val += t.tv_usec / 1000; |
| return val; |
| }; |
| #define MAKELCID(a,b) 0 |
| #endif |
| |
| |
| // |
| // Command line option variables |
| // These global variables are set according to the options specified |
| // on the command line by the user. |
| char * opt_fName = 0; |
| char * opt_locale = "en_US"; |
| int opt_langid = 0; // Defaults to value corresponding to opt_locale. |
| char * opt_rules = 0; |
| UBool opt_help = FALSE; |
| int opt_time = 0; |
| int opt_loopCount = 0; |
| int opt_passesCount= 1; |
| UBool opt_terse = FALSE; |
| UBool opt_icu = TRUE; |
| UBool opt_win = FALSE; // Run with Windows native functions. |
| UBool opt_unix = FALSE; // Run with UNIX strcoll, strxfrm functions. |
| UBool opt_mac = FALSE; // Run with MacOSX word break services. |
| UBool opt_uselen = FALSE; |
| UBool opt_dump = FALSE; |
| UBool opt_char = FALSE; |
| UBool opt_word = FALSE; |
| UBool opt_line = FALSE; |
| UBool opt_sentence = FALSE; |
| UBool opt_capi = FALSE; |
| |
| UBool opt_next = FALSE; |
| UBool opt_isBound = FALSE; |
| |
| |
| |
| // |
| // Definitions for the command line options |
| // |
| struct OptSpec { |
| const char *name; |
| enum {FLAG, NUM, STRING} type; |
| void *pVar; |
| }; |
| |
| OptSpec opts[] = { |
| {"-file", OptSpec::STRING, &opt_fName}, |
| {"-locale", OptSpec::STRING, &opt_locale}, |
| {"-langid", OptSpec::NUM, &opt_langid}, |
| {"-win", OptSpec::FLAG, &opt_win}, |
| {"-unix", OptSpec::FLAG, &opt_unix}, |
| {"-mac", OptSpec::FLAG, &opt_mac}, |
| {"-uselen", OptSpec::FLAG, &opt_uselen}, |
| {"-loop", OptSpec::NUM, &opt_loopCount}, |
| {"-time", OptSpec::NUM, &opt_time}, |
| {"-passes", OptSpec::NUM, &opt_passesCount}, |
| {"-char", OptSpec::FLAG, &opt_char}, |
| {"-word", OptSpec::FLAG, &opt_word}, |
| {"-line", OptSpec::FLAG, &opt_line}, |
| {"-sentence", OptSpec::FLAG, &opt_sentence}, |
| {"-terse", OptSpec::FLAG, &opt_terse}, |
| {"-dump", OptSpec::FLAG, &opt_dump}, |
| {"-capi", OptSpec::FLAG, &opt_capi}, |
| {"-next", OptSpec::FLAG, &opt_next}, |
| {"-isBound", OptSpec::FLAG, &opt_isBound}, |
| {"-help", OptSpec::FLAG, &opt_help}, |
| {"-?", OptSpec::FLAG, &opt_help}, |
| {0, OptSpec::FLAG, 0} |
| }; |
| |
| |
| //--------------------------------------------------------------------------- |
| // |
| // Global variables pointing to and describing the test file |
| // |
| //--------------------------------------------------------------------------- |
| |
| //DWORD gWinLCID; |
| BreakIterator *brkit = NULL; |
| UChar *text = NULL; |
| int32_t textSize = 0; |
| |
| |
| |
| #ifdef U_DARWIN |
| #include <ApplicationServices/ApplicationServices.h> |
| enum{ |
| kUCTextBreakAllMask = (kUCTextBreakClusterMask | kUCTextBreakWordMask | kUCTextBreakLineMask) |
| }; |
| UCTextBreakType breakTypes[4] = {kUCTextBreakCharMask, kUCTextBreakClusterMask, kUCTextBreakWordMask, kUCTextBreakLineMask}; |
| TextBreakLocatorRef breakRef; |
| UCTextBreakType macBreakType; |
| |
| void createMACBrkIt() { |
| OSStatus status = noErr; |
| LocaleRef lref; |
| status = LocaleRefFromLocaleString(opt_locale, &lref); |
| status = UCCreateTextBreakLocator(lref, 0, kUCTextBreakAllMask, (TextBreakLocatorRef*)&breakRef); |
| if(opt_char == TRUE) { |
| macBreakType = kUCTextBreakClusterMask; |
| } else if(opt_word == TRUE) { |
| macBreakType = kUCTextBreakWordMask; |
| } else if(opt_line == TRUE) { |
| macBreakType = kUCTextBreakLineMask; |
| } else if(opt_sentence == TRUE) { |
| // error |
| // brkit = BreakIterator::createSentenceInstance(opt_locale, status); |
| } else { |
| // default is character iterator |
| macBreakType = kUCTextBreakClusterMask; |
| } |
| } |
| #endif |
| |
| void createICUBrkIt() { |
| // |
| // Set up an ICU break iterator |
| // |
| UErrorCode status = U_ZERO_ERROR; |
| if(opt_char == TRUE) { |
| brkit = BreakIterator::createCharacterInstance(opt_locale, status); |
| } else if(opt_word == TRUE) { |
| brkit = BreakIterator::createWordInstance(opt_locale, status); |
| } else if(opt_line == TRUE) { |
| brkit = BreakIterator::createLineInstance(opt_locale, status); |
| } else if(opt_sentence == TRUE) { |
| brkit = BreakIterator::createSentenceInstance(opt_locale, status); |
| } else { |
| // default is character iterator |
| brkit = BreakIterator::createCharacterInstance(opt_locale, status); |
| } |
| if (status==U_USING_DEFAULT_WARNING && opt_terse==FALSE) { |
| fprintf(stderr, "Warning, U_USING_DEFAULT_WARNING for %s\n", opt_locale); |
| } |
| if (status==U_USING_FALLBACK_WARNING && opt_terse==FALSE) { |
| fprintf(stderr, "Warning, U_USING_FALLBACK_ERROR for %s\n", opt_locale); |
| } |
| |
| } |
| |
| //--------------------------------------------------------------------------- |
| // |
| // ProcessOptions() Function to read the command line options. |
| // |
| //--------------------------------------------------------------------------- |
| UBool ProcessOptions(int argc, const char **argv, OptSpec opts[]) |
| { |
| int i; |
| int argNum; |
| const char *pArgName; |
| OptSpec *pOpt; |
| |
| for (argNum=1; argNum<argc; argNum++) { |
| pArgName = argv[argNum]; |
| for (pOpt = opts; pOpt->name != 0; pOpt++) { |
| if (strcmp(pOpt->name, pArgName) == 0) { |
| switch (pOpt->type) { |
| case OptSpec::FLAG: |
| *(UBool *)(pOpt->pVar) = TRUE; |
| break; |
| case OptSpec::STRING: |
| argNum ++; |
| if (argNum >= argc) { |
| fprintf(stderr, "value expected for \"%s\" option.\n", pOpt->name); |
| return FALSE; |
| } |
| *(const char **)(pOpt->pVar) = argv[argNum]; |
| break; |
| case OptSpec::NUM: |
| argNum ++; |
| if (argNum >= argc) { |
| fprintf(stderr, "value expected for \"%s\" option.\n", pOpt->name); |
| return FALSE; |
| } |
| char *endp; |
| i = strtol(argv[argNum], &endp, 0); |
| if (endp == argv[argNum]) { |
| fprintf(stderr, "integer value expected for \"%s\" option.\n", pOpt->name); |
| return FALSE; |
| } |
| *(int *)(pOpt->pVar) = i; |
| } |
| break; |
| } |
| } |
| if (pOpt->name == 0) |
| { |
| fprintf(stderr, "Unrecognized option \"%s\"\n", pArgName); |
| return FALSE; |
| } |
| } |
| return TRUE; |
| } |
| |
| |
| void doForwardTest() { |
| if (opt_terse == FALSE) { |
| printf("Doing the forward test\n"); |
| } |
| int32_t noBreaks = 0; |
| int32_t i = 0; |
| unsigned long startTime = timeGetTime(); |
| unsigned long elapsedTime = 0; |
| if(opt_icu) { |
| createICUBrkIt(); |
| brkit->setText(UnicodeString(text, textSize)); |
| brkit->first(); |
| if (opt_terse == FALSE) { |
| printf("Warmup\n"); |
| } |
| int j; |
| while((j = brkit->next()) != BreakIterator::DONE) { |
| noBreaks++; |
| //fprintf(stderr, "%d ", j); |
| } |
| |
| if (opt_terse == FALSE) { |
| printf("Measure\n"); |
| } |
| startTime = timeGetTime(); |
| for(i = 0; i < opt_loopCount; i++) { |
| brkit->first(); |
| while(brkit->next() != BreakIterator::DONE) { |
| } |
| } |
| |
| elapsedTime = timeGetTime()-startTime; |
| } else if(opt_mac) { |
| #ifdef U_DARWIN |
| createMACBrkIt(); |
| UniChar* filePtr = text; |
| OSStatus status = noErr; |
| UniCharCount startOffset = 0, breakOffset = 0, numUniChars = textSize; |
| startOffset = 0; |
| //printf("\t---Search forward--\n"); |
| |
| while (startOffset < numUniChars) |
| { |
| status = UCFindTextBreak(breakRef, macBreakType, kUCTextBreakLeadingEdgeMask, filePtr, numUniChars, |
| startOffset, &breakOffset); |
| //require_action(status == noErr, EXIT, printf( "**UCFindTextBreak failed: startOffset %d, status %d\n", (int)startOffset, (int)status)); |
| //require_action((breakOffset <= numUniChars),EXIT, printf("**UCFindTextBreak breakOffset too big: startOffset %d, breakOffset %d\n", (int)startOffset, (int)breakOffset)); |
| |
| // Output break |
| //printf("\t%d\n", (int)breakOffset); |
| |
| // Increment counters |
| noBreaks++; |
| startOffset = breakOffset; |
| } |
| startTime = timeGetTime(); |
| for(i = 0; i < opt_loopCount; i++) { |
| startOffset = 0; |
| |
| while (startOffset < numUniChars) |
| { |
| status = UCFindTextBreak(breakRef, macBreakType, kUCTextBreakLeadingEdgeMask, filePtr, numUniChars, |
| startOffset, &breakOffset); |
| // Increment counters |
| startOffset = breakOffset; |
| } |
| } |
| elapsedTime = timeGetTime()-startTime; |
| UCDisposeTextBreakLocator(&breakRef); |
| #endif |
| |
| |
| } |
| |
| |
| if (opt_terse == FALSE) { |
| int32_t loopTime = (int)(float(1000) * ((float)elapsedTime/(float)opt_loopCount)); |
| int32_t timePerCU = (int)(float(1000) * ((float)loopTime/(float)textSize)); |
| int32_t timePerBreak = (int)(float(1000) * ((float)loopTime/(float)noBreaks)); |
| printf("forward break iteration average loop time %d\n", loopTime); |
| printf("number of code units %d average time per code unit %d\n", textSize, timePerCU); |
| printf("number of breaks %d average time per break %d\n", noBreaks, timePerBreak); |
| } else { |
| printf("time=%d\nevents=%d\nsize=%d\n", elapsedTime, noBreaks, textSize); |
| } |
| |
| |
| } |
| |
| void doIsBoundTest() { |
| int32_t noBreaks = 0, hit = 0; |
| int32_t i = 0, j = 0; |
| unsigned long startTime = timeGetTime(); |
| unsigned long elapsedTime = 0; |
| createICUBrkIt(); |
| brkit->setText(UnicodeString(text, textSize)); |
| brkit->first(); |
| for(j = 0; j < textSize; j++) { |
| if(brkit->isBoundary(j)) { |
| noBreaks++; |
| //fprintf(stderr, "%d ", j); |
| } |
| } |
| /* |
| while(brkit->next() != BreakIterator::DONE) { |
| noBreaks++; |
| } |
| */ |
| |
| startTime = timeGetTime(); |
| for(i = 0; i < opt_loopCount; i++) { |
| for(j = 0; j < textSize; j++) { |
| if(brkit->isBoundary(j)) { |
| hit++; |
| } |
| } |
| } |
| |
| elapsedTime = timeGetTime()-startTime; |
| int32_t loopTime = (int)(float(1000) * ((float)elapsedTime/(float)opt_loopCount)); |
| if (opt_terse == FALSE) { |
| int32_t timePerCU = (int)(float(1000) * ((float)loopTime/(float)textSize)); |
| int32_t timePerBreak = (int)(float(1000) * ((float)loopTime/(float)noBreaks)); |
| printf("forward break iteration average loop time %d\n", loopTime); |
| printf("number of code units %d average time per code unit %d\n", textSize, timePerCU); |
| printf("number of breaks %d average time per break %d\n", noBreaks, timePerBreak); |
| } else { |
| printf("time=%d\nevents=%d\nsize=%d\n", elapsedTime, noBreaks, textSize); |
| } |
| } |
| |
| //---------------------------------------------------------------------------------------- |
| // |
| // UnixConvert -- Convert the lines of the file to the encoding for UNIX |
| // Since it appears that Unicode support is going in the general |
| // direction of the use of UTF-8 locales, that is the approach |
| // that is used here. |
| // |
| //---------------------------------------------------------------------------------------- |
| void UnixConvert() { |
| #if 0 |
| int line; |
| |
| UConverter *cvrtr; // An ICU code page converter. |
| UErrorCode status = U_ZERO_ERROR; |
| |
| |
| cvrtr = ucnv_open("utf-8", &status); // we are just doing UTF-8 locales for now. |
| if (U_FAILURE(status)) { |
| fprintf(stderr, "ICU Converter open failed.: %d\n", &status); |
| exit(-1); |
| } |
| // redo for unix |
| for (line=0; line < gNumFileLines; line++) { |
| int sizeNeeded = ucnv_fromUChars(cvrtr, |
| 0, // ptr to target buffer. |
| 0, // length of target buffer. |
| gFileLines[line].name, |
| -1, // source is null terminated |
| &status); |
| if (status != U_BUFFER_OVERFLOW_ERROR && status != U_ZERO_ERROR) { |
| fprintf(stderr, "Conversion from Unicode, something is wrong.\n"); |
| exit(-1); |
| } |
| status = U_ZERO_ERROR; |
| gFileLines[line].unixName = new char[sizeNeeded+1]; |
| sizeNeeded = ucnv_fromUChars(cvrtr, |
| gFileLines[line].unixName, // ptr to target buffer. |
| sizeNeeded+1, // length of target buffer. |
| gFileLines[line].name, |
| -1, // source is null terminated |
| &status); |
| if (U_FAILURE(status)) { |
| fprintf(stderr, "ICU Conversion Failed.: %d\n", status); |
| exit(-1); |
| } |
| gFileLines[line].unixName[sizeNeeded] = 0; |
| }; |
| ucnv_close(cvrtr); |
| #endif |
| } |
| |
| |
| //---------------------------------------------------------------------------------------- |
| // |
| // class UCharFile Class to hide all the gorp to read a file in |
| // and produce a stream of UChars. |
| // |
| //---------------------------------------------------------------------------------------- |
| class UCharFile { |
| public: |
| UCharFile(const char *fileName); |
| ~UCharFile(); |
| UChar get(); |
| UBool eof() {return fEof;}; |
| UBool error() {return fError;}; |
| int32_t size() { return fFileSize; }; |
| |
| private: |
| UCharFile (const UCharFile &other) {}; // No copy constructor. |
| UCharFile & operator = (const UCharFile &other) {return *this;}; // No assignment op |
| |
| FILE *fFile; |
| const char *fName; |
| UBool fEof; |
| UBool fError; |
| UChar fPending2ndSurrogate; |
| int32_t fFileSize; |
| |
| enum {UTF16LE, UTF16BE, UTF8} fEncoding; |
| }; |
| |
| UCharFile::UCharFile(const char * fileName) { |
| fEof = FALSE; |
| fError = FALSE; |
| fName = fileName; |
| struct stat buf; |
| int32_t result = stat(fileName, &buf); |
| if(result != 0) { |
| fprintf(stderr, "Error getting info\n"); |
| fFileSize = -1; |
| } else { |
| fFileSize = buf.st_size; |
| } |
| fFile = fopen(fName, "rb"); |
| fPending2ndSurrogate = 0; |
| if (fFile == NULL) { |
| fprintf(stderr, "Can not open file \"%s\"\n", opt_fName); |
| fError = TRUE; |
| return; |
| } |
| // |
| // Look for the byte order mark at the start of the file. |
| // |
| int BOMC1, BOMC2, BOMC3; |
| BOMC1 = fgetc(fFile); |
| BOMC2 = fgetc(fFile); |
| |
| if (BOMC1 == 0xff && BOMC2 == 0xfe) { |
| fEncoding = UTF16LE; } |
| else if (BOMC1 == 0xfe && BOMC2 == 0xff) { |
| fEncoding = UTF16BE; } |
| else if (BOMC1 == 0xEF && BOMC2 == 0xBB && (BOMC3 = fgetc(fFile)) == 0xBF ) { |
| fEncoding = UTF8; } |
| else |
| { |
| fprintf(stderr, "collperf: file \"%s\" encoding must be UTF-8 or UTF-16, and " |
| "must include a BOM.\n", fileName); |
| fError = true; |
| return; |
| } |
| } |
| |
| |
| UCharFile::~UCharFile() { |
| fclose(fFile); |
| } |
| |
| |
| |
| UChar UCharFile::get() { |
| UChar c; |
| switch (fEncoding) { |
| case UTF16LE: |
| { |
| int cL, cH; |
| cL = fgetc(fFile); |
| cH = fgetc(fFile); |
| c = cL | (cH << 8); |
| if (cH == EOF) { |
| c = 0; |
| fEof = TRUE; |
| } |
| break; |
| } |
| case UTF16BE: |
| { |
| int cL, cH; |
| cH = fgetc(fFile); |
| cL = fgetc(fFile); |
| c = cL | (cH << 8); |
| if (cL == EOF) { |
| c = 0; |
| fEof = TRUE; |
| } |
| break; |
| } |
| case UTF8: |
| { |
| if (fPending2ndSurrogate != 0) { |
| c = fPending2ndSurrogate; |
| fPending2ndSurrogate = 0; |
| break; |
| } |
| |
| int ch = fgetc(fFile); // Note: c and ch are separate cause eof test doesn't work on UChar type. |
| if (ch == EOF) { |
| c = 0; |
| fEof = TRUE; |
| break; |
| } |
| |
| if (ch <= 0x7f) { |
| // It's ascii. No further utf-8 conversion. |
| c = ch; |
| break; |
| } |
| |
| // Figure out the lenght of the char and read the rest of the bytes |
| // into a temp array. |
| int nBytes; |
| if (ch >= 0xF0) {nBytes=4;} |
| else if (ch >= 0xE0) {nBytes=3;} |
| else if (ch >= 0xC0) {nBytes=2;} |
| else { |
| fprintf(stderr, "not likely utf-8 encoded file %s contains corrupt data at offset %d.\n", fName, ftell(fFile)); |
| fError = TRUE; |
| return 0; |
| } |
| |
| unsigned char bytes[10]; |
| bytes[0] = (unsigned char)ch; |
| int i; |
| for (i=1; i<nBytes; i++) { |
| bytes[i] = fgetc(fFile); |
| if (bytes[i] < 0x80 || bytes[i] >= 0xc0) { |
| fprintf(stderr, "utf-8 encoded file %s contains corrupt data at offset %d. Expected %d bytes, byte %d is invalid. First byte is %02X\n", fName, ftell(fFile), nBytes, i, ch); |
| fError = TRUE; |
| return 0; |
| } |
| } |
| |
| // Convert the bytes from the temp array to a Unicode char. |
| i = 0; |
| uint32_t cp; |
| UTF8_NEXT_CHAR_UNSAFE(bytes, i, cp); |
| c = (UChar)cp; |
| |
| if (cp >= 0x10000) { |
| // The code point needs to be broken up into a utf-16 surrogate pair. |
| // Process first half this time through the main loop, and |
| // remember the other half for the next time through. |
| UChar utf16Buf[3]; |
| i = 0; |
| UTF16_APPEND_CHAR_UNSAFE(utf16Buf, i, cp); |
| fPending2ndSurrogate = utf16Buf[1]; |
| c = utf16Buf[0]; |
| } |
| break; |
| }; |
| } |
| return c; |
| } |
| |
| |
| //---------------------------------------------------------------------------------------- |
| // |
| // Main -- process command line, read in and pre-process the test file, |
| // call other functions to do the actual tests. |
| // |
| //---------------------------------------------------------------------------------------- |
| int main(int argc, const char** argv) { |
| if (ProcessOptions(argc, argv, opts) != TRUE || opt_help || opt_fName == 0) { |
| printf(gUsageString); |
| exit (1); |
| } |
| // Make sure that we've only got one API selected. |
| if (opt_mac || opt_unix || opt_win) opt_icu = FALSE; |
| if (opt_mac || opt_unix) opt_win = FALSE; |
| if (opt_mac) opt_unix = FALSE; |
| |
| UErrorCode status = U_ZERO_ERROR; |
| |
| |
| |
| // |
| // Set up a Windows LCID |
| // |
| /* |
| if (opt_langid != 0) { |
| gWinLCID = MAKELCID(opt_langid, SORT_DEFAULT); |
| } |
| else { |
| gWinLCID = uloc_getLCID(opt_locale); |
| } |
| */ |
| |
| // |
| // Set the UNIX locale |
| // |
| if (opt_unix) { |
| if (setlocale(LC_ALL, opt_locale) == 0) { |
| fprintf(stderr, "setlocale(LC_ALL, %s) failed.\n", opt_locale); |
| exit(-1); |
| } |
| } |
| |
| // Read in the input file. |
| // File assumed to be utf-16. |
| // Lines go onto heap buffers. Global index array to line starts is created. |
| // Lines themselves are null terminated. |
| // |
| |
| UCharFile f(opt_fName); |
| if (f.error()) { |
| exit(-1); |
| } |
| int32_t fileSize = f.size(); |
| const int STARTSIZE = 70000; |
| int32_t bufSize = 0; |
| int32_t charCount = 0; |
| if(fileSize != -1) { |
| text = (UChar *)malloc(fileSize*sizeof(UChar)); |
| bufSize = fileSize; |
| } else { |
| text = (UChar *)malloc(STARTSIZE*sizeof(UChar)); |
| bufSize = STARTSIZE; |
| } |
| if(text == NULL) { |
| fprintf(stderr, "Allocating buffer failed\n"); |
| exit(-1); |
| } |
| |
| |
| // Read the file, split into lines, and save in memory. |
| // Loop runs once per utf-16 value from the input file, |
| // (The number of bytes read from file per loop iteration depends on external encoding.) |
| for (;;) { |
| |
| UChar c = f.get(); |
| if(f.eof()) { |
| break; |
| } |
| if (f.error()){ |
| exit(-1); |
| } |
| // We now have a good UTF-16 value in c. |
| text[charCount++] = c; |
| if(charCount == bufSize) { |
| text = (UChar *)realloc(text, 2*bufSize*sizeof(UChar)); |
| if(text == NULL) { |
| fprintf(stderr, "Reallocating buffer failed\n"); |
| exit(-1); |
| } |
| bufSize *= 2; |
| } |
| } |
| |
| |
| if (opt_terse == FALSE) { |
| printf("file \"%s\", %d charCount code units.\n", opt_fName, charCount); |
| } |
| |
| textSize = charCount; |
| |
| |
| |
| |
| // |
| // Dump file contents if requested. |
| // |
| if (opt_dump) { |
| // dump file, etc... possibly |
| } |
| |
| |
| // |
| // We've got the file read into memory. Go do something with it. |
| // |
| int32_t i = 0; |
| for(i = 0; i < opt_passesCount; i++) { |
| if(opt_loopCount != 0) { |
| if(opt_next) { |
| doForwardTest(); |
| } else if(opt_isBound) { |
| doIsBoundTest(); |
| } else { |
| doForwardTest(); |
| } |
| } else if(opt_time != 0) { |
| |
| } |
| } |
| |
| if(text != NULL) { |
| free(text); |
| } |
| if(brkit != NULL) { |
| delete brkit; |
| } |
| |
| return 0; |
| } |
