| /* |
| ********************************************************************** |
| * Copyright (C) 2002, International Business Machines |
| * Corporation and others. All Rights Reserved. |
| ********************************************************************** |
| * file name: utfperf.c |
| * encoding: US-ASCII |
| * tab size: 8 (not used) |
| * indentation:4 |
| * |
| * created on: 2002apr17 |
| * created by: Markus W. Scherer |
| * |
| * Performance test program for Unicode converters |
| * (converters that support all Unicode code points). |
| * Takes a UTF-8 file as input. |
| */ |
| |
| #include <stdio.h> |
| #include <string.h> |
| |
| #include <fcntl.h> /* for _O_BINARY */ |
| #include <io.h> /* for _setmode() */ |
| |
| #if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64) |
| # include <windows.h> |
| #else |
| # include <sys/time.h> |
| static unsigned long |
| timeGetTime() { |
| struct timeval t; |
| |
| gettimeofday(&t, 0); |
| return t.tv_sec*1000+t.tv_usec/1000; |
| }; |
| #endif |
| |
| #include "unicode/utypes.h" |
| #include "unicode/ucnv.h" |
| #include "unicode/ustring.h" |
| |
| /* definitions and text buffers */ |
| |
| #define INPUT_CAPACITY (1024*1024) |
| #define INTERMEDIATE_CAPACITY 4096 |
| #define INTERMEDIATE_SMALL_CAPACITY 20 |
| #define OUTPUT_CAPACITY INPUT_CAPACITY |
| |
| #define TARGET_MEASURE_TIME_MS 2000 |
| |
| #define PERCENT(a, b) (int)(((a)*200+1)/(2*(b))) |
| |
| #define ARRAY_LENGTH(a) (sizeof(a)/sizeof((a)[0])) |
| |
| static UChar input[INPUT_CAPACITY], output[OUTPUT_CAPACITY]; |
| static char intermediate[INTERMEDIATE_CAPACITY]; |
| |
| static int32_t inputLength, encodedLength, outputLength, countInputCodePoints; |
| |
| static int32_t utf8Length=0; |
| static double utf8Time=0.; |
| |
| static const char *const |
| utfNames[]={ |
| "UTF-8", /* UTF-8 should always be first to serve as percentage reference */ |
| "SCSU", "BOCU-1" /*, "CESU-8" *//*, "UTF-16BE", "UTF-16LE"*//*, "GB18030"*/ |
| }; |
| |
| /* functions */ |
| |
| typedef void |
| RoundtripFn(UConverter *cnv, int32_t intermediateCapacity, UErrorCode *pErrorCode); |
| |
| static void |
| roundtrip(UConverter *cnv, int32_t intermediateCapacity, UErrorCode *pErrorCode) { |
| const UChar *pIn, *pInLimit; |
| UChar *pOut, *pOutLimit; |
| char *pInter, *pInterLimit, *p; |
| UBool flush; |
| |
| ucnv_reset(cnv); |
| |
| pIn=input; |
| pInLimit=input+inputLength; |
| |
| pOut=output; |
| pOutLimit=output+OUTPUT_CAPACITY; |
| |
| pInterLimit=intermediate+intermediateCapacity; |
| |
| encodedLength=outputLength=0; |
| flush=FALSE; |
| |
| while(pIn<pInLimit || !flush) { |
| /* convert a block of [pIn..pInLimit[ to the encoding in intermediate[] */ |
| pInter=intermediate; |
| flush=(UBool)(pIn==pInLimit); |
| ucnv_fromUnicode(cnv, |
| &pInter, pInterLimit, |
| &pIn, pInLimit, |
| NULL, flush, |
| pErrorCode); |
| encodedLength+=(int32_t)(pInter-intermediate); |
| |
| if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) { |
| /* in case flush was TRUE make sure that we convert once more to really flush */ |
| flush=FALSE; |
| *pErrorCode=U_ZERO_ERROR; |
| } else if(U_FAILURE(*pErrorCode)) { |
| return; |
| } |
| |
| /* convert the block [intermediate..pInter[ back to UTF-16 */ |
| p=intermediate; |
| ucnv_toUnicode(cnv, |
| &pOut, pOutLimit, |
| &p, pInter, |
| NULL, flush, |
| pErrorCode); |
| if(U_FAILURE(*pErrorCode)) { |
| return; |
| } |
| /* intermediate must have been consumed (p==pInter) because of the converter semantics */ |
| } |
| |
| outputLength=pOut-output; |
| if(inputLength!=outputLength) { |
| fprintf(stderr, "error: roundtrip failed, inputLength %d!=outputLength %d\n", inputLength, outputLength); |
| *pErrorCode=U_INTERNAL_PROGRAM_ERROR; |
| } |
| } |
| |
| static void |
| noop(UConverter *cnv, int32_t intermediateCapacity, UErrorCode *pErrorCode) { |
| /* do nothing */ |
| } |
| |
| static unsigned long |
| measureRoundtrips(RoundtripFn *fn, UConverter *cnv, const char *encName, int32_t intermediateCapacity, int32_t n) { |
| unsigned long _time; |
| UErrorCode errorCode; |
| |
| _time=timeGetTime(); |
| errorCode=U_ZERO_ERROR; |
| do { |
| fn(cnv, intermediateCapacity, &errorCode); |
| } while(U_SUCCESS(errorCode) && --n>0); |
| _time=timeGetTime()-_time; |
| |
| if(U_FAILURE(errorCode)) { |
| fprintf(stderr, "error in roundtrip conversion (%s): %s\n", encName, u_errorName(errorCode)); |
| return 0x7fffffff; |
| } |
| |
| if(0!=u_memcmp(input, output, inputLength)) { |
| fprintf(stderr, "error: roundtrip failed, input[]!=output[]\n"); |
| return 0x7fffffff; |
| } |
| |
| return _time; |
| } |
| |
| static void |
| perEncAndCapacity(UConverter *cnv, const char *encName, int32_t intermediateCapacity) { |
| double rtTime; |
| unsigned long _time; |
| int32_t n; |
| |
| /*printf("test performance for %s with intermediate capacity %d\n", encName, intermediateCapacity);*/ |
| |
| /* warm up caches and estimate loop time */ |
| n=10; |
| for(;;) { |
| _time=measureRoundtrips(roundtrip, cnv, encName, intermediateCapacity, n); |
| if(_time<500 && _time<TARGET_MEASURE_TIME_MS/10) { |
| n*=10; |
| } else { |
| break; |
| } |
| } |
| |
| if(_time<TARGET_MEASURE_TIME_MS) { |
| n=(n*TARGET_MEASURE_TIME_MS)/_time+1; |
| } |
| |
| /* run actual measurement with a target test time of 10s */ |
| _time=measureRoundtrips(roundtrip, cnv, encName, intermediateCapacity, n); |
| |
| /* subtract same number of loops over no-operation function */ |
| _time-=measureRoundtrips(noop, cnv, encName, intermediateCapacity, n); |
| |
| rtTime=((double)_time*1000.)/(double)n; |
| |
| /* report */ |
| printf("* performance report for %8s:\n", encName); |
| printf(" intermediate buffer capacity %8d B\n", intermediateCapacity); |
| if(intermediateCapacity==INTERMEDIATE_CAPACITY && utf8Length!=0) { |
| printf(" number of encoding bytes %8d B (%3d%% of UTF-8)\n", encodedLength, PERCENT(encodedLength, utf8Length)); |
| printf(" roundtrip conversion time %8g μs (%3d%% of UTF-8)\n", rtTime, PERCENT(rtTime, utf8Time)); |
| } else { |
| printf(" number of encoding bytes %8d B\n", encodedLength); |
| printf(" roundtrip conversion time %8g μs\n", rtTime); |
| } |
| printf(" average bytes/code point %8g B/cp\n", (double)encodedLength/countInputCodePoints); |
| puts(""); |
| |
| /* set UTF-8 values */ |
| if(intermediateCapacity==INTERMEDIATE_CAPACITY && 0==strcmp(encName, "UTF-8")) { |
| utf8Length=encodedLength; |
| utf8Time=rtTime; |
| } |
| } |
| |
| static void |
| perEnc(UConverter *cnv, const char *encName) { |
| /*printf("test performance for %s\n", encName);*/ |
| perEncAndCapacity(cnv, encName, INTERMEDIATE_CAPACITY); |
| perEncAndCapacity(cnv, encName, INTERMEDIATE_SMALL_CAPACITY); |
| } |
| |
| static void |
| testPerformance() { |
| UConverter *cnv; |
| UErrorCode errorCode; |
| int32_t i; |
| |
| printf("number of code points %8d cp\n", countInputCodePoints); |
| printf("platform endianness: %8s-endian\n", U_IS_BIG_ENDIAN ? "big" : "little"); |
| puts(""); |
| for(i=0; i<ARRAY_LENGTH(utfNames); ++i) { |
| errorCode=U_ZERO_ERROR; |
| cnv=ucnv_open(utfNames[i], &errorCode); |
| if(U_SUCCESS(errorCode)) { |
| perEnc(cnv, utfNames[i]); |
| ucnv_close(cnv); |
| } else { |
| fprintf(stderr, "error opening converter for \"%s\" - %s\n", utfNames[i], u_errorName(errorCode)); |
| } |
| } |
| } |
| |
| /* read a complete block from the input file */ |
| static int32_t |
| readBlock(FILE *in) { |
| int length, blockLength; |
| |
| blockLength=0; |
| while(blockLength<INTERMEDIATE_CAPACITY && !feof(in)) { |
| length=fread(intermediate, 1, INTERMEDIATE_CAPACITY-blockLength, in); |
| if(length<0 || ferror(in)) { |
| return -1; |
| } |
| blockLength+=length; |
| } |
| |
| return (int32_t)blockLength; |
| } |
| |
| static UBool |
| readInput(FILE *in, const char *encName) { |
| UConverter *cnv; |
| UChar *pOut, *pOutLimit; |
| const char *p, *limit; |
| int32_t length; |
| UErrorCode errorCode; |
| |
| pOut=input; |
| pOutLimit=input+INPUT_CAPACITY; |
| |
| errorCode=U_ZERO_ERROR; |
| |
| /* read the first block and open the converter */ |
| length=readBlock(in); |
| if(length<0) { |
| return FALSE; |
| } |
| |
| if(encName==NULL) { |
| int32_t signatureLength; |
| encName=ucnv_detectUnicodeSignature(intermediate, length, |
| &signatureLength, |
| &errorCode); |
| if(U_FAILURE(errorCode) || encName==NULL) { |
| /* default to UTF-8 */ |
| printf("no Unicode signature - using UTF-8\n"); |
| encName="UTF-8"; |
| errorCode=U_ZERO_ERROR; |
| } else { |
| printf("detected signature for %s (removing %d bytes)\n", encName, signatureLength); |
| /* remove signature byte sequence */ |
| memmove(intermediate, intermediate+signatureLength, length-=signatureLength); |
| } |
| } |
| |
| cnv=ucnv_open(encName, &errorCode); |
| if(U_FAILURE(errorCode)) { |
| fprintf(stderr, "error: unable to ucnv_open(\"%s\") - %s\n", encName, u_errorName(errorCode)); |
| return FALSE; |
| } |
| |
| while(length>0) { |
| /* convert the block */ |
| p=intermediate; |
| limit=p+length; |
| |
| ucnv_toUnicode(cnv, |
| &pOut, pOutLimit, |
| &p, limit, |
| NULL, FALSE, |
| &errorCode); |
| if(U_FAILURE(errorCode)) { |
| fprintf(stderr, "error converting input to UTF-16: %s\n", u_errorName(errorCode)); |
| ucnv_close(cnv); |
| return FALSE; |
| } |
| |
| /* read the next block */ |
| length=readBlock(in); |
| if(length<0) { |
| ucnv_close(cnv); |
| return FALSE; |
| } |
| } |
| |
| /* flush the converter */ |
| ucnv_toUnicode(cnv, |
| &pOut, pOutLimit, |
| &p, p, |
| NULL, TRUE, |
| &errorCode); |
| ucnv_close(cnv); |
| |
| if(U_FAILURE(errorCode)) { |
| fprintf(stderr, "error converting input to UTF-16: %s\n", u_errorName(errorCode)); |
| return FALSE; |
| } |
| |
| inputLength=(int32_t)(pOut-input); |
| countInputCodePoints=u_countChar32(input, inputLength); |
| if(inputLength<=0) { |
| fprintf(stderr, "warning: input is empty\n"); |
| return FALSE; |
| } |
| |
| return TRUE; |
| } |
| |
| static void |
| showUsage(const char *myName) { |
| fprintf(stderr, |
| "Usage:\n" |
| "%s [-e encoding-name] filename | '-'\n" |
| " encoding-name must be the name of an encoding supported by ICU\n" |
| " the filename of the input file with text to be used\n" |
| " can be a dash (-) for standard input\n", |
| myName); |
| } |
| |
| /* |
| * Read file using some encoding, convert to 1M UTF-16 input buffer. |
| * For each UTF to be tested: |
| * n times: |
| * convert from UTF-16 input buffer to UTF, 4kB buffer |
| * convert from 4kB buffer to 1M UTF-16 output buffer |
| * adjust n so that time elapsed is 10s (#define) |
| * ->divide 10s by time, increase n by that factor, run 2nd time |
| * n times: |
| * empty function |
| * subtract out loop/function overhead |
| * display #code points - #UTF bytes - time per roundtrip |
| * |
| * * do the same again with an intermediate buffer size of 20 instead of 4kB |
| * |
| * Test following UTFs: |
| * UTF-16BE, UTF-16LE, UTF-8, SCSU, BOCU-1, CESU-8 |
| * |
| * Command-line arguments: |
| * - encoding (default UTF-8, detect BOM) |
| * - filename (allow "-") |
| */ |
| extern int |
| main(int argc, const char *argv[]) { |
| FILE *in; |
| const char *myName, *encName, *filename, *basename; |
| |
| myName=argv[0]; |
| if(argc<2) { |
| showUsage(myName); |
| return 1; |
| } |
| |
| /* get encoding name argument */ |
| if(argv[1][0]=='-' && argv[1][1]=='e') { |
| encName=argv[1]+2; |
| --argc; |
| ++argv; |
| if(*encName==0) { |
| if(argc<2) { |
| showUsage(myName); |
| return 1; |
| } |
| encName=argv[1]; |
| --argc; |
| ++argv; |
| } |
| } else { |
| encName=NULL; |
| } |
| |
| /* get filename argument */ |
| if(argc<2) { |
| showUsage(myName); |
| return 1; |
| } |
| filename=argv[1]; |
| if(filename[0]=='-' && filename[1]==0) { |
| filename="(standard input)"; |
| in=stdin; |
| /* set stdin to binary mode */ |
| _setmode(_fileno(stdin), _O_BINARY); |
| } else { |
| in=fopen(filename, "rb"); |
| if(in==NULL) { |
| fprintf(stderr, "error opening \"%s\"\n", filename); |
| showUsage(myName); |
| return 2; |
| } |
| } |
| |
| /* read input */ |
| basename=strrchr(filename, U_FILE_SEP_CHAR); |
| if(basename!=NULL) { |
| ++basename; |
| } else { |
| basename=filename; |
| } |
| printf("# testing converter performance with file \"%s\"\n", basename); |
| if(!readInput(in, encName)) { |
| fprintf(stderr, "error reading \"%s\" (encoding %s)\n", filename, encName); |
| showUsage(myName); |
| return 2; |
| } |
| if(in!=stdin) { |
| fclose(in); |
| } |
| |
| /* test performance */ |
| testPerformance(); |
| return 0; |
| } |
