#!/usr/local/bin/perl | |
# ******************************************************************************* | |
# * Copyright (C) 2002-2012 International Business Machines Corporation and * | |
# * others. All Rights Reserved. * | |
# ******************************************************************************* | |
use XML::LibXML; | |
# Assume we are running within the icu4j root directory | |
use lib 'src/com/ibm/icu/dev/test/perf'; | |
use Dataset; | |
my $OS=$^O; | |
my $CLASSPATH; | |
if ($OS eq "linux" || $OS eq "darwin") { | |
$CLASSPATH="../icu4j.jar:../tools/misc/out/lib/icu4j-tools.jar:out/bin"; | |
} else { | |
$CLASSPATH="../icu4j.jar;../tools/misc/out/lib/icu4j-tools.jar;out/bin"; | |
} | |
#--------------------------------------------------------------------- | |
# Methods to be tested. Each pair represents a test method and | |
# a baseline method which is used for comparison. | |
my @METHODS = ( | |
['TestJDKConstruction', 'TestICUConstruction'], | |
['TestJDKParse', 'TestICUParse'], | |
['TestJDKFormat', 'TestICUFormat'] | |
); | |
# Patterns which define the set of characters used for testing. | |
my @OPTIONS = ( | |
# locale pattern date string | |
[ "en_US", "dddd MMM yyyy", "15 Jan 2007"], | |
[ "sw_KE", "dddd MMM yyyy", "15 Jan 2007"], | |
[ "en_US", "HH:mm", "13:13"], | |
[ "en_US", "HH:mm zzzz", "13:13 Pacific Standard Time"], | |
[ "en_US", "HH:mm z", "13:13 PST"], | |
[ "en_US", "HH:mm Z", "13:13 -0800"], | |
); | |
my $THREADS; # number of threads (input from command-line args) | |
my $CALIBRATE = 2; # duration in seconds for initial calibration | |
my $DURATION = 10; # duration in seconds for each pass | |
my $NUMPASSES = 4; # number of passes. If > 1 then the first pass | |
# is discarded as a JIT warm-up pass. | |
my $TABLEATTR = 'BORDER="1" CELLPADDING="4" CELLSPACING="0"'; | |
my $PLUS_MINUS = "±"; | |
if ($NUMPASSES < 3) { | |
die "Need at least 3 passes. One is discarded (JIT warmup) and need two to have 1 degree of freedom (t distribution)."; | |
} | |
# run all tests with the specified number of threads from command-line input | |
# (if there is no arguments, use $THREADS = 1) | |
foreach my $arg ($#ARGV >= 0 ? @ARGV : "1") { | |
$THREADS = $arg; | |
main(); | |
} | |
#--------------------------------------------------------------------- | |
sub main { | |
#-----------DATE FORMAT PERFORMANCE TESTS----------------------------- | |
my $testclass = 'com.ibm.icu.dev.test.perf.DateFormatPerformanceTest'; | |
#my $threads = ($THREADS > 1) ? "($THREADS threads)" : ""; | |
my $doc = XML::LibXML::Document->new('1.0', 'utf-8'); | |
my $root = $doc->createElement("perfTestResults"); | |
# my $raw = ""; | |
my @shortNames = ( "open" , "parse", "fmt"); | |
my $index=0; | |
for my $methodPair (@METHODS) { | |
my $testMethod = $methodPair->[0]; | |
my $baselineMethod = $methodPair->[1]; | |
my $testname = $shortNames[$index]; | |
$index++; | |
$OUT = ''; | |
my $patternCounter=1; | |
for my $pat (@OPTIONS) { | |
# measure the test method | |
my $t = measure2($testclass, $testMethod, $pat, -$DURATION); | |
my $testResult = $t->getMean(); | |
my $jdkElement = $doc->createElement("perfTestResult"); | |
my $testName = "DateFmt-$testname-pat$patternCounter-JDK"; | |
$jdkElement->setAttribute("test" => $testName); | |
$jdkElement->setAttribute("iterations" => 1); | |
$jdkElement->setAttribute("time" => $testResult); | |
$root->appendChild($jdkElement); | |
# measure baseline method | |
my $b = measure2($testclass, $baselineMethod, $pat, -$DURATION); | |
my $baseResult = $b->getMean(); | |
my $icuElement = $doc->createElement("perfTestResult"); | |
my $testName = "DateFmt-$testname-pat$patternCounter"; | |
$patternCounter++; | |
$icuElement->setAttribute("test"=> $testName); | |
$icuElement->setAttribute("iterations" => 1); | |
$icuElement->setAttribute("time" => $baseResult); | |
$root->appendChild($icuElement); | |
} | |
} | |
#------------------DECIMAL FORMAT TESTS--------------------------------- | |
my $testclass = 'com.ibm.icu.dev.test.perf.DecimalFormatPerformanceTest'; | |
my @OPTIONS = ( | |
# locale pattern date string | |
[ "en_US", "#,###.##", "1,234.56"], | |
[ "de_DE", "#,###.##", "1.234,56"], | |
); | |
my $index=0; | |
for my $methodPair (@METHODS) { | |
my $testMethod = $methodPair->[0]; | |
my $baselineMethod = $methodPair->[1]; | |
my $testname = $shortNames[$index]; | |
$index++; | |
for my $pat (@OPTIONS) { | |
my $patternName = $pat->[0]; | |
# measure the test method | |
my $t = measure2($testclass, $testMethod, $pat, -$DURATION); | |
my $testResult = $t->getMean(); | |
my $jdkElement = $doc->createElement("perfTestResult"); | |
my $testName = "NumFmt-$testname-$patternName-JDK"; | |
$jdkElement->setAttribute("test" => $testName); | |
$jdkElement->setAttribute("iterations"=>1); | |
$jdkElement->setAttribute("time" => $testResult); | |
$root->appendChild($jdkElement); | |
# measure baseline method | |
my $b = measure2($testclass, $baselineMethod, $pat, -$DURATION); | |
my $baseResult = $b->getMean(); | |
my $icuElement = $doc->createElement("perfTestResult"); | |
my $testName = "NumFmt-$testname-$patternName"; | |
$icuElement->setAttribute("test"=> $testName); | |
$icuElement->setAttribute("iterations"=>1); | |
$icuElement->setAttribute("time" => $baseResult); | |
$root->appendChild($icuElement); | |
} | |
} | |
#----------------COLLATION PERFORMANCE TESTS--------------------------_ | |
%dataFiles = ( | |
"en_US", "TestNames_Latin.txt", | |
"da_DK", "TestNames_Latin.txt", | |
"de_DE", "TestNames_Latin.txt", | |
"de__PHONEBOOK", "TestNames_Latin.txt", | |
"fr_FR", "TestNames_Latin.txt", | |
"ja_JP", "TestNames_Latin.txt TestNames_Japanese_h.txt TestNames_Japanese_k.txt TestNames_Asian.txt", | |
"zh_CN", "TestNames_Latin.txt TestNames_Chinese.txt", | |
"zh_TW", "TestNames_Latin.txt TestNames_Chinese.txt", | |
"zh__PINYIN", "TestNames_Latin.txt TestNames_Chinese.txt", | |
"ru_RU", "TestNames_Latin.txt TestNames_Russian.txt", | |
"th", "TestNames_Latin.txt TestNames_Thai.txt", | |
"ko_KR", "TestNames_Latin.txt TestNames_Korean.txt", | |
); | |
# Outer loop runs through the locales to test | |
# (Edit this list dirctly to make changes) | |
# | |
foreach $locale ( | |
"en_US", | |
"da_DK", | |
"de_DE", | |
"de__PHONEBOOK", | |
"fr_FR", | |
"ja_JP", | |
"zh_CN", | |
"zh_TW", | |
"zh__PINYIN", | |
"ko_KR", | |
"ru_RU", | |
"th", | |
) | |
{ | |
# | |
# Inner loop runs over the set of data files specified for each locale. | |
# (Edit the %datafiles initialization, above, to make changes. | |
# | |
$ff = $dataFiles{$locale}; | |
@ff = split(/[\s]+/, $ff); | |
$counter = 1; | |
foreach $data (@ff) { | |
# | |
# Run ICU Test for this (locale, data file) pair. | |
# | |
$iStrCol = `java -classpath $CLASSPATH com.ibm.icu.dev.test.perf.CollationPerformanceTest -terse -file data/collation/$data -locale $locale -loop 1000 -binsearch`; | |
print "java -classpath $CLASSPATH com.ibm.icu.dev.test.perf.CollationPerformanceTest -terse -file data/collation/$data -locale $locale -loop 1000 -binsearch\n"; | |
$iStrCol =~s/[,\s]*//g; # whack off the leading " ," in the returned result. | |
doKeyTimes("java -classpath $CLASSPATH com.ibm.icu.dev.test.perf.CollationPerformanceTest -terse -file data/collation/$data -locale $locale -loop 1000 -keygen", | |
my $iKeyGen, my $iKeyLen); | |
# | |
# Run Windows test for this (locale, data file) pair. Only do if | |
# we are not on Windows 98/ME and we hava a windows langID | |
# for the locale. | |
# | |
$wStrCol = $wKeyGen = $wKeyLen = 0; | |
my $wStrCol = `java -classpath $CLASSPATH com.ibm.icu.dev.test.perf.CollationPerformanceTest -terse -file data/collation/$data -locale $locale -loop 1000 -binsearch -java`; | |
$wStrCol =~s/[,\s]*//g; # whack off the leading " ," in the returned result. | |
doKeyTimes("java -classpath $CLASSPATH com.ibm.icu.dev.test.perf.CollationPerformanceTest -terse -file data/collation/$data -locale $locale -loop 1000 -keygen -java", | |
$wKeyGen, $wKeyLen); | |
$collDiff = $keyGenDiff = $keyLenDiff = 0; | |
if ($wKeyLen > 0) { | |
$collDiff = (($wStrCol - $iStrCol) / $iStrCol) * 100; | |
$keyGenDiff = (($wKeyGen - $iKeyGen) / $iKeyGen) * 100; | |
$keyLenDiff = (($wKeyLen - $iKeyLen) / $iKeyLen) * 100; | |
} | |
my $ICU = $doc->createElement("perfTestResult"); | |
my $testname = "Coll-$locale-data$counter-StrCol"; | |
#write the results corresponding to this local,data pair | |
$ICU->setAttribute("test"=> $testname); | |
$ICU->setAttribute("iterations"=>1000); | |
$ICU->setAttribute("time"=> $iStrCol); | |
$root->appendChild($ICU); | |
my $Key = $doc->createElement("perfTestResult"); | |
my $testname = "Coll-$locale-data$counter-keyGen"; | |
$Key->setAttribute("test"=> $testname); | |
$Key->setAttribute("iterations"=>1000); | |
$Key->setAttribute("time"=>$iKeyGen); | |
$root->appendChild($Key); | |
my $JDK = $doc->createElement("perfTestResult"); | |
my $testname = "Coll-$locale-data$counter-StrCol-JDK"; | |
$JDK->setAttribute("test"=>$testname); | |
$JDK->setAttribute("iterations"=>1000); | |
$JDK->setAttribute("time"=>$wStrCol); | |
$root->appendChild($JDK); | |
my $Key = $doc->createElement("perfTestResult"); | |
my $testname = "Coll-$locale-data$counter-keyGen-JDK"; | |
$Key->setAttribute("test"=>$testname); | |
$Key->setAttribute("iterations"=>1000); | |
$Key->setAttribute("time"=>$wKeyGen); | |
$root->appendChild($Key); | |
$counter++; | |
} | |
} | |
#----------WRITE RESULTS TO perf.xml----------------------- | |
$doc->setDocumentElement($root); | |
open my $out_fh, '>', "perf.xml"; | |
print {$out_fh} $doc->toString; | |
} | |
#--------------------------------------------------------------------- | |
# Append text to the global variable $OUT | |
sub out { | |
$OUT .= join('', @_); | |
} | |
#--------------------------------------------------------------------- | |
# Measure a given test method with a give test pattern using the | |
# global run parameters. | |
# | |
# @param the method to run | |
# @param the pattern defining characters to test | |
# @param if >0 then the number of iterations per pass. If <0 then | |
# (negative of) the number of seconds per pass. | |
# | |
# @return a Dataset object, scaled by iterations per pass and | |
# events per iteration, to give time per event | |
# | |
sub measure2 { | |
my @data = measure1(@_); | |
my $iterPerPass = shift(@data); | |
my $eventPerIter = shift(@data); | |
shift(@data) if (@data > 1); # discard first run | |
my $ds = Dataset->new(@data); | |
$ds->setScale(1.0e-3 / ($iterPerPass * $eventPerIter)); | |
$ds; | |
} | |
#--------------------------------------------------------------------- | |
# Measure a given test method with a give test pattern using the | |
# global run parameters. | |
# | |
# @param the method to run | |
# @param the pattern defining characters to test | |
# @param if >0 then the number of iterations per pass. If <0 then | |
# (negative of) the number of seconds per pass. | |
# | |
# @return array of: | |
# [0] iterations per pass | |
# [1] events per iteration | |
# [2..] ms reported for each pass, in order | |
# | |
sub measure1 { | |
my $testclass = shift; | |
my $method = shift; | |
my $pat = shift; | |
my $iterCount = shift; # actually might be -seconds/pass | |
# is $iterCount actually -seconds/pass? | |
if ($iterCount < 0) { | |
# calibrate: estimate ms/iteration | |
print "Calibrating..."; | |
my @t = callJava($testclass, $method, $pat, -$CALIBRATE, 1); | |
print "done.\n"; | |
my @data = split(/\s+/, $t[0]->[2]); | |
$data[0] *= 1.0e+3; | |
my $timePerIter = 1.0e-3 * $data[0] / $data[1]; | |
# determine iterations/pass | |
$iterCount = int(-$iterCount / $timePerIter + 0.5); | |
} | |
# run passes | |
print "Measuring $iterCount iterations x $NUMPASSES passes..."; | |
my @t = callJava($testclass, $method, $pat, $iterCount, $NUMPASSES); | |
print "done.\n"; | |
my @ms = (); | |
my @b; # scratch | |
for my $a (@t) { | |
# $a->[0]: method name, corresponds to $method | |
# $a->[1]: 'begin' data, == $iterCount | |
# $a->[2]: 'end' data, of the form <ms> <loops> <eventsPerIter> | |
# $a->[3...]: gc messages from JVM during pass | |
@b = split(/\s+/, $a->[2]); | |
push(@ms, $b[0] * 1.0e+3); | |
} | |
my $eventsPerIter = $b[2]; | |
my @ms_str = @ms; | |
$ms_str[0] .= " (discarded)" if (@ms_str > 1); | |
($iterCount, $eventsPerIter, @ms); | |
} | |
#--------------------------------------------------------------------- | |
# Invoke java to run $TESTCLASS, passing it the given parameters. | |
# | |
# @param the method to run | |
# @param the number of iterations, or if negative, the duration | |
# in seconds. If more than on pass is desired, pass in | |
# a string, e.g., "100 100 100". | |
# @param the pattern defining characters to test | |
# | |
# @return an array of results. Each result is an array REF | |
# describing one pass. The array REF contains: | |
# ->[0]: The method name as reported | |
# ->[1]: The params on the '= <meth> begin ...' line | |
# ->[2]: The params on the '= <meth> end ...' line | |
# ->[3..]: GC messages from the JVM, if any | |
# | |
sub callJava { | |
my $testclass = shift; | |
my $method = shift; | |
my $pat = shift; | |
my $n = shift; | |
my $passes = shift; | |
my $n = ($n < 0) ? "-t ".(-$n) : "-i ".$n; | |
my $cmd = "java -classpath $CLASSPATH $testclass $method $n -p $passes -L @$pat[0] \"@$pat[1]\" \"@$pat[2]\" -r $THREADS"; | |
print "[$cmd]\n"; # for debugging | |
open(PIPE, "$cmd|") or die "Can't run \"$cmd\""; | |
my @out; | |
while (<PIPE>) { | |
push(@out, $_); | |
} | |
close(PIPE) or die "Java failed: \"$cmd\""; | |
@out = grep(!/^\#/, @out); # filter out comments | |
#print "[", join("\n", @out), "]\n"; | |
my @results; | |
my $method = ''; | |
my $data = []; | |
foreach (@out) { | |
next unless (/\S/); | |
if (/^=\s*(\w+)\s*(\w+)\s*(.*)/) { | |
my ($m, $state, $d) = ($1, $2, $3); | |
#print "$_ => [[$m $state $data]]\n"; | |
if ($state eq 'begin') { | |
die "$method was begun but not finished" if ($method); | |
$method = $m; | |
push(@$data, $d); | |
push(@$data, ''); # placeholder for end data | |
} elsif ($state eq 'end') { | |
if ($m ne $method) { | |
die "$method end does not match: $_"; | |
} | |
$data->[1] = $d; # insert end data at [1] | |
#print "#$method:", join(";",@$data), "\n"; | |
unshift(@$data, $method); # add method to start | |
push(@results, $data); | |
$method = ''; | |
$data = []; | |
} else { | |
die "Can't parse: $_"; | |
} | |
} | |
elsif (/^\[/) { | |
if ($method) { | |
push(@$data, $_); | |
} else { | |
# ignore extraneous GC notices | |
} | |
} | |
else { | |
die "Can't parse: $_"; | |
} | |
} | |
die "$method was begun but not finished" if ($method); | |
@results; | |
} | |
#----------------------------------------------------------------------------------- | |
# doKeyGenTimes($Command_to_run, $time, $key_length) | |
# Do a key-generation test and return the time and key length/char values. | |
# | |
sub doKeyTimes($$$) { | |
# print "$_[0]\n"; | |
local($x) = `$_[0]`; # execute the collperf command. | |
($_[1], $_[2]) = split(/\,/, $x); # collperf returns "time, keylength" string. | |
} | |
#eof |