| package test.translit; |
| import test.IntlTest; |
| import com.ibm.text.*; |
| import java.text.*; |
| import java.util.*; |
| |
| /** |
| * @test |
| * @summary General test of Transliterator |
| */ |
| public class TransliteratorTest extends IntlTest { |
| |
| public static void main(String[] args) throws Exception { |
| new TransliteratorTest().run(args); |
| } |
| |
| /** |
| * A CommonPoint legacy round-trip test for the Kana transliterator. |
| */ |
| // public void TestKanaRoundTrip() { |
| // Transliterator t = Transliterator.getInstance("Kana"); |
| // StringTokenizer tok = new StringTokenizer(KANA_RT_DATA); |
| // while (tok.hasMoreTokens()) { |
| // String str = tok.nextToken(); |
| // ReplaceableString tmp = new ReplaceableString(str); |
| // t.transliterate(tmp, Transliterator.FORWARD); |
| // |
| // str = tmp.toString(); |
| // tmp = new ReplaceableString(str); |
| // t.transliterate(tmp, Transliterator.REVERSE); |
| // t.transliterate(tmp, Transliterator.FORWARD); |
| // if (!tmp.toString().equals(str)) { |
| // tmp = new ReplaceableString(str); |
| // t.transliterate(tmp, Transliterator.REVERSE); |
| // String a = tmp.toString(); |
| // t.transliterate(tmp, Transliterator.FORWARD); |
| // errln("FAIL: " + escape(str) + " -> " + |
| // escape(a) + " -> " + escape(tmp.toString())); |
| // } |
| // } |
| // } |
| |
| public void TestInstantiation() { |
| long ms = System.currentTimeMillis(); |
| String ID; |
| for (Enumeration e = Transliterator.getAvailableIDs(); e.hasMoreElements(); ) { |
| ID = (String) e.nextElement(); |
| try { |
| Transliterator t = Transliterator.getInstance(ID); |
| // We should get a new instance if we try again |
| Transliterator t2 = Transliterator.getInstance(ID); |
| if (t != t2) { |
| logln(ID + ":" + t); |
| } else { |
| errln("FAIL: " + ID + " returned identical instances"); |
| } |
| } catch (IllegalArgumentException ex) { |
| errln("FAIL: " + ID); |
| throw ex; |
| } |
| } |
| |
| // Now test the failure path |
| try { |
| ID = "<Not a valid Transliterator ID>"; |
| Transliterator t = Transliterator.getInstance(ID); |
| errln("FAIL: " + ID + " returned " + t); |
| } catch (IllegalArgumentException ex) { |
| logln("OK: Bogus ID handled properly"); |
| } |
| |
| ms = System.currentTimeMillis() - ms; |
| logln("Elapsed time: " + ms + " ms"); |
| } |
| |
| public void TestDisplayName() { |
| String ID; |
| for (Enumeration e = Transliterator.getAvailableIDs(); e.hasMoreElements(); ) { |
| ID = (String) e.nextElement(); |
| logln(ID + " -> " + Transliterator.getDisplayName(ID)); |
| } |
| } |
| |
| public void TestSimpleRules() { |
| /* Example: rules 1. ab>x|y |
| * 2. yc>z |
| * |
| * []|eabcd start - no match, copy e to tranlated buffer |
| * [e]|abcd match rule 1 - copy output & adjust cursor |
| * [ex|y]cd match rule 2 - copy output & adjust cursor |
| * [exz]|d no match, copy d to transliterated buffer |
| * [exzd]| done |
| */ |
| expect("ab>x|y;" + |
| "yc>z", |
| "eabcd", "exzd"); |
| |
| /* Another set of rules: |
| * 1. ab>x|yzacw |
| * 2. za>q |
| * 3. qc>r |
| * 4. cw>n |
| * |
| * []|ab Rule 1 |
| * [x|yzacw] No match |
| * [xy|zacw] Rule 2 |
| * [xyq|cw] Rule 4 |
| * [xyqn]| Done |
| */ |
| expect("ab>x|yzacw;" + |
| "za>q;" + |
| "qc>r;" + |
| "cw>n", |
| "ab", "xyqn"); |
| |
| /* Test categories |
| */ |
| Transliterator t = new RuleBasedTransliterator("<ID>", |
| "dummy=\uE100;" + |
| "vowel=[aeiouAEIOU];" + |
| "lu=[:Lu:];" + |
| "{vowel} ({lu}) > !;" + |
| "{vowel} > &;" + |
| "(!) {lu} > ^;" + |
| "{lu} > *;" + |
| "a>ERROR"); |
| expect(t, "abcdefgABCDEFGU", "&bcd&fg!^**!^*&"); |
| } |
| |
| // Restore this test if/when it's been deciphered. In general, |
| // tests that depend on a specific transliterator are subject |
| // to the same fragility as tests that depend on resource data. |
| |
| // public void TestKana() { |
| // String DATA[] = { |
| // "a", "\u3042", |
| // "A", "\u30A2", |
| // "aA", "\u3042\u30A2", |
| // "aaaa", "\u3042\u3042\u3042\u3042", |
| // "akasata", "\u3042\u304B\u3055\u305F", |
| // }; |
| // |
| // Transliterator t = Transliterator.getInstance("Latin-Kana"); |
| // Transliterator rt = Transliterator.getInstance("Kana-Latin"); |
| // for (int i=0; i<DATA.length; i+=2) { |
| // expect(t, DATA[i], DATA[i+1], rt); |
| // } |
| // } |
| |
| /** |
| * Test inline set syntax and set variable syntax. |
| */ |
| public void TestInlineSet() { |
| expect("[:Ll:] (x) > y; [:Ll:] > z;", "aAbxq", "zAyzz"); |
| expect("a[0-9]b > qrs", "1a7b9", "1qrs9"); |
| |
| expect("digit = [0-9];" + |
| "alpha = [a-zA-Z];" + |
| "alphanumeric = [{digit}{alpha}];" + // *** |
| "special = [^{alphanumeric}];" + // *** |
| "{alphanumeric} > -;" + |
| "{special} > *;", |
| |
| "thx-1138", "---*----"); |
| } |
| |
| /** |
| * Create some inverses and confirm that they work. We have to be |
| * careful how we do this, since the inverses will not be true |
| * inverses -- we can't throw any random string at the composition |
| * of the transliterators and expect the identity function. F x |
| * F' != I. However, if we are careful about the input, we will |
| * get the expected results. |
| */ |
| public void TestRuleBasedInverse() { |
| String RULES = |
| "abc>zyx;" + |
| "ab>yz;" + |
| "bc>zx;" + |
| "ca>xy;" + |
| "a>x;" + |
| "b>y;" + |
| "c>z;" + |
| |
| "abc<zyx;" + |
| "ab<yz;" + |
| "bc<zx;" + |
| "ca<xy;" + |
| "a<x;" + |
| "b<y;" + |
| "c<z;" + |
| |
| ""; |
| |
| String[] DATA = { |
| // Careful here -- random strings will not work. If we keep |
| // the left side to the domain and the right side to the range |
| // we will be okay though (left, abc; right xyz). |
| "a", "x", |
| "abcacab", "zyxxxyy", |
| "caccb", "xyzzy", |
| }; |
| |
| Transliterator fwd = new RuleBasedTransliterator("<ID>", RULES); |
| Transliterator rev = new RuleBasedTransliterator("<ID>", RULES, |
| RuleBasedTransliterator.REVERSE, null); |
| for (int i=0; i<DATA.length; i+=2) { |
| expect(fwd, DATA[i], DATA[i+1]); |
| expect(rev, DATA[i+1], DATA[i]); |
| } |
| } |
| |
| /** |
| * Basic test of keyboard. |
| */ |
| public void TestKeyboard() { |
| Transliterator t = new RuleBasedTransliterator("<ID>", |
| "psch>Y;" |
| +"ps>y;" |
| +"ch>x;" |
| +"a>A;"); |
| String DATA[] = { |
| // insertion, buffer |
| "a", "A", |
| "p", "Ap", |
| "s", "Aps", |
| "c", "Apsc", |
| "a", "AycA", |
| "psch", "AycAY", |
| null, "AycAY", // null means finishKeyboardTransliteration |
| }; |
| |
| keyboardAux(t, DATA); |
| } |
| |
| /** |
| * Basic test of keyboard with cursor. |
| */ |
| public void TestKeyboard2() { |
| Transliterator t = new RuleBasedTransliterator("<ID>", |
| "ych>Y;" |
| +"ps>|y;" |
| +"ch>x;" |
| +"a>A;"); |
| String DATA[] = { |
| // insertion, buffer |
| "a", "A", |
| "p", "Ap", |
| "s", "Ay", |
| "c", "Ayc", |
| "a", "AycA", |
| "p", "AycAp", |
| "s", "AycAy", |
| "c", "AycAyc", |
| "h", "AycAY", |
| null, "AycAY", // null means finishKeyboardTransliteration |
| }; |
| |
| keyboardAux(t, DATA); |
| } |
| |
| /** |
| * Test keyboard transliteration with back-replacement. |
| */ |
| public void TestKeyboard3() { |
| // We want th>z but t>y. Furthermore, during keyboard |
| // transliteration we want t>y then yh>z if t, then h are |
| // typed. |
| String RULES = |
| "t>|y;" + |
| "yh>z;" + |
| ""; |
| |
| String[] DATA = { |
| // Column 1: characters to add to buffer (as if typed) |
| // Column 2: expected appearance of buffer after |
| // keyboard xliteration. |
| "a", "a", |
| "b", "ab", |
| "t", "aby", |
| "c", "abyc", |
| "t", "abycy", |
| "h", "abycz", |
| null, "abycz", // null means finishKeyboardTransliteration |
| }; |
| |
| Transliterator t = new RuleBasedTransliterator("<ID>", RULES); |
| keyboardAux(t, DATA); |
| } |
| |
| private void keyboardAux(Transliterator t, String[] DATA) { |
| int[] index = {0, 0, 0}; |
| ReplaceableString s = new ReplaceableString(); |
| for (int i=0; i<DATA.length; i+=2) { |
| StringBuffer log; |
| if (DATA[i] != null) { |
| log = new StringBuffer(s.toString() + " + " |
| + DATA[i] |
| + " -> "); |
| t.keyboardTransliterate(s, index, DATA[i]); |
| } else { |
| log = new StringBuffer(s.toString() + " => "); |
| t.finishKeyboardTransliteration(s, index); |
| } |
| String str = s.toString(); |
| // Show the start index '{' and the cursor '|' |
| log.append(str.substring(0, index[Transliterator.START])). |
| append('{'). |
| append(str.substring(index[Transliterator.START], |
| index[Transliterator.CURSOR])). |
| append('|'). |
| append(str.substring(index[Transliterator.CURSOR])); |
| if (str.equals(DATA[i+1])) { |
| logln(log.toString()); |
| } else { |
| errln("FAIL: " + log.toString() + ", expected " + DATA[i+1]); |
| } |
| } |
| } |
| |
| public void TestArabic() { |
| String DATA[] = { |
| "Arabic", |
| "\u062a\u062a\u0645\u062a\u0639 "+ |
| "\u0627\u0644\u0644\u063a\u0629 "+ |
| "\u0627\u0644\u0639\u0631\u0628\u0628\u064a\u0629 "+ |
| "\u0628\u0628\u0646\u0638\u0645 "+ |
| "\u0643\u062a\u0627\u0628\u0628\u064a\u0629 "+ |
| "\u062c\u0645\u064a\u0644\u0629" |
| }; |
| |
| Transliterator t = Transliterator.getInstance("Latin-Arabic"); |
| for (int i=0; i<DATA.length; i+=2) { |
| expect(t, DATA[i], DATA[i+1]); |
| } |
| } |
| |
| /** |
| * Compose the Kana transliterator forward and reverse and try |
| * some strings that should come out unchanged. |
| */ |
| public void TestCompoundKana() { |
| Transliterator kana = Transliterator.getInstance("Latin-Kana"); |
| Transliterator rkana = Transliterator.getInstance("Kana-Latin"); |
| Transliterator[] trans = { kana, rkana }; |
| Transliterator t = new CompoundTransliterator("<ID>", trans); |
| |
| expect(t, "aaaaa", "aaaaa"); |
| } |
| |
| /** |
| * Compose the hex transliterators forward and reverse. |
| */ |
| public void TestCompoundHex() { |
| Transliterator a = Transliterator.getInstance("Unicode-Hex"); |
| Transliterator b = Transliterator.getInstance("Hex-Unicode"); |
| Transliterator[] trans = { a, b }; |
| Transliterator ab = new CompoundTransliterator("ab", trans); |
| String s = "abcde"; |
| expect(ab, s, s); |
| |
| trans = new Transliterator[] { b, a }; |
| Transliterator ba = new CompoundTransliterator("ba", trans); |
| ReplaceableString str = new ReplaceableString(s); |
| a.transliterate(str); |
| expect(ba, str.toString(), str.toString()); |
| } |
| |
| /** |
| * Do some basic tests of filtering. |
| */ |
| public void TestFiltering() { |
| Transliterator hex = Transliterator.getInstance("Unicode-Hex"); |
| hex.setFilter(new UnicodeFilter() { |
| public boolean isIn(char c) { |
| return c != 'c'; |
| } |
| }); |
| String s = "abcde"; |
| String out = hex.transliterate(s); |
| String exp = "\\u0061\\u0062c\\u0064\\u0065"; |
| if (out.equals(exp)) { |
| logln("Ok: \"" + exp + "\""); |
| } else { |
| logln("FAIL: \"" + out + "\", wanted \"" + exp + "\""); |
| } |
| } |
| |
| /** |
| * Test pattern quoting and escape mechanisms. |
| */ |
| public void TestPatternQuoting() { |
| // Array of 3n items |
| // Each item is <rules>, <input>, <expected output> |
| String[] DATA = { |
| "\u4E01>'[male adult]'", "\u4E01", "[male adult]", |
| }; |
| |
| for (int i=0; i<DATA.length; i+=3) { |
| logln("Pattern: " + escape(DATA[i])); |
| Transliterator t = new RuleBasedTransliterator("<ID>", DATA[i]); |
| expect(t, DATA[i+1], DATA[i+2]); |
| } |
| } |
| |
| //====================================================================== |
| // Support methods |
| //====================================================================== |
| |
| void expect(String rules, String source, String expectedResult) { |
| expect(new RuleBasedTransliterator("<ID>", rules), source, expectedResult); |
| } |
| |
| void expect(Transliterator t, String source, String expectedResult, |
| Transliterator reverseTransliterator) { |
| expect(t, source, expectedResult); |
| if (reverseTransliterator != null) { |
| expect(reverseTransliterator, expectedResult, source); |
| } |
| } |
| |
| void expect(Transliterator t, String source, String expectedResult) { |
| String result = t.transliterate(source); |
| expectAux(t.getID() + ":String", source, result, expectedResult); |
| |
| ReplaceableString rsource = new ReplaceableString(source); |
| t.transliterate(rsource); |
| result = rsource.toString(); |
| expectAux(t.getID() + ":Replaceable", source, result, expectedResult); |
| |
| // Test keyboard (incremental) transliteration -- this result |
| // must be the same after we finalize (see below). |
| rsource.getStringBuffer().setLength(0); |
| int[] index = { 0, 0, 0 }; |
| StringBuffer log = new StringBuffer(); |
| |
| for (int i=0; i<source.length(); ++i) { |
| if (i != 0) { |
| log.append(" + "); |
| } |
| log.append(source.charAt(i)).append(" -> "); |
| t.keyboardTransliterate(rsource, index, |
| String.valueOf(source.charAt(i))); |
| // Append the string buffer with a vertical bar '|' where |
| // the committed index is. |
| String s = rsource.toString(); |
| log.append(s.substring(0, index[Transliterator.CURSOR])). |
| append('|'). |
| append(s.substring(index[Transliterator.CURSOR])); |
| } |
| |
| // As a final step in keyboard transliteration, we must call |
| // transliterate to finish off any pending partial matches that |
| // were waiting for more input. |
| t.finishKeyboardTransliteration(rsource, index); |
| result = rsource.toString(); |
| log.append(" => ").append(rsource.toString()); |
| |
| expectAux(t.getID() + ":Keyboard", log.toString(), |
| result.equals(expectedResult), |
| expectedResult); |
| } |
| |
| void expectAux(String tag, String source, |
| String result, String expectedResult) { |
| expectAux(tag, source + " -> " + result, |
| result.equals(expectedResult), |
| expectedResult); |
| } |
| |
| void expectAux(String tag, String summary, boolean pass, |
| String expectedResult) { |
| if (pass) { |
| logln("("+tag+") " + escape(summary)); |
| } else { |
| errln("FAIL: ("+tag+") " |
| + escape(summary) |
| + ", expected " + escape(expectedResult)); |
| } |
| } |
| |
| /** |
| * Escape non-ASCII characters as Unicode. |
| */ |
| public static final String escape(String s) { |
| StringBuffer buf = new StringBuffer(); |
| for (int i=0; i<s.length(); ++i) { |
| char c = s.charAt(i); |
| if (c >= ' ' && c <= 0x007F) { |
| buf.append(c); |
| } else { |
| buf.append("\\u"); |
| if (c < 0x1000) { |
| buf.append('0'); |
| if (c < 0x100) { |
| buf.append('0'); |
| if (c < 0x10) { |
| buf.append('0'); |
| } |
| } |
| } |
| buf.append(Integer.toHexString(c)); |
| } |
| } |
| return buf.toString(); |
| } |
| } |