| /* |
| ********************************************************************** |
| * Copyright (C) 1999-2003, International Business Machines |
| * Corporation and others. All Rights Reserved. |
| ********************************************************************** |
| * Date Name Description |
| * 12/09/99 aliu Ported from Java. |
| ********************************************************************** |
| */ |
| |
| #include "unicode/utypes.h" |
| |
| #if !UCONFIG_NO_COLLATION |
| |
| #include "thcoll.h" |
| #include "unicode/utypes.h" |
| #include "unicode/coll.h" |
| #include "unicode/sortkey.h" |
| #include "unicode/ustring.h" |
| #include "cstring.h" |
| #include "filestrm.h" |
| |
| /** |
| * The TestDictionary test expects a file of this name, with this |
| * encoding, to be present in the directory $ICU/source/test/testdata. |
| */ |
| //#define TEST_FILE "th18057.txt" |
| #define TEST_FILE "riwords.txt" |
| #define TEST_FILE_ENCODING "UTF8" |
| |
| /** |
| * This is the most failures we show in TestDictionary. If this number |
| * is < 0, we show all failures. |
| */ |
| #define MAX_FAILURES_TO_SHOW -1 |
| |
| #define CASE(id,test) \ |
| case id: \ |
| name = #test; \ |
| if (exec) { \ |
| logln(#test "---"); \ |
| logln((UnicodeString)""); \ |
| test(); \ |
| } \ |
| break; |
| |
| CollationThaiTest::CollationThaiTest() { |
| UErrorCode status = U_ZERO_ERROR; |
| coll = Collator::createInstance(Locale("th", "TH", ""), status); |
| if (coll && U_SUCCESS(status)) { |
| //coll->setStrength(Collator::TERTIARY); |
| } else { |
| delete coll; |
| coll = 0; |
| } |
| } |
| |
| CollationThaiTest::~CollationThaiTest() { |
| delete coll; |
| } |
| |
| void CollationThaiTest::runIndexedTest(int32_t index, UBool exec, const char* &name, |
| char* /*par*/) { |
| |
| if((!coll) && exec) { |
| errln(__FILE__ " cannot test - failed to create collator."); |
| name = ""; |
| return; |
| } |
| |
| switch (index) { |
| CASE(0,TestDictionary) |
| CASE(1,TestCornerCases) |
| CASE(2,TestNamesList) |
| CASE(3,TestInvalidThai) |
| CASE(4,TestReordering) |
| default: name = ""; break; |
| } |
| } |
| |
| /** |
| * Read a line terminated by a single ^J or ^M, and convert it from |
| * the TEST_FILE_ENCODING to Unicode. ASSUMES FILE LINES ARE 127 |
| * characters long or less. This is true for th18057.txt, which |
| * has 80-char or shorter lines. DOES NOT HANDLE ^M^J sequence. |
| */ |
| static UBool readLine(FileStream *in, UnicodeString& line, const char* encoding) { |
| if (T_FileStream_eof(in)) { |
| return FALSE; |
| } |
| char buffer[1024]; |
| char* p = buffer; |
| char* limit = p + sizeof(buffer) - 1; // Leave space for 0 |
| while (p<limit) { |
| int c = T_FileStream_getc(in); |
| if (c < 0 || c == 0xD || c == 0xA) { |
| break; |
| } |
| *p++ = c; |
| } |
| *p = 0; |
| line = UnicodeString(buffer, encoding); |
| return TRUE; |
| } |
| |
| |
| /** |
| * Read the external names list, and confirms that the collator |
| * gets the same results when comparing lines one to another |
| * using regular and iterative comparison. |
| */ |
| void CollationThaiTest::TestNamesList(void) { |
| if (coll == 0) { |
| errln("Error: could not construct Thai collator"); |
| return; |
| } |
| |
| // Read in a dictionary of Thai words |
| UErrorCode status = U_ZERO_ERROR; |
| char buffer[1024]; |
| uprv_strcpy(buffer,IntlTest::loadTestData(status) ); |
| char* index = 0; |
| |
| if (U_FAILURE(status)) { |
| errln("ERROR: could not open test data %s", u_errorName(status)); |
| return; |
| } |
| index=strrchr(buffer,(char)U_FILE_SEP_CHAR); |
| |
| if((unsigned int)(index-buffer) != (strlen(buffer)-1)){ |
| *(index+1)=0; |
| } |
| uprv_strcat(buffer,".."U_FILE_SEP_STRING); |
| uprv_strcat(buffer, "TestNames_Thai.txt"); |
| |
| FileStream *in = T_FileStream_open(buffer, "rb"); |
| if (in == 0) { |
| logln((UnicodeString)"Could not find file: " + buffer +" will not do this test"); |
| return; |
| } |
| |
| // |
| // Loop through each word in the dictionary and compare it to the previous |
| // word. They should be in sorted order. |
| // |
| UnicodeString lastWord, word; |
| int32_t line = 0; |
| //int32_t failed = 0; |
| int32_t wordCount = 0; |
| while (readLine(in, word, "UTF16LE")) { |
| line++; |
| |
| // Skip comments and blank lines |
| if (word.charAt(0) == 0x23 || word.length() == 0) { |
| continue; |
| } |
| |
| // Show the first 8 words being compared, so we can see what's happening |
| ++wordCount; |
| if (wordCount <= 8) { |
| UnicodeString str; |
| logln((UnicodeString)"Word " + wordCount + ": " + IntlTest::prettify(word, str)); |
| } |
| |
| if (lastWord.length() > 0) { |
| Collator::EComparisonResult result = coll->compare(lastWord, word); |
| doTest(coll, lastWord, word, result); |
| } |
| lastWord = word; |
| } |
| |
| |
| logln((UnicodeString)"Words checked: " + wordCount); |
| T_FileStream_close(in); |
| } |
| |
| /** |
| * Read the external dictionary file, which is already in proper |
| * sorted order, and confirm that the collator compares each line as |
| * preceding the following line. |
| */ |
| void CollationThaiTest::TestDictionary(void) { |
| if (coll == 0) { |
| errln("Error: could not construct Thai collator"); |
| return; |
| } |
| |
| // Read in a dictionary of Thai words |
| UErrorCode status = U_ZERO_ERROR; |
| char buffer[1024]; |
| uprv_strcpy(buffer,IntlTest::loadTestData(status) ); |
| char* index = 0; |
| |
| if (U_FAILURE(status)) { |
| errln("ERROR: could not open test data %s", u_errorName(status)); |
| return; |
| } |
| index=strrchr(buffer,(char)U_FILE_SEP_CHAR); |
| |
| if((unsigned int)(index-buffer) != (strlen(buffer)-1)){ |
| *(index+1)=0; |
| } |
| uprv_strcat(buffer,".."U_FILE_SEP_STRING); |
| uprv_strcat(buffer, TEST_FILE); |
| |
| FileStream *in = T_FileStream_open(buffer, "rb"); |
| if (in == 0) { |
| infoln((UnicodeString)"INFO: could not open test file " + buffer + ". Aborting test."); |
| return; |
| } |
| |
| // |
| // Loop through each word in the dictionary and compare it to the previous |
| // word. They should be in sorted order. |
| // |
| UnicodeString lastWord, word; |
| int32_t line = 0; |
| int32_t failed = 0; |
| int32_t wordCount = 0; |
| while (readLine(in, word, "UTF8")) { |
| line++; |
| |
| // Skip comments and blank lines |
| if (word.charAt(0) == 0x23 || word.length() == 0) { |
| continue; |
| } |
| |
| // Show the first 8 words being compared, so we can see what's happening |
| ++wordCount; |
| if (wordCount <= 8) { |
| UnicodeString str; |
| logln((UnicodeString)"Word " + wordCount + ": " + IntlTest::prettify(word, str)); |
| } |
| |
| if (lastWord.length() > 0) { |
| // line enabled for j2720 |
| doTest(coll, lastWord, word, Collator::LESS); |
| int32_t result = coll->compare(lastWord, word); |
| |
| if (result >= 0) { |
| failed++; |
| if (MAX_FAILURES_TO_SHOW < 0 || failed <= MAX_FAILURES_TO_SHOW) { |
| UnicodeString str; |
| UnicodeString msg = |
| UnicodeString("--------------------------------------------\n") |
| + line |
| + " compare(" + IntlTest::prettify(lastWord, str); |
| msg += UnicodeString(", ") |
| + IntlTest::prettify(word, str) + ") returned " + result |
| + ", expected -1\n"; |
| UErrorCode status = U_ZERO_ERROR; |
| CollationKey k1, k2; |
| coll->getCollationKey(lastWord, k1, status); |
| coll->getCollationKey(word, k2, status); |
| if (U_FAILURE(status)) { |
| errln((UnicodeString)"Fail: getCollationKey returned " + u_errorName(status)); |
| return; |
| } |
| msg.append("key1: ").append(prettify(k1, str)).append("\n"); |
| msg.append("key2: ").append(prettify(k2, str)); |
| errln(msg); |
| } |
| } |
| } |
| lastWord = word; |
| } |
| |
| if (failed != 0) { |
| if (failed > MAX_FAILURES_TO_SHOW) { |
| errln((UnicodeString)"Too many failures; only the first " + |
| MAX_FAILURES_TO_SHOW + " failures were shown"); |
| } |
| errln((UnicodeString)"Summary: " + failed + " of " + (line - 1) + |
| " comparisons failed"); |
| } |
| |
| logln((UnicodeString)"Words checked: " + wordCount); |
| T_FileStream_close(in); |
| } |
| |
| /** |
| * Odd corner conditions taken from "How to Sort Thai Without Rewriting Sort", |
| * by Doug Cooper, http://seasrc.th.net/paper/thaisort.zip |
| */ |
| void CollationThaiTest::TestCornerCases(void) { |
| const char* TESTS[] = { |
| // Shorter words precede longer |
| "\\u0e01", "<", "\\u0e01\\u0e01", |
| |
| // Tone marks are considered after letters (i.e. are primary ignorable) |
| "\\u0e01\\u0e32", "<", "\\u0e01\\u0e49\\u0e32", |
| |
| // ditto for other over-marks |
| "\\u0e01\\u0e32", "<", "\\u0e01\\u0e32\\u0e4c", |
| |
| // commonly used mark-in-context order. |
| // In effect, marks are sorted after each syllable. |
| "\\u0e01\\u0e32\\u0e01\\u0e49\\u0e32", "<", "\\u0e01\\u0e48\\u0e32\\u0e01\\u0e49\\u0e32", |
| |
| // Hyphens and other punctuation follow whitespace but come before letters |
| "\\u0e01\\u0e32", "<", "\\u0e01\\u0e32-", |
| "\\u0e01\\u0e32-", "<", "\\u0e01\\u0e32\\u0e01\\u0e32", |
| |
| // Doubler follows an indentical word without the doubler |
| "\\u0e01\\u0e32", "<", "\\u0e01\\u0e32\\u0e46", |
| "\\u0e01\\u0e32\\u0e46", "<", "\\u0e01\\u0e32\\u0e01\\u0e32", |
| |
| |
| // \\u0e45 after either \\u0e24 or \\u0e26 is treated as a single |
| // combining character, similar to "c < ch" in traditional spanish. |
| // TODO: beef up this case |
| "\\u0e24\\u0e29\\u0e35", "<", "\\u0e24\\u0e45\\u0e29\\u0e35", |
| "\\u0e26\\u0e29\\u0e35", "<", "\\u0e26\\u0e45\\u0e29\\u0e35", |
| |
| // Vowels reorder, should compare \\u0e2d and \\u0e34 |
| "\\u0e40\\u0e01\\u0e2d", "<", "\\u0e40\\u0e01\\u0e34", |
| |
| // Tones are compared after the rest of the word (e.g. primary ignorable) |
| "\\u0e01\\u0e32\\u0e01\\u0e48\\u0e32", "<", "\\u0e01\\u0e49\\u0e32\\u0e01\\u0e32", |
| |
| // Periods are ignored entirely |
| "\\u0e01.\\u0e01.", "<", "\\u0e01\\u0e32", |
| }; |
| const int32_t TESTS_length = (int32_t)(sizeof(TESTS)/sizeof(TESTS[0])); |
| |
| if (coll == 0) { |
| errln("Error: could not construct Thai collator"); |
| return; |
| } |
| compareArray(*coll, TESTS, TESTS_length); |
| } |
| |
| //------------------------------------------------------------------------ |
| // Internal utilities |
| //------------------------------------------------------------------------ |
| |
| void CollationThaiTest::compareArray(Collator& c, const char* tests[], |
| int32_t testsLength) { |
| UErrorCode status = U_ZERO_ERROR; |
| for (int32_t i = 0; i < testsLength; i += 3) { |
| |
| Collator::EComparisonResult expect; |
| if (tests[i+1][0] == '<') { |
| expect = Collator::LESS; |
| } else if (tests[i+1][0] == '>') { |
| expect = Collator::GREATER; |
| } else if (tests[i+1][0] == '=') { |
| expect = Collator::EQUAL; |
| } else { |
| // expect = Integer.decode(tests[i+1]).intValue(); |
| errln((UnicodeString)"Error: unknown operator " + tests[i+1]); |
| return; |
| } |
| |
| UnicodeString s1, s2; |
| parseChars(s1, tests[i]); |
| parseChars(s2, tests[i+2]); |
| |
| doTest(&c, s1, s2, expect); |
| #if 0 |
| int32_t result = c.compare(s1, s2); |
| if (sign(result) != sign(expect)) |
| { |
| UnicodeString t1, t2; |
| errln(UnicodeString("") + |
| i/3 + ": compare(" + IntlTest::prettify(s1, t1) |
| + " , " + IntlTest::prettify(s2, t2) |
| + ") got " + result + "; expected " + expect); |
| |
| CollationKey k1, k2; |
| c.getCollationKey(s1, k1, status); |
| c.getCollationKey(s2, k2, status); |
| if (U_FAILURE(status)) { |
| errln((UnicodeString)"Fail: getCollationKey returned " + u_errorName(status)); |
| return; |
| } |
| errln((UnicodeString)" key1: " + prettify(k1, t1) ); |
| errln((UnicodeString)" key2: " + prettify(k2, t2) ); |
| } |
| else |
| { |
| // Collator.compare worked OK; now try the collation keys |
| CollationKey k1, k2; |
| c.getCollationKey(s1, k1, status); |
| c.getCollationKey(s2, k2, status); |
| if (U_FAILURE(status)) { |
| errln((UnicodeString)"Fail: getCollationKey returned " + u_errorName(status)); |
| return; |
| } |
| |
| result = k1.compareTo(k2); |
| if (sign(result) != sign(expect)) { |
| UnicodeString t1, t2; |
| errln(UnicodeString("") + |
| i/3 + ": key(" + IntlTest::prettify(s1, t1) |
| + ").compareTo(key(" + IntlTest::prettify(s2, t2) |
| + ")) got " + result + "; expected " + expect); |
| |
| errln((UnicodeString)" " + prettify(k1, t1) + " vs. " + prettify(k2, t2)); |
| } |
| } |
| #endif |
| } |
| } |
| |
| int8_t CollationThaiTest::sign(int32_t i) { |
| if (i < 0) return -1; |
| if (i > 0) return 1; |
| return 0; |
| } |
| |
| /** |
| * Set a UnicodeString corresponding to the given string. Use |
| * UnicodeString and the default converter, unless we see the sequence |
| * "\\u", in which case we interpret the subsequent escape. |
| */ |
| UnicodeString& CollationThaiTest::parseChars(UnicodeString& result, |
| const char* chars) { |
| return result = CharsToUnicodeString(chars); |
| } |
| |
| UCollator *thaiColl = NULL; |
| |
| U_CDECL_BEGIN |
| static int U_CALLCONV |
| StrCmp(const void *p1, const void *p2) { |
| return ucol_strcoll(thaiColl, *(UChar **) p1, -1, *(UChar **)p2, -1); |
| } |
| U_CDECL_END |
| |
| |
| #define LINES 6 |
| |
| void CollationThaiTest::TestInvalidThai(void) { |
| const char *tests[LINES] = { |
| "\\u0E44\\u0E01\\u0E44\\u0E01", |
| "\\u0E44\\u0E01\\u0E01\\u0E44", |
| "\\u0E01\\u0E44\\u0E01\\u0E44", |
| "\\u0E01\\u0E01\\u0E44\\u0E44", |
| "\\u0E44\\u0E44\\u0E01\\u0E01", |
| "\\u0E01\\u0E44\\u0E44\\u0E01", |
| }; |
| |
| UChar strings[LINES][20]; |
| |
| UChar *toSort[LINES]; |
| |
| int32_t i = 0, j = 0, len = 0; |
| |
| UErrorCode coll_status = U_ZERO_ERROR; |
| UnicodeString iteratorText; |
| |
| thaiColl = ucol_open ("th_TH", &coll_status); |
| if (U_FAILURE(coll_status)) { |
| errln("Error opening Thai collator: %s", u_errorName(coll_status)); |
| return; |
| } |
| |
| CollationElementIterator* c = ((RuleBasedCollator *)coll)->createCollationElementIterator( iteratorText ); |
| |
| for(i = 0; i < sizeof(tests)/sizeof(tests[0]); i++) { |
| len = u_unescape(tests[i], strings[i], 20); |
| strings[i][len] = 0; |
| toSort[i] = strings[i]; |
| } |
| |
| qsort (toSort, LINES, sizeof (UChar *), StrCmp); |
| |
| for (i=0; i < LINES; i++) |
| { |
| logln("%i", i); |
| for (j=i+1; j < LINES; j++) { |
| if (ucol_strcoll (thaiColl, toSort[i], -1, toSort[j], -1) == UCOL_GREATER) |
| { |
| // inconsistency ordering found! |
| errln("Inconsistent ordering between strings %i and %i", i, j); |
| } |
| } |
| iteratorText.setTo(toSort[i]); |
| c->setText(iteratorText, coll_status); |
| backAndForth(*c); |
| } |
| |
| |
| ucol_close(thaiColl); |
| delete c; |
| } |
| |
| void CollationThaiTest::TestReordering(void) { |
| const char *tests[] = { |
| "\\u0E41c\\u0301", "=", "\\u0E41\\u0107", // composition |
| "\\u0E41\\uD835\\uDFCE", "<", "\\u0E41\\uD835\\uDFCF", // supplementaries |
| "\\u0E41\\uD834\\uDD5F", "=", "\\u0E41\\uD834\\uDD58\\uD834\\uDD65", // supplementary composition decomps to supplementary |
| "\\u0E41\\uD87E\\uDC02", "=", "\\u0E41\\u4E41", // supplementary composition decomps to BMP |
| "\\u0E41\\u0301", "=", "\\u0E41\\u0301", // unsafe (just checking backwards iteration) |
| "\\u0E41\\u0301\\u0316", "=", "\\u0E41\\u0316\\u0301", |
| "\\u0e24\\u0e41", "=", "\\u0e41\\u0e24", // exiting contraction bug |
| "\\u0e3f\\u0e3f\\u0e24\\u0e41", "=", "\\u0e3f\\u0e3f\\u0e41\\u0e24", |
| |
| "abc\\u0E41c\\u0301", "=", "abc\\u0E41\\u0107", // composition |
| "abc\\u0E41\\uD834\\uDC00", "<", "abc\\u0E41\\uD834\\uDC01", // supplementaries |
| "abc\\u0E41\\uD834\\uDD5F", "=", "abc\\u0E41\\uD834\\uDD58\\uD834\\uDD65", // supplementary composition decomps to supplementary |
| "abc\\u0E41\\uD87E\\uDC02", "=", "abc\\u0E41\\u4E41", // supplementary composition decomps to BMP |
| "abc\\u0E41\\u0301", "=", "abc\\u0E41\\u0301", // unsafe (just checking backwards iteration) |
| "abc\\u0E41\\u0301\\u0316", "=", "abc\\u0E41\\u0316\\u0301", |
| |
| "\\u0E41c\\u0301abc", "=", "\\u0E41\\u0107abc", // composition |
| "\\u0E41\\uD834\\uDC00abc", "<", "\\u0E41\\uD834\\uDC01abc", // supplementaries |
| "\\u0E41\\uD834\\uDD5Fabc", "=", "\\u0E41\\uD834\\uDD58\\uD834\\uDD65abc", // supplementary composition decomps to supplementary |
| "\\u0E41\\uD87E\\uDC02abc", "=", "\\u0E41\\u4E41abc", // supplementary composition decomps to BMP |
| "\\u0E41\\u0301abc", "=", "\\u0E41\\u0301abc", // unsafe (just checking backwards iteration) |
| "\\u0E41\\u0301\\u0316abc", "=", "\\u0E41\\u0316\\u0301abc", |
| |
| "abc\\u0E41c\\u0301abc", "=", "abc\\u0E41\\u0107abc", // composition |
| "abc\\u0E41\\uD834\\uDC00abc", "<", "abc\\u0E41\\uD834\\uDC01abc", // supplementaries |
| "abc\\u0E41\\uD834\\uDD5Fabc", "=", "abc\\u0E41\\uD834\\uDD58\\uD834\\uDD65abc", // supplementary composition decomps to supplementary |
| "abc\\u0E41\\uD87E\\uDC02abc", "=", "abc\\u0E41\\u4E41abc", // supplementary composition decomps to BMP |
| "abc\\u0E41\\u0301abc", "=", "abc\\u0E41\\u0301abc", // unsafe (just checking backwards iteration) |
| "abc\\u0E41\\u0301\\u0316abc", "=", "abc\\u0E41\\u0316\\u0301abc", |
| }; |
| |
| compareArray(*coll, tests, sizeof(tests)/sizeof(tests[0])); |
| |
| const char *rule = "& c < ab"; |
| const char *testcontraction[] = { "\\u0E41ab", "<", "\\u0E41c"}; |
| UnicodeString rules; |
| UErrorCode status = U_ZERO_ERROR; |
| parseChars(rules, rule); |
| RuleBasedCollator *rcoll = new RuleBasedCollator(rules, status); |
| if(U_SUCCESS(status)) { |
| compareArray(*rcoll, testcontraction, 3); |
| delete rcoll; |
| } else { |
| errln("Couldn't instantiate collator from rules"); |
| } |
| |
| } |
| |
| |
| #endif /* #if !UCONFIG_NO_COLLATION */ |
