Google Git
Sign in
skia / external / github.com / unicode-org / icu / refs/tags/icu-release-2-6-d01 / . / source / test / intltest / rbbitst.cpp
blob: 3bcc6ad8dae6bd066109315e8f28de8239550682 [file] [log] [blame]
/********************************************************************
* COPYRIGHT:
* Copyright (c) 1999-2003, International Business Machines Corporation and
* others. All Rights Reserved.
********************************************************************/
/************************************************************************
* Date Name Description
* 12/15/99 Madhu Creation.
* 01/12/2000 Madhu Updated for changed API and added new tests
************************************************************************/
#include "unicode/utypes.h"
#if !UCONFIG_NO_BREAK_ITERATION
#include "unicode/utypes.h"
#include "unicode/brkiter.h"
#include "unicode/rbbi.h"
#include "unicode/uchar.h"
#include "unicode/utf16.h"
#include "unicode/ucnv.h"
#include "unicode/schriter.h"
#include "unicode/regex.h"
#include "intltest.h"
#include "rbbitst.h"
#include <string.h>
#include "uvector.h"
#include "uvectr32.h"
#include "charstr.h"
#include <stdio.h>
//---------------------------------------------------------------------------
//
// class BITestData Holds a set of Break iterator test data and results
// Includes
// - the string data to be broken
// - a vector of the expected break positions.
// - a vector of source line numbers for the data,
// (to help see where errors occured.)
// - The expected break tag values.
// - Vectors of actual break positions and tag values.
// - Functions for comparing actual with expected and
// reporting errors.
//
//----------------------------------------------------------------------------
class BITestData {
public:
UnicodeString fDataToBreak;
UVector fExpectedBreakPositions;
UVector fExpectedTags;
UVector fLineNum;
UVector fActualBreakPositions; // Test Results.
UVector fActualTags;
BITestData(UErrorCode &status);
void addDataChunk(const char *data, int32_t tag, int32_t lineNum, UErrorCode status);
void checkResults(const char *heading, RBBITest *test);
void err(const char *heading, RBBITest *test, int32_t expectedIdx, int32_t actualIdx);
void clearResults();
};
//
// Constructor.
//
BITestData::BITestData(UErrorCode &status)
: fExpectedBreakPositions(status), fExpectedTags(status), fLineNum(status), fActualBreakPositions(status),
fActualTags(status)
{
};
//
// addDataChunk. Add a section (non-breaking) piece if data to the test data.
// The macro form collects the line number, which is helpful
// when tracking down failures.
//
// A null data item is inserted at the start of each test's data
// to put the starting zero into the data list. The position saved for
// each non-null item is its ending position.
//
#define ADD_DATACHUNK(td, data, tag, status) td.addDataChunk(data, tag, __LINE__, status);
void BITestData::addDataChunk(const char *data, int32_t tag, int32_t lineNum, UErrorCode status) {
if (U_FAILURE(status)) {return;}
if (data != NULL) {
fDataToBreak.append(CharsToUnicodeString(data));
}
fExpectedBreakPositions.addElement(fDataToBreak.length(), status);
fExpectedTags.addElement(tag, status);
fLineNum.addElement(lineNum, status);
};
//
// checkResults. Compare the actual and expected break positions, report any differences.
//
void BITestData::checkResults(const char *heading, RBBITest *test) {
int32_t expectedIndex = 0;
int32_t actualIndex = 0;
for (;;) {
// If we've run through both the expected and actual results vectors, we're done.
// break out of the loop.
if (expectedIndex >= fExpectedBreakPositions.size() &&
actualIndex >= fActualBreakPositions.size()) {
break;
}
if (expectedIndex >= fExpectedBreakPositions.size()) {
err(heading, test, expectedIndex-1, actualIndex);
actualIndex++;
continue;
}
if (actualIndex >= fActualBreakPositions.size()) {
err(heading, test, expectedIndex, actualIndex-1);
expectedIndex++;
continue;
}
if (fActualBreakPositions.elementAti(actualIndex) != fExpectedBreakPositions.elementAti(expectedIndex)) {
err(heading, test, expectedIndex, actualIndex);
// Try to resync the positions of the indices, to avoid a rash of spurious erros.
if (fActualBreakPositions.elementAti(actualIndex) < fExpectedBreakPositions.elementAti(expectedIndex)) {
actualIndex++;
} else {
expectedIndex++;
}
continue;
}
if (fActualTags.elementAti(actualIndex) != fExpectedTags.elementAti(expectedIndex)) {
test->errln("%s, tag mismatch. Test Line = %d, expected tag=%d, got %d",
heading, fLineNum.elementAt(expectedIndex),
fExpectedTags.elementAti(expectedIndex), fActualTags.elementAti(actualIndex));
}
actualIndex++;
expectedIndex++;
}
}
//
// err - An error was found. Report it, along with information about where the
// incorrectly broken test data appeared in the source file.
//
void BITestData::err(const char *heading, RBBITest *test, int32_t expectedIdx, int32_t actualIdx)
{
int32_t expected = fExpectedBreakPositions.elementAti(expectedIdx);
int32_t actual = fActualBreakPositions.elementAti(actualIdx);
int32_t o = 0;
int32_t line = fLineNum.elementAti(expectedIdx);
if (expectedIdx > 0) {
// The line numbers are off by one because a premature break occurs somewhere
// within the previous item, rather than at the start of the current (expected) item.
// We want to report the offset of the unexpected break from the start of
// this previous item.
o = actual - fExpectedBreakPositions.elementAti(expectedIdx-1);
}
if (actual < expected) {
test->errln("%s unexpected break at offset %d in test item from line %d", heading, o, line);
} else {
test->errln("%s Failed to find break at end of item from line %d", heading, line);
}
}
void BITestData::clearResults() {
fActualBreakPositions.removeAllElements();
fActualTags.removeAllElements();
}
//-----------------------------------------------------------------------------------
//
// Cannned Test Characters
//
//-----------------------------------------------------------------------------------
static const UChar cannedTestArray[] = {
0x0001, 0x0002, 0x0003, 0x0004, 0x0020, 0x0021, '\\', 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0028, 0x0029, 0x002b, 0x002d, 0x0030, 0x0031,
0x0032, 0x0033, 0x0034, 0x003c, 0x003d, 0x003e, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x005b, 0x005d, 0x005e, 0x005f, 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x007b,
0x007d, 0x007c, 0x002c, 0x00a0, 0x00a2,
0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00ab, 0x00ad, 0x00ae, 0x00af, 0x00b0, 0x00b2, 0x00b3,
0x00b4, 0x00b9, 0x00bb, 0x00bc, 0x00bd, 0x02b0, 0x02b1, 0x02b2, 0x02b3, 0x02b4, 0x0300, 0x0301, 0x0302, 0x0303,
0x0304, 0x05d0, 0x05d1, 0x05d2, 0x05d3, 0x05d4, 0x0903, 0x093e, 0x093f, 0x0940, 0x0949, 0x0f3a, 0x0f3b, 0x2000,
0x2001, 0x2002, 0x200c, 0x200d, 0x200e, 0x200f, 0x2010, 0x2011, 0x2012, 0x2028, 0x2029, 0x202a, 0x203e, 0x203f,
0x2040, 0x20dd, 0x20de, 0x20df, 0x20e0, 0x2160, 0x2161, 0x2162, 0x2163, 0x2164, 0x0000
};
static UnicodeString* cannedTestChars = 0;
#define halfNA "\\u0928\\u094d\\u200d"
#define halfSA "\\u0938\\u094d\\u200d"
#define halfCHA "\\u091a\\u094d\\u200d"
#define halfKA "\\u0915\\u094d\\u200d"
#define deadTA "\\u0924\\u094d"
//--------------------------------------------------------------------------------------
//
// RBBITest constructor and destructor
//
//--------------------------------------------------------------------------------------
RBBITest::RBBITest() {
UnicodeString temp(cannedTestArray);
cannedTestChars = new UnicodeString();
*cannedTestChars += (UChar)0x0000;
*cannedTestChars += temp;
}
RBBITest::~RBBITest() {
delete cannedTestChars;
}
//--------------------------------------------------------------------
//tests default rules based character iteration
//--------------------------------------------------------------------
void RBBITest::TestDefaultRuleBasedCharacterIteration()
{
// RuleBasedBreakIterator* rbbi=(RuleBasedBreakIterator*)RuleBasedBreakIterator::createCharacterInstance();
logln((UnicodeString)"Testing the RBBI for character iteration by using default rules");
//fetch the rules used to create the above RuleBasedBreakIterator
// UnicodeString defaultRules=rbbi->getRules();
// RuleBasedCharacterIterator charIterDefault = new RuleBasedBreakIterator(defaultRules);
UErrorCode status=U_ZERO_ERROR;
RuleBasedBreakIterator* charIterDefault=(RuleBasedBreakIterator*)RuleBasedBreakIterator::createCharacterInstance(Locale::getDefault(), status);
if(U_FAILURE(status)){
errln("FAIL : in construction");
return;
}
BITestData chardata(status);
ADD_DATACHUNK(chardata, NULL, 0, status); // Starting break
ADD_DATACHUNK(chardata, "H", 0, status);
ADD_DATACHUNK(chardata, "e", 0, status);
ADD_DATACHUNK(chardata, "l", 0, status);
ADD_DATACHUNK(chardata, "l", 0, status);
ADD_DATACHUNK(chardata, "o", 0, status);
ADD_DATACHUNK(chardata, "e\\u0301", 0, status); //acuteE
ADD_DATACHUNK(chardata, "&", 0, status);
ADD_DATACHUNK(chardata, "e\\u0303", 0, status); //tildaE
ADD_DATACHUNK(chardata, "S\\u0300", 0, status); //graveS
ADD_DATACHUNK(chardata, "i\\u0301", 0, status); // acuteBelowI
ADD_DATACHUNK(chardata, "m", 0, status);
ADD_DATACHUNK(chardata, "p", 0, status);
ADD_DATACHUNK(chardata, "l", 0, status);
ADD_DATACHUNK(chardata, "e\\u0301", 0, status); // acuteE
ADD_DATACHUNK(chardata, " ", 0, status);
ADD_DATACHUNK(chardata, "s", 0, status);
ADD_DATACHUNK(chardata, "a\\u0302", 0, status); // circumflexA
ADD_DATACHUNK(chardata, "m", 0, status);
ADD_DATACHUNK(chardata, "p", 0, status);
ADD_DATACHUNK(chardata, "l", 0, status);
ADD_DATACHUNK(chardata, "e\\u0303", 0, status); // tildeE
ADD_DATACHUNK(chardata, ".", 0, status);
ADD_DATACHUNK(chardata, "w", 0, status);
ADD_DATACHUNK(chardata, "a\\u0302", 0, status); // circumflexA
ADD_DATACHUNK(chardata, "w", 0, status);
ADD_DATACHUNK(chardata, "a", 0, status);
ADD_DATACHUNK(chardata, "f", 0, status);
ADD_DATACHUNK(chardata, "q", 0, status);
ADD_DATACHUNK(chardata, "\n", 0, status);
ADD_DATACHUNK(chardata, "\r", 0, status);
ADD_DATACHUNK(chardata, "\r\n", 0, status);
ADD_DATACHUNK(chardata, "\n", 0, status);
//devanagiri characters for Hindi support
ADD_DATACHUNK(chardata, "\\u0906", 0, status); //devanagiri AA
//ADD_DATACHUNK(chardata, "\\u093e\\u0901", 0); //devanagiri vowelsign AA+ chandrabindhu
ADD_DATACHUNK(chardata, "\\u0906\\u0901", 0, status); // Devanagari AA + chandrabindu
ADD_DATACHUNK(chardata, "\\u0915", 0, status); // Devanagari KA + AA vowelsign + chandrabindu
ADD_DATACHUNK(chardata, "\\u093e\\u0901", 0, status); // Devanagari KA + AA vowelsign + chandrabindu
ADD_DATACHUNK(chardata, "\\u0916\\u0947", 0, status); //devanagiri KHA+vowelsign E
ADD_DATACHUNK(chardata, "\\u0938\\u0941\\u0902", 0, status); //devanagiri SA+vowelsign U + anusvara(bindu)
ADD_DATACHUNK(chardata, "\\u0926", 0, status); //devanagiri consonant DA
ADD_DATACHUNK(chardata, "\\u0930", 0, status); //devanagiri consonant RA
ADD_DATACHUNK(chardata, "\\u0939", 0, status); //devanagiri HA+vowel sign AI
ADD_DATACHUNK(chardata, "\\u094c", 0, status); //devanagiri HA+vowel sign AI
ADD_DATACHUNK(chardata, "\\u0964", 0, status); //devanagiri danda
//end hindi characters
ADD_DATACHUNK(chardata, "A\\u0302", 0, status); //circumflexA
ADD_DATACHUNK(chardata, "i\\u0301", 0, status); //acuteBelowI
// conjoining jamo->..
ADD_DATACHUNK(chardata, "\\u1109\\u1161\\u11bc", 0, status);
ADD_DATACHUNK(chardata, "\\u1112\\u1161\\u11bc", 0, status);
ADD_DATACHUNK(chardata, "\n", 0, status);
ADD_DATACHUNK(chardata, "\r\n", 0, status); //keep CRLF sequences together
ADD_DATACHUNK(chardata, "S\\u0300", 0, status); //graveS
ADD_DATACHUNK(chardata, "i\\u0301", 0, status); //acuteBelowI
ADD_DATACHUNK(chardata, "!", 0, status);
// What follows is a string of Korean characters (I found it in the Yellow Pages
// ad for the Korean Presbyterian Church of San Francisco, and I hope I transcribed
// it correctly), first as precomposed syllables, and then as conjoining jamo.
// Both sequences should be semantically identical and break the same way.
// precomposed syllables...
ADD_DATACHUNK(chardata, "\\uc0c1", 0, status);
ADD_DATACHUNK(chardata, "\\ud56d", 0, status);
ADD_DATACHUNK(chardata, " ", 0, status);
ADD_DATACHUNK(chardata, "\\ud55c", 0, status);
ADD_DATACHUNK(chardata, "\\uc778", 0, status);
ADD_DATACHUNK(chardata, " ", 0, status);
ADD_DATACHUNK(chardata, "\\uc5f0", 0, status);
ADD_DATACHUNK(chardata, "\\ud569", 0, status);
ADD_DATACHUNK(chardata, " ", 0, status);
ADD_DATACHUNK(chardata, "\\uc7a5", 0, status);
ADD_DATACHUNK(chardata, "\\ub85c", 0, status);
ADD_DATACHUNK(chardata, "\\uad50", 0, status);
ADD_DATACHUNK(chardata, "\\ud68c", 0, status);
ADD_DATACHUNK(chardata, " ", 0, status);
// conjoining jamo...
ADD_DATACHUNK(chardata, "\\u1109\\u1161\\u11bc", 0, status);
ADD_DATACHUNK(chardata, "\\u1112\\u1161\\u11bc", 0, status);
ADD_DATACHUNK(chardata, " ", 0, status);
ADD_DATACHUNK(chardata, "\\u1112\\u1161\\u11ab", 0, status);
ADD_DATACHUNK(chardata, "\\u110b\\u1175\\u11ab", 0, status);
ADD_DATACHUNK(chardata, " ", 0, status);
ADD_DATACHUNK(chardata, "\\u110b\\u1167\\u11ab", 0, status);
ADD_DATACHUNK(chardata, "\\u1112\\u1161\\u11b8", 0, status);
ADD_DATACHUNK(chardata, " ", 0, status);
ADD_DATACHUNK(chardata, "\\u110c\\u1161\\u11bc", 0, status);
ADD_DATACHUNK(chardata, "\\u1105\\u1169", 0, status);
ADD_DATACHUNK(chardata, "\\u1100\\u116d", 0, status);
ADD_DATACHUNK(chardata, "\\u1112\\u116c", 0, status);
// Surrogate pairs stay together
ADD_DATACHUNK(chardata, "\\ud800\\udc00", 0, status);
ADD_DATACHUNK(chardata, "\\udbff\\udfff", 0, status);
ADD_DATACHUNK(chardata, "x", 0, status);
// 0xffff is a legal character, and should not stop the break iterator early.
// (Requires special casing in implementation, which is why it gets a test.)
ADD_DATACHUNK(chardata, "\\uffff", 0, status);
ADD_DATACHUNK(chardata, "\\uffff", 0, status);
ADD_DATACHUNK(chardata, " ", 0, status);
ADD_DATACHUNK(chardata, "a", 0, status);
// Regression test for bug 1889
ADD_DATACHUNK(chardata, "\\u0f40\\u0f7d", 0, status);
ADD_DATACHUNK(chardata, "\\u0000", 0, status);
ADD_DATACHUNK(chardata, "\\u0f7e", 0, status);
// \u0f7d\u0000\u0f7e
if(U_FAILURE(status)){
errln("FAIL : in BITestData construction");
return;
}
// Run the test...
generalIteratorTest(*charIterDefault, chardata);
delete charIterDefault;
// delete rbbi;
}
static const int T_NUMBER = 100;
static const int T_LETTER = 200;
static const int T_H_OR_K = 300;
static const int T_IDEO = 400;
//--------------------------------------------------------------------
//tests default rules based word iteration
//--------------------------------------------------------------------
void RBBITest::TestDefaultRuleBasedWordIteration()
{
logln((UnicodeString)"Testing the RBBI for word iteration using default rules");
UErrorCode status=U_ZERO_ERROR;
RuleBasedBreakIterator* wordIterDefault=(RuleBasedBreakIterator*)RuleBasedBreakIterator::createWordInstance(Locale::getDefault(), status);
if(U_FAILURE(status)){
errln("FAIL : in construction");
return;
}
BITestData worddata(status);
ADD_DATACHUNK(worddata, NULL, 0, status);
ADD_DATACHUNK(worddata, "Write", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "wordrules", T_LETTER, status);
ADD_DATACHUNK(worddata, ".", 0, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "123.456", T_NUMBER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "alpha\\u00adbeta\\u00adgamma", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "\\u092f\\u0939", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "\\u0939\\u093f" halfNA "\\u0926\\u0940", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "\\u0939\\u0948", T_LETTER, status);
//ADD_DATACHUNK(worddata, "\\u0964", 0); //danda followed by a space "\u0964->danda: hindi phrase seperator"
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "\\u0905\\u093e\\u092a", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "\\u0938\\u093f\\u0916\\u094b\\u0917\\u0947", T_LETTER, status);
ADD_DATACHUNK(worddata, "?", 0, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "\\u0968\\u0966.\\u0969\\u096f", T_NUMBER, status); //hindi numbers
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "\\u0967\\u0966\\u0966.\\u0966\\u0966", T_NUMBER, status); //postnumeric
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "\\u20a8", 0, status); // //pre-number India currency symbol Rs->\\u20aD
ADD_DATACHUNK(worddata, "\\u0967,\\u0967\\u0966\\u0966.\\u0966\\u0966", T_NUMBER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "\\u0905\\u092e\\u091c", T_LETTER, status);
ADD_DATACHUNK(worddata, "\n", 0, status);
ADD_DATACHUNK(worddata, halfSA "\\u0935\\u0924\\u0902" deadTA "\\u0930", T_LETTER, status);
ADD_DATACHUNK(worddata, "\r", 0, status);
ADD_DATACHUNK(worddata, "It's", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "$", 0, status);
ADD_DATACHUNK(worddata, "30.10", T_NUMBER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "12,34", T_NUMBER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "\\u00A2", 0, status); //cent sign
ADD_DATACHUNK(worddata, "\\u00A3", 0, status); //pound sign
ADD_DATACHUNK(worddata, "\\u00A4", 0, status); //currency sign
ADD_DATACHUNK(worddata, "\\u00A5", 0, status); //yen sign
ADD_DATACHUNK(worddata, "alpha\\u05f3beta\\u05f4gamma", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "Badges", T_LETTER, status);
ADD_DATACHUNK(worddata, "?", 0, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "BADGES", T_LETTER, status);
ADD_DATACHUNK(worddata, "!", 0, status);
ADD_DATACHUNK(worddata, "?", 0, status);
ADD_DATACHUNK(worddata, "!", 0, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "We", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "don't", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "need", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "no", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "STINKING", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "BADGES", T_LETTER, status);
ADD_DATACHUNK(worddata, "!", 0, status);
ADD_DATACHUNK(worddata, "!", 0, status);
ADD_DATACHUNK(worddata, "1000,233,456.000", T_NUMBER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "1,23.322", T_NUMBER, status);
ADD_DATACHUNK(worddata, "%", 0, status);
ADD_DATACHUNK(worddata, "123.1222", T_NUMBER, status);
ADD_DATACHUNK(worddata, "$", 0, status);
ADD_DATACHUNK(worddata, "123,000.20", T_NUMBER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "179.01", T_NUMBER, status);
ADD_DATACHUNK(worddata, "%", 0, status);
ADD_DATACHUNK(worddata, "X", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "Now", T_LETTER, status);
ADD_DATACHUNK(worddata, "\r", 0, status);
ADD_DATACHUNK(worddata, "is", T_LETTER, status);
ADD_DATACHUNK(worddata, "\n", 0, status);
ADD_DATACHUNK(worddata, "the", T_LETTER, status);
ADD_DATACHUNK(worddata, "\r\n", 0, status);
ADD_DATACHUNK(worddata, "time", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "\\uc5f0\\ud569", T_LETTER, status); // Hangul Syllables
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "\\uc7a5\\ub85c\\uad50\\ud68c", T_LETTER, status); // Hangul
ADD_DATACHUNK(worddata, " ", 0, status);
// conjoining jamo...
ADD_DATACHUNK(worddata, "\\u1109\\u1161\\u11bc\\u1112\\u1161\\u11bc", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "\\u1112\\u1161\\u11ab\\u110b\\u1175\\u11ab", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "Hello", T_LETTER, status);
ADD_DATACHUNK(worddata, ",", 0, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "how", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "are", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "you", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
// Words containing non-BMP letters
ADD_DATACHUNK(worddata, "abc\\U00010300", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "abc\\U0001044D", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "abc\\U0001D433", T_LETTER, status); //MATHEMATICAL BOLD SMALL Z
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "abc\\U0001D7C9", T_LETTER, status); //MATHEMATICAL SANS-SERIF BOLD ITALIC PI
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "abc", T_LETTER, status); // same test outside of letter range.
ADD_DATACHUNK(worddata, "\\U0001D800", 0, status);
ADD_DATACHUNK(worddata, "def", T_LETTER, status);
ADD_DATACHUNK(worddata, "\\U0001D3FF", 0, status);
ADD_DATACHUNK(worddata, " ", 0, status);
// Hiragana & Katakana stay together, but separates from each other and Latin.
// TODO: Hira and Kata ranges from UnicodeSet differ slightly from
// what's in Unicode Scripts file. Investigate.
ADD_DATACHUNK(worddata, "abc", T_LETTER, status);
ADD_DATACHUNK(worddata, "\\u3041", T_H_OR_K, status); // Hiragana
ADD_DATACHUNK(worddata, "\\u3094\\u0301", T_H_OR_K, status); // Hiragana
ADD_DATACHUNK(worddata, "\\u309d", T_H_OR_K, status); // Hiragana
ADD_DATACHUNK(worddata, "\\u30a1\\u30fd\\uff66\\uff9d", T_H_OR_K, status); // Katakana
ADD_DATACHUNK(worddata, "def", T_LETTER, status);
ADD_DATACHUNK(worddata, "#", 0, status);
// Words with interior formatting characters
ADD_DATACHUNK(worddata, "def\\u0301\\u070Fabc", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
// to test for bug #4097779
ADD_DATACHUNK(worddata, "aa\\u0300a", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
// to test for bug #4098467
// What follows is a string of Korean characters (I found it in the Yellow Pages
// ad for the Korean Presbyterian Church of San Francisco, and I hope I transcribed
// it correctly), first as precomposed syllables, and then as conjoining jamo.
// Both sequences should be semantically identical and break the same way.
// precomposed syllables...
ADD_DATACHUNK(worddata, "\\uc0c1\\ud56d", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "\\ud55c\\uc778", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "\\uc5f0\\ud569", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "\\uc7a5\\ub85c\\uad50\\ud68c", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
// conjoining jamo...
ADD_DATACHUNK(worddata, "\\u1109\\u1161\\u11bc\\u1112\\u1161\\u11bc", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "\\u1112\\u1161\\u11ab\\u110b\\u1175\\u11ab", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "\\u110b\\u1167\\u11ab\\u1112\\u1161\\u11b8", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "\\u110c\\u1161\\u11bc\\u1105\\u1169\\u1100\\u116d\\u1112\\u116c", T_LETTER, status);
ADD_DATACHUNK(worddata, " ", 0, status);
// this is a test for bug #4117554: the ideographic iteration mark (U+3005) should
// count as a Kanji character for the purposes of word breaking
ADD_DATACHUNK(worddata, "abc", T_LETTER, status);
// Unicode TR29: Ideographs do NOT group together into words.
//wordSelectionData->addElement(CharsToUnicodeString("\\u4e01\\u4e02\\u3005\\u4e03\\u4e03"));
ADD_DATACHUNK(worddata, "\\u4e01", T_IDEO, status);
ADD_DATACHUNK(worddata, "\\u4e02", T_IDEO, status);
ADD_DATACHUNK(worddata, "\\u3005", T_LETTER, status); // TODO: 3005 is ideographic iteration mark
// Treating as letter is according to TR.
// Check whether this is really intended.
ADD_DATACHUNK(worddata, "\\u4e03", T_IDEO, status);
ADD_DATACHUNK(worddata, "\\u4e03", T_IDEO, status);
ADD_DATACHUNK(worddata, "abc", T_LETTER, status);
//
// Try some words from other scripts.
//
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "\\u0391\\u0392\\u0393", T_LETTER, status); // Greek
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "\\u0411\\u0412\\u0413", T_LETTER, status); // Cyrillic
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "\\u05D0\\u05D1\\u05D2\\u0593", T_LETTER, status); // Hebrew
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "\\u0627\\u0628\\u062A", T_LETTER, status); // Arabic
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "\\u0661\\u0662\\u0663", T_NUMBER, status); // Arabic
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "\\u10A0\\u10A1\\u10A2", T_LETTER, status); // Georgian
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "ABC", T_LETTER, status); // Latin
if (U_FAILURE(status)){
errln("FAIL : in BITestData construction");
return;
}
generalIteratorTest(*wordIterDefault, worddata);
delete wordIterDefault;
}
//--------------------------------------------------------------------
//Testing the BreakIterator for devanagari script
//--------------------------------------------------------------------
#define deadRA "\\u0930\\u094d" /*deadform RA = devanagari RA + virama*/
#define deadPHA "\\u092b\\u094d" /*deadform PHA = devanagari PHA + virama*/
#define deadTTHA "\\u0920\\u094d"
#define deadPA "\\u092a\\u094d"
#define deadSA "\\u0938\\u094d"
#define visarga "\\u0903" /*devanagari visarga looks like a english colon*/
void RBBITest::TestHindiCharacterBreak()
{
UErrorCode status= U_ZERO_ERROR;
BITestData hindicharData(status);
ADD_DATACHUNK(hindicharData, NULL, 0, status); // Break at start of data
//devanagari characters for Hindi support
ADD_DATACHUNK(hindicharData, "\\u0906", 0, status); //devanagari AA
//hindi character break should make sure that it
// doesn't break in-between a vowelsign and a chandrabindu
ADD_DATACHUNK(hindicharData, "\\u000a", 0, status); // Force break so following can appear stand-alone.
ADD_DATACHUNK(hindicharData, "\\u093e\\u0901", 0, status); //devanagari vowelsign AA+ chandrabindu
ADD_DATACHUNK(hindicharData, "\\u0906\\u0901", 0, status); // Devanagari AA + chandrabindu
ADD_DATACHUNK(hindicharData, "\\u0915", 0, status); // Devanagari KA
ADD_DATACHUNK(hindicharData, "\\u093e\\u0901", 0, status); // Devanagari AA vowelsign + chandrabindu
ADD_DATACHUNK(hindicharData, "\\u0916\\u0947", 0, status); //devanagari KHA+vowelsign E
ADD_DATACHUNK(hindicharData, "\\u0938\\u0941\\u0902", 0, status); //devanagari SA+vowelsign U + anusvara(bindu)
ADD_DATACHUNK(hindicharData, "\\u0926", 0, status); //devanagari consonant DA
ADD_DATACHUNK(hindicharData, "\\u0930", 0, status); //devanagari consonant RA
ADD_DATACHUNK(hindicharData, "\\u0939", 0, status); //devanagari consonant HA+
ADD_DATACHUNK(hindicharData, "\\u094c", 0, status); // +dependent vowel sign AI
ADD_DATACHUNK(hindicharData, "\\u0964", 0, status); //devanagari danda
ADD_DATACHUNK(hindicharData, "\\u0950", 0, status); //devanagari OM
ADD_DATACHUNK(hindicharData, "\\u0915\\u0943", 0, status); //devanagari KA+dependent vowel RI->KRI
//dependent half-forms. 2002-8-7: New Char Break rules no longer join the half-sequences.
ADD_DATACHUNK(hindicharData, /* halfSA */ "\\u0924", 0, status); //halfSA+base consonant TA->STA
ADD_DATACHUNK(hindicharData, /* halfSA */ "\\u0925", 0, status); //halfSA+base consonant THA->STHA
ADD_DATACHUNK(hindicharData, /* halfSA */ "\\u092e", 0, status); //halfSA+base consonant MA->SMA
ADD_DATACHUNK(hindicharData, /* halfCHA */ "\\u091b", 0, status); //halfCHA+base consonant CHHA->CHHHA
ADD_DATACHUNK(hindicharData, /* halfNA */ "\\u0917", 0, status); //halfNA+base consonant GA->NGA
// ADD_DATACHUNK(hindicharData, "\\u092a\\u094d\\u200d\\u092f", 0, status); //halfPA(PA+virama+zerowidthjoiner+base consonant YA->PYA
ADD_DATACHUNK(hindicharData, "\\u092a\\u094d", 0, status); //halfPA(PA+virama+zerowidthjoiner+base consonant YA->PYA
ADD_DATACHUNK(hindicharData, "\\u200d", 0, status); //halfPA(PA+virama+zerowidthjoiner+base consonant YA->PYA
ADD_DATACHUNK(hindicharData, "\\u092f", 0, status); //halfPA(PA+virama+zerowidthjoiner+base consonant YA->PYA
//consonant RA rules ----------
//if the dead consonant RA precedes either a consonant or an independent vowel,
//then it is replaced by its superscript non-spacing mark
ADD_DATACHUNK(hindicharData, /* deadRA */ "\\u0915", 0, status); //deadRA+devanagari consonant KA->KA+superRA
ADD_DATACHUNK(hindicharData, /* deadRA */ "\\u0923", 0, status); //deadRA+devanagari consonant NNA->NNA+superRA
ADD_DATACHUNK(hindicharData, /* deadRA */ "\\u0917", 0, status); //deadRA+devanagari consonant GA->GA+superRA
// ADD_DATACHUNK(hindicharData, deadRA+ "\\u0960", 0); //deadRA+devanagari cosonant RRI->RRI+superRA
//if any dead consonant(other than dead RA)precedes the consonant RA, then
//it is replaced with its nominal forma nd RA is replaced by the subscript non-spacing mark.
ADD_DATACHUNK(hindicharData, /* deadPHA */ "\\u0930", 0, status); //deadPHA+devanagari consonant RA->PHA+subRA
ADD_DATACHUNK(hindicharData, /* deadPA */ "\\u0930", 0, status); //deadPA+devanagari consonant RA->PA+subRA
ADD_DATACHUNK(hindicharData, /* deadTTHA */ "\\u0930", 0, status); //deadTTHA+devanagari consonant RA->TTHA+subRA
ADD_DATACHUNK(hindicharData, /* deadTA */ "\\u0930", 0, status); //deadTA+RA->TRA
// ADD_DATACHUNK(hindicharData, "\\u0936\\u094d\\u0930", 0, status); //deadSHA(SHA+virama)+RA->SHRA
ADD_DATACHUNK(hindicharData, "\\u0936\\u094d", 0, status); //deadSHA(SHA+virama)+RA->SHRA
ADD_DATACHUNK(hindicharData, "\\u0930", 0, status); //deadSHA(SHA+virama)+RA->SHRA
//conjuct ligatures
// 2002-08-7 virma no longer forces joining.
// ADD_DATACHUNK(hindicharData, "\\u0915\\u094d\\u0937", 0, status); //deadKA(KA+virama) followed by SSHA wraps up into a single character KSSHA
ADD_DATACHUNK(hindicharData, "\\u0915\\u094d", 0, status); //deadKA(KA+virama) followed by SSHA wraps up into a single character KSSHA
ADD_DATACHUNK(hindicharData, "\\u0937", 0, status); //deadKA(KA+virama) followed by SSHA wraps up into a single character KSSHA
ADD_DATACHUNK(hindicharData, /* deadTA */ "\\u0924", 0, status); //deadTA+TA wraps up into glyph TTHA
//ADD_DATACHUNK(hindicharData, "\\u0926\\u094d\\u0935", 0, status); //deadDA(DA+virama)+VA wraps up into DVA
//ADD_DATACHUNK(hindicharData, "\\u091c\\u094d\\u091e", 0, status); //deadJA(JA+virama)+NYA wraps up into JNYA
RuleBasedBreakIterator *e=(RuleBasedBreakIterator*)RuleBasedBreakIterator::createCharacterInstance(Locale::getDefault(), status);
if(U_FAILURE(status)){
errln("FAIL : in construction");
return;
}
generalIteratorTest(*e, hindicharData);
delete e;
}
void RBBITest::TestHindiWordBreak()
{
UErrorCode status= U_ZERO_ERROR;
BITestData hindiWordData(status);
//hindi
ADD_DATACHUNK(hindiWordData, NULL, 0, status); // Break at start of data
ADD_DATACHUNK(hindiWordData, "\\u0917\\u092a\\u00ad\\u0936\\u092a", 200, status);
ADD_DATACHUNK(hindiWordData, "!", 0, status);
ADD_DATACHUNK(hindiWordData, "\\u092f\\u0939", 200, status);
ADD_DATACHUNK(hindiWordData, " ", 0, status);
ADD_DATACHUNK(hindiWordData, "\\u0939\\u093f" halfNA "\\u0926\\u0940", 200, status);
ADD_DATACHUNK(hindiWordData, " ", 0, status);
ADD_DATACHUNK(hindiWordData, "\\u0939\\u0948", 200, status);
//danda is similar to full stop. danda is a hindi phrase seperator
//Make sure it breaks before danda and after danda when it is followed by a space
//ADD_DATACHUNK(hindiWordData, "\\u0964", 0); //fails here doesn't break at danda
ADD_DATACHUNK(hindiWordData, " ", 0, status);
ADD_DATACHUNK(hindiWordData, "\\u0905\\u093e\\u092a", 200, status);
ADD_DATACHUNK(hindiWordData, " ", 0, status);
ADD_DATACHUNK(hindiWordData, "\\u0938\\u093f\\u0916\\u094b\\u0917\\u0947", 200, status);
ADD_DATACHUNK(hindiWordData, "?", 0, status);
ADD_DATACHUNK(hindiWordData, "\n", 0, status);
ADD_DATACHUNK(hindiWordData, ":", 0, status);
ADD_DATACHUNK(hindiWordData, deadPA "\\u0930\\u093e\\u092f" visarga, 200, status); //no break before visarga
ADD_DATACHUNK(hindiWordData, " ", 0, status);
ADD_DATACHUNK(hindiWordData, "\\u0935" deadRA "\\u0937\\u093e", 200, status);
ADD_DATACHUNK(hindiWordData, "\r\n", 0, status);
ADD_DATACHUNK(hindiWordData, deadPA "\\u0930\\u0915\\u093e\\u0936", 200, status); //deadPA+RA+KA+vowel AA+SHA -> prakash
ADD_DATACHUNK(hindiWordData, ",", 0, status);
ADD_DATACHUNK(hindiWordData, "\\u0924\\u0941\\u092e\\u093e\\u0930\\u094b", 200, status);
ADD_DATACHUNK(hindiWordData, " ", 0, status);
ADD_DATACHUNK(hindiWordData, "\\u092e\\u093f" deadTA "\\u0930", 200, status); //MA+vowel I+ deadTA + RA
ADD_DATACHUNK(hindiWordData, " ", 0, status);
ADD_DATACHUNK(hindiWordData, "\\u0915\\u093e", 200, status);
ADD_DATACHUNK(hindiWordData, " ", 0, status);
ADD_DATACHUNK(hindiWordData, "\\u092a" deadTA "\\u0930", 200, status); //PA + deadTA + RA
ADD_DATACHUNK(hindiWordData, " ", 0, status);
ADD_DATACHUNK(hindiWordData, "\\u092a\\u095d\\u094b", 200, status);
// ADD_DATACHUNK(hindiWordData, "\\u0964", 0); //fails here doesn't break at danda
ADD_DATACHUNK(hindiWordData, " ", 0, status);
ADD_DATACHUNK(hindiWordData, deadSA deadTA "\\u0930\\u093f", 200, status); //deadSA+deadTA+RA+vowel I->sthri
ADD_DATACHUNK(hindiWordData, ".", 0, status);
ADD_DATACHUNK(hindiWordData, " ", 0, status);
ADD_DATACHUNK(hindiWordData, "\\u0968\\u0966.\\u0969\\u096f", 100, status); //hindi numbers
ADD_DATACHUNK(hindiWordData, " ", 0, status);
ADD_DATACHUNK(hindiWordData, "\\u0967\\u0966\\u0966.\\u0966\\u0966", 100, status); //postnumeric
ADD_DATACHUNK(hindiWordData, "\\u20a8", 0, status);
ADD_DATACHUNK(hindiWordData, "\\u0967,\\u0967\\u0966\\u0966.\\u0966\\u0966", 100, status); //pre-number India currency symbol Rs.\\u20aD
ADD_DATACHUNK(hindiWordData, " ", 0, status);
ADD_DATACHUNK(hindiWordData, "\\u0905\\u092e\\u091c", 200, status);
ADD_DATACHUNK(hindiWordData, "\n", 0, status);
ADD_DATACHUNK(hindiWordData, halfSA "\\u0935\\u0924\\u0902" deadTA "\\u0930", 200, status);
ADD_DATACHUNK(hindiWordData, "\r", 0, status);
RuleBasedBreakIterator *e=(RuleBasedBreakIterator*)RuleBasedBreakIterator::createWordInstance(Locale::getDefault(), status);
if(U_FAILURE(status)){
errln("FAIL : in construction");
return;
}
generalIteratorTest(*e, hindiWordData);
delete e;
}
void RBBITest::TestTitleBreak()
{
UErrorCode status= U_ZERO_ERROR;
RuleBasedBreakIterator* titleI=(RuleBasedBreakIterator*)RuleBasedBreakIterator::createTitleInstance(Locale::getDefault(), status);
if(U_FAILURE(status)){
errln("FAIL : in construction");
return;
}
BITestData titleData(status);
ADD_DATACHUNK(titleData, NULL, 0, status); // Break at start of data
ADD_DATACHUNK(titleData, " ", 0, status);
ADD_DATACHUNK(titleData, "This ", 0, status);
ADD_DATACHUNK(titleData, "is ", 0, status);
ADD_DATACHUNK(titleData, "a ", 0, status);
ADD_DATACHUNK(titleData, "simple ", 0, status);
ADD_DATACHUNK(titleData, "sample ", 0, status);
ADD_DATACHUNK(titleData, "sentence. ", 0, status);
ADD_DATACHUNK(titleData, "This ", 0, status);
generalIteratorTest(*titleI, titleData);
delete titleI;
}
//-----------------------------------------------------------------------------------
//
// Test for status {tag} return value from break rules.
// TODO: a more thorough test.
//
//-----------------------------------------------------------------------------------
void RBBITest::TestStatusReturn() {
UnicodeString rulesString1 = "$Letters = [:L:];\n"
"$Numbers = [:N:];\n"
"$Letters+{1};\n"
"$Numbers+{2};\n"
"Help\\ {4}/me\\!;\n"
"[^$Letters $Numbers];\n"
"!.*;\n";
UnicodeString testString1 = "abc123..abc Help me Help me!";
// 01234567890123456789012345678
int32_t bounds1[] = {0, 3, 6, 7, 8, 11, 12, 16, 17, 19, 20, 25, 27, 28, -1};
int32_t brkStatus[] = {0, 1, 2, 0, 0, 1, 0, 1, 0, 1, 0, 4, 1, 0, -1};
UErrorCode status=U_ZERO_ERROR;
UParseError parseError;
RuleBasedBreakIterator *bi = new RuleBasedBreakIterator(rulesString1, parseError, status);
if(U_FAILURE(status)) {
errln("FAIL : in construction");
} else {
int32_t pos;
int32_t i = 0;
bi->setText(testString1);
for (pos=bi->first(); pos!= BreakIterator::DONE; pos=bi->next()) {
if (pos != bounds1[i]) {
errln("FAIL: expected break at %d, got %d\n", bounds1[i], pos);
break;
}
int tag = bi->getRuleStatus();
if (tag != brkStatus[i]) {
errln("FAIL: break at %d, expected tag %d, got tag %d\n", pos, brkStatus[i], tag);
break;
}
i++;
}
}
delete bi;
}
/*
//Bug: if there is no word break before and after danda when it is followed by a space
void RBBITest::TestDanda()
{
Vector *hindiWordData = new Vector();
//hindi
ADD_DATACHUNK(hindiWordData, CharsToUnicodeString("\\u092f\\u0939"), 0, status);
ADD_DATACHUNK(hindiWordData, " ", 0, status);
//Danda is similar to full stop, danda is a hindi phrase seperator.
//Make sure there is a word break before and after danda when it is followed by a space
//following fail----
ADD_DATACHUNK(hindiWordData, CharsToUnicodeString("\\u0939\\u0948"), 0, status);
// ADD_DATACHUNK(hindiWordData, CharsToUnicodeString("\\u0964"), 0); // devanagari danda
ADD_DATACHUNK(hindiWordData, " ", 0, status);
ADD_DATACHUNK(hindiWordData, CharsToUnicodeString("\\u092f\\u0939"), 0, status);
// ADD_DATACHUNK(hindiWordData, CharsToUnicodeString("\\u0965"), 0); //devanagari double danda
ADD_DATACHUNK(hindiWordData, " ", 0, status);
RuleBasedBreakIterator* e=(RuleBasedBreakIterator*)RuleBasedBreakIterator::createWordInstance();
generalIteratorTest(*e, hindiWordData);
delete e;
delete hindiWordData;
}
//Make sure the character wrapping is done correctly
void RBBITest::TestHindiCharacterWrapping()
{
Vector *hindicharData = new Vector();
//if the dead consonant RA precedes either a consonant or an independent vowel,
//then it is replaced by its superscript non-spacing mark
ADD_DATACHUNK(hindicharData, deadRA+ CharsToUnicodeString("\\u0917"), 0, status); //deadRA+devanagari consonant GA->GA+superRA
//following fail----
// ADD_DATACHUNK(hindicharData, deadRA+ CharsToUnicodeString("\\u0960"), 0); //deadRA+devanagari RRI->RRI+superRA
RuleBasedBreakIterator* e=(RuleBasedBreakIterator*)RuleBasedBreakIterator::createCharacterInstance();
generalIteratorTest(*e, hindicharData);
delete e;
delete hindicharData;
}*/
//----------------------------------------------------------------------------------
//adds rules for telugu support and tests the behaviour of chracterIterator of RBBI
//----------------------------------------------------------------------------------
/*void RBBITest::TestTeluguRuleBasedCharacterIteration()
{
logln((UnicodeString)"Testing the RBBI by adding rules for Telugu(Indian Language) Support");
//get the default rules
RuleBasedBreakIterator *rb= (RuleBasedBreakIterator*)BreakIterator::createCharacterInstance();
//additional rules for Telugu(Indian Language) support
UnicodeString crules1 = rb->getRules() + //default rules +
"<telvirama>=[\\u0c4d];" + //telugu virama
"<telVowelSign>=[\\u0c3e-\\u0c44\\u0c46\\u0c47\\u0c48\\u0c4a\\u0c4b\\u0c4c];" + //telugu dependent vowel signs
"<telConsonant>=[\\u0c15-\\u0c28\\u0c2a-\\u0c33\\u0c35-\\u0c39];" + //telugu consonants
"<telCharEnd>=[\\u0c02\\u0c03\\u0c55\\u0c56];" + //to create half forms and dead forms
"<telConjunct>=({<telConsonant><telvirama>{<zwj>}}<telConsonant>);" +
"<telConjunct>{<telVowelSign>}{<telCharEnd>};";
RuleBasedBreakIterator charIter=null;
charIter = new RuleBasedBreakIterator(crules1);
Vector *chardata = new Vector();
//behaviour of telugu characters from specified rules
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u0c15"), 0, status); //telugu consonant KA
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u0c30\\u0c47"), 0, status); //telugu consonant RA+telugu dependent vowel EE
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u0c1b\\u0c3e"), 0, status); //telugu consonant CHA+telegu depenednt vowel AA
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u0c17\\u0c48"), 0, status); //telegu consonant GA+teleugu dependent vowel AI
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u0c17\\u0c46\\u0c56"), 0, status); //telugu consonant GA+telugu dependent vowel sign E+telugu AI length mark
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u0c28\\u0c4d\\u200d\\u0c28"), 0, status); //telugu consonant NA+telugu virama+zwj=>halfNA+NA->NNA(dependent half-form)
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u0c17\\u0c4d\\u0c30"), 0, status); //GA+deadRA(RA+telvirama)->GA+subRA->GRA
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u0c66"), 0, status); //telugu digit
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u0c37\\u0c4d\\u0c15"), 0, status); //deadSSA(SSA+telvirama)+KA+subSSA->KSHA
//behaviour of other characters from default rules
ADD_DATACHUNK(chardata, "h", 0, status);
ADD_DATACHUNK(chardata, CharsToUnicodeString("A\\u0302"), 0, status); // circumflexA
ADD_DATACHUNK(chardata, CharsToUnicodeString("i\\u0301"), 0, status); // acuteBelowI
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u1109\\u1161\\u11bc"), 0, status);
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u1112\\u1161\\u11bc"), 0, status);
ADD_DATACHUNK(chardata, "\n", 0, status);
ADD_DATACHUNK(chardata, "\r\n", 0, status);
generalIteratorTest(charIter, chardata);
delete charIter;
delete charData;
delete rb;
}
//--------------------------------------------------------------------
//tests the behaviour of character iteration of RBBI with custom rules
//--------------------------------------------------------------------
void RBBITest::TestCustomRuleBasedCharacterIteration()
{
logln((UnicodeString)"Testing the RBBI by using custom rules for character iteration");
UnicodeString crules2="<ignore>=[e];" + //ignore the character "e"
".;" +
"<devVowelSign>=[\\u093e-\\u094c\\u0962\\u0963];" + //devanagiri vowel = \\u093e tp \\u094c and \\u0962.\\u0963
"<devConsonant>=[\\u0915-\\u0939];" + //devanagiri consonant = \\u0915 to \\u0939
"<devConsonant>{<devVowelSign>};" ; //break at all places except the following
//devanagiri consonants+ devanagiri vowelsign
RuleBasedCharacterIterator charIterCustom = new RuleBasedBreakIterator(crules2);
Vector *chardata = new Vector();
ADD_DATACHUNK(chardata, "He", 0, status); //ignores 'e'
ADD_DATACHUNK(chardata, "l", 0, status);
ADD_DATACHUNK(chardata, "l", 0, status);
ADD_DATACHUNK(chardata, "oe", 0, status); //ignores 'e' hence wraps it into 'o' instead of wrapping with
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u0301"), 0, status); //'\\u0301' to form 'acuteE '
ADD_DATACHUNK(chardata, "&e", 0, status); //ignores 'e' hence wraps it into '&' instead of wrapping with
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u0303"), 0, status); //'\\u0303 to form 'tildaE'
//devanagiri characters
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u0906"), 0, status); //devanagiri AA
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u093e"), 0, status); //devanagiri vowelsign AA:--breaks at \\u0901 which is devanagiri
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u0901"), 0, status); //chandra bindhu since it is not mentioned in the rules
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u0916\\u0947"), 0, status); //devanagiri KHA+vowelsign E
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u0938\\u0941"), 0, status); //devanagiri SA+vowelsign U : - breaks at
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u0902"), 0, status); //\\u0902 devanagiri anusvara since it is not mentioned in the rules
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u0926"), 0, status); //devanagiri consonant DA
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u0930"), 0, status); //devanagiri consonant RA
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u0939\\u094c"), 0, status); //devanagiri HA+vowel sign AI
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u0964"), 0, status); //devanagiri danda
// devanagiri chracters end
ADD_DATACHUNK(chardata, "A", 0, status); //breaks in between since it is not mentioned in the rules
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u0302"), 0, status); // circumflexA
ADD_DATACHUNK(chardata, "i", 0, status); //breaks in between since not mentioned in the rules
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u0301"), 0, status); // acuteBelowI
//Rules don't support conjoining jamo->->..
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u1109"), 0, status); //break at every character since rules
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u1161"), 0, status); //don't support conjoining jamo
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u11bc"), 0, status);
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u1112"), 0, status);
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u1161"), 0, status);
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u11bc"), 0, status);
ADD_DATACHUNK(chardata, "\n", 0, status);
ADD_DATACHUNK(chardata, "\r", 0, status); //doesn't keep CRLGF together since rules do not mention it
ADD_DATACHUNK(chardata, "\n", 0, status);
ADD_DATACHUNK(chardata, "S", 0, status); //graveS
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u0300"), 0, status); //breaks in between since it is not mentioned in the rules
ADD_DATACHUNK(chardata, "i", 0, status); //acuteBelowI
ADD_DATACHUNK(chardata, CharsToUnicodeString("\\u0301"), 0, status); //breaks in between since it is not mentioned in the rules
generalIteratorTest(charIterCustom, chardata);
delete charIterCustom;
delete chardata;
}*/
/*//--------------------------------------------------------------------
//tests custom rules based word iteration
//--------------------------------------------------------------------
void RBBITest::TestCustomRuleBasedWordIteration(){
logln("(UnicodeString)Testing the RBBI by using custom rules for word iteration");
UnicodeString wrules1="<ignore>=[:Mn::Me::Cf:];" + //ignore non-spacing marks, enclosing marks, and format characters,
"<danda>=[\\u0964\\u0965];" + //Hindi Phrase seperator
"<let>=[:L::Mc:];" + //uppercase(Lu), lowercase(Ll), titlecase(Lt), modifier(Lm) letters, Mc-combining space mark
"<mid-word>=[:Pd:\\\"\\\'\\.];" + //dashes, quotation, apostraphes, period
"<ls>=[\\n\\u000c\\u2028\\u2029];" + //line separators: LF, FF, PS, and LS
"<ws>=[:Zs:\\t];" + //all space separators and the tab character
"<word>=((<let><let>*(<mid-word><let><let>*)*));" +
".;" + //break after every character, with the following exceptions
"{<word>};" +
"<ws>*{\\r}{<ls>}{<danda>};" ;
RuleBasedBreakIterator wordIterCustom = new RuleBasedBreakIterator(wrules1);
Vector *worddata = new Vector();
ADD_DATACHUNK(worddata, "Write", 0, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "wordrules", 0, status);
ADD_DATACHUNK(worddata, ".", 0, status);
ADD_DATACHUNK(worddata, " ", 0, status);
//play with hindi
ADD_DATACHUNK(worddata, CharsToUnicodeString("\\u092f\\u0939"), 0, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, CharsToUnicodeString("\\u0939\\u093f") + halfNA + CharsToUnicodeString("\\u0926\\u0940"), 0, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, CharsToUnicodeString("\\u0939\\u0948"), 0, status);
ADD_DATACHUNK(worddata, CharsToUnicodeString("\\u0964"), 0, status); //Danda is similar to full stop-> Danda followed by a space
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, CharsToUnicodeString("\\u0905\\u093e\\u092a"), 0, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, CharsToUnicodeString("\\u0938\\u093f\\u0916\\u094b\\u0917\\u0947"), 0, status);
ADD_DATACHUNK(worddata, "?", 0, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "It's", 0, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "$", 0, status);
ADD_DATACHUNK(worddata, "3", 0, status);
ADD_DATACHUNK(worddata, "0", 0, status);
ADD_DATACHUNK(worddata, ".", 0, status);
ADD_DATACHUNK(worddata, "1", 0, status);
ADD_DATACHUNK(worddata, "0", 0, status);
ADD_DATACHUNK(worddata, " ", 0, status);
// ADD_DATACHUNK(worddata, " ", 0);
generalIteratorTest(wordIterCustom, worddata);
delete wordIterCustom;
delete worddata;
}
//-------------------------------------------------------------------------------
//adds extra rules to deal with abbrevations(limited) and test the word Iteration
//-------------------------------------------------------------------------------
void RBBITest::TestAbbrRuleBasedWordIteration()
{
logln((UnicodeString)"Testing the RBBI for word iteration by adding rules to support abbreviation");
RuleBasedBreakIterator *rb =(RuleBasedBreakIterator*)BreakIterator::createWordInstance();
UnicodeString wrules2="<abbr>=((Mr.)|(Mrs.)|(Ms.)|(Dr.)|(U.S.));" + // abbreviations.
rb->getRules() +
"{(<abbr><ws>)*<word>};";
RuleBasedBreakIterator wordIter=null;
//try{
wordIter = new RuleBasedBreakIterator(wrules2);
// }catch(IllegalArgumentException iae){
// errln("ERROR: failed construction illegal rules");
// }
Vector *worddata = new Vector();
ADD_DATACHUNK(worddata, "Mr. George", 0, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "is", 0, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "from", 0, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "U.S. Navy", 0, status);
ADD_DATACHUNK(worddata, ".", 0, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "His", 0, status);
ADD_DATACHUNK(worddata, "\n", 0, status);
ADD_DATACHUNK(worddata, "friend", 0, status);
ADD_DATACHUNK(worddata, "\t", 0, status);
ADD_DATACHUNK(worddata, "Dr. Steven", 0, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "married", 0, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "Ms. Benneth", 0, status);
ADD_DATACHUNK(worddata, "!", 0, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "Mrs. Johnson", 0, status);
ADD_DATACHUNK(worddata, "\r\n", 0, status);
ADD_DATACHUNK(worddata, "paid", 0, status);
ADD_DATACHUNK(worddata, " ", 0, status);
ADD_DATACHUNK(worddata, "$2,400.00", 0, status);
generalIteratorTest(wordIter, worddata);
delete wordIter;
delete worddata;
delete rb;
} */
void RBBITest::TestThaiLineBreak() {
UErrorCode status = U_ZERO_ERROR;
BITestData thaiLineSelection(status);
// \u0e2f-- the Thai paiyannoi character-- isn't a letter. It's a symbol that
// represents elided letters at the end of a long word. It should be bound to
// the end of the word and not treated as an independent punctuation mark.
ADD_DATACHUNK(thaiLineSelection, NULL, 0, status); // Break at start of data
ADD_DATACHUNK(thaiLineSelection, "\\u0e2a\\u0e16\\u0e32\\u0e19\\u0e35\\u0e2f", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e08\\u0e30", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e23\\u0e30\\u0e14\\u0e21", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e40\\u0e08\\u0e49\\u0e32", 0, status);
// ADD_DATACHUNK(thaiLineSelection, "\\u0e2b\\u0e19\\u0e49\\u0e32", 0, status);
// ADD_DATACHUNK(thaiLineSelection, "\\u0e17\\u0e35\\u0e48", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e2b\\u0e19\\u0e49\\u0e32\\u0e17\\u0e35\\u0e48", 0, status);
// the commented-out lines (I think) are the preferred result; this line is what our current dictionary is giving us
ADD_DATACHUNK(thaiLineSelection, "\\u0e2d\\u0e2d\\u0e01", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e21\\u0e32", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e40\\u0e23\\u0e48\\u0e07", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e23\\u0e30\\u0e1a\\u0e32\\u0e22", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e2d\\u0e22\\u0e48\\u0e32\\u0e07", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e40\\u0e15\\u0e47\\u0e21", 0, status);
// the one time where the paiyannoi occurs somewhere other than at the end
// of a word is in the Thai abbrevation for "etc.", which both begins and
// ends with a paiyannoi
ADD_DATACHUNK(thaiLineSelection, "\\u0e2f\\u0e25\\u0e2f", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e17\\u0e35\\u0e48", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e19\\u0e31\\u0e49\\u0e19", 0, status);
RuleBasedBreakIterator* e = (RuleBasedBreakIterator *)BreakIterator::createLineInstance(
Locale("th"), status);
if (U_FAILURE(status))
{
errln("Failed to create the BreakIterator for Thai locale in TestThaiLineBreak.\n");
return;
}
generalIteratorTest(*e, thaiLineSelection);
delete e;
}
void RBBITest::TestMixedThaiLineBreak()
{
UErrorCode status = U_ZERO_ERROR;
BITestData thaiLineSelection(status);
ADD_DATACHUNK(thaiLineSelection, NULL, 0, status); // Break at start of data
// Arabic numerals should always be separated from surrounding Thai text
/*
ADD_DATACHUNK(thaiLineSelection, "\\u0e04\\u0e48\\u0e32", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e40\\u0e07\\u0e34\\u0e19", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e1a\\u0e32\\u0e17", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e41\\u0e15\\u0e30", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e23\\u0e30\\u0e14\\u0e31\\u0e1a", 0, status);
thaiLineSelection->addElement("39");
ADD_DATACHUNK(thaiLineSelection, "\\u0e1a\\u0e32\\u0e17 ", 0, status);
// words in non-Thai scripts should always be separated from surrounding Thai text
ADD_DATACHUNK(thaiLineSelection, "\\u0e17\\u0e14", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e2a\\u0e2d\\u0e1a", 0, status);
thaiLineSelection->addElement("Java");
ADD_DATACHUNK(thaiLineSelection, "\\u0e1a\\u0e19", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e40\\u0e04\\u0e23\\u0e37\\u0e48\\u0e2d\\u0e07", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e44\\u0e2d\\u0e1a\\u0e35\\u0e40\\u0e2d\\u0e47\\u0e21 ", 0, status);
// Thai numerals should always be separated from the text surrounding them
ADD_DATACHUNK(thaiLineSelection, "\\u0e04\\u0e48\\u0e32", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e40\\u0e07\\u0e34\\u0e19", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e1a\\u0e32\\u0e17", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e41\\u0e15\\u0e30", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e23\\u0e30\\u0e14\\u0e31\\u0e1a", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e53\\u0e59", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e1a\\u0e32\\u0e17 ", 0, status);
// Thai text should interact correctly with punctuation and symbols
ADD_DATACHUNK(thaiLineSelection, "\\u0e44\\u0e2d\\u0e1a\\u0e35\\u0e40\\u0e2d\\u0e47\\u0e21", 0, status);
// ADD_DATACHUNK(thaiLineSelection, "(\\u0e1b\\u0e23\\u0e30\\u0e40\\u0e17\\u0e28", 0, status);
// ADD_DATACHUNK(thaiLineSelection, "\\u0e44\\u0e17\\u0e22)", 0, status);
ADD_DATACHUNK(thaiLineSelection, "(\\u0e1b\\u0e23\\u0e30\\u0e40\\u0e17\\u0e28\\u0e44\\u0e17\\u0e22)", 0, status);
// I believe the commented-out reading above to be the correct one, but this is what passes with our current dictionary
ADD_DATACHUNK(thaiLineSelection, "\\u0e08\\u0e33\\u0e01\\u0e31\\u0e14", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e40\\u0e1b\\u0e34\\u0e14", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e15\\u0e31\\u0e27\"", 0, status);
*/
// The Unicode Linebreak TR says do not break before or after quotes.
// So this test is changed ot not break around the quote.
// TODO: should Thai break around the around the quotes, like the original behavior here?
// ADD_DATACHUNK(thaiLineSelection, "\\u0e2e\\u0e32\\u0e23\\u0e4c\\u0e14\\u0e14\\u0e34\\u0e2a\\u0e01\\u0e4c\"", 0, status);
// ADD_DATACHUNK(thaiLineSelection, "\\u0e23\\u0e38\\u0e48\\u0e19", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e2e\\u0e32\\u0e23\\u0e4c\\u0e14\\u0e14\\u0e34\\u0e2a\\u0e01\\u0e4c\""
"\\u0e23\\u0e38\\u0e48\\u0e19", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e43\\u0e2b\\u0e21\\u0e48", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e40\\u0e14\\u0e37\\u0e2d\\u0e19\\u0e21\\u0e34.", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e22.", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e19\\u0e35\\u0e49", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e23\\u0e32\\u0e04\\u0e32", 0, status);
ADD_DATACHUNK(thaiLineSelection, "$200", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e40\\u0e17\\u0e48\\u0e32", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e19\\u0e31\\u0e49\\u0e19 ", 0, status);
ADD_DATACHUNK(thaiLineSelection, "(\"\\u0e2e\\u0e32\\u0e23\\u0e4c\\u0e14\\u0e14\\u0e34\\u0e2a\\u0e01\\u0e4c\").", 0, status);
RuleBasedBreakIterator* e = (RuleBasedBreakIterator *)BreakIterator::createLineInstance(Locale("th"), status);
if (U_FAILURE(status))
{
errln("Failed to create the BreakIterator for Thai locale in TestMixedThaiLineBreak.\n");
return;
}
generalIteratorTest(*e, thaiLineSelection);
delete e;
}
void RBBITest::TestMaiyamok()
{
UErrorCode status = U_ZERO_ERROR;
BITestData thaiLineSelection(status);
ADD_DATACHUNK(thaiLineSelection, NULL, 0, status); // Break at start of data
// the Thai maiyamok character is a shorthand symbol that means "repeat the previous
// word". Instead of appearing as a word unto itself, however, it's kept together
// with the word before it
ADD_DATACHUNK(thaiLineSelection, "\\u0e44\\u0e1b\\u0e46", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e21\\u0e32\\u0e46", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e23\\u0e30\\u0e2b\\u0e27\\u0e48\\u0e32\\u0e07", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e01\\u0e23\\u0e38\\u0e07\\u0e40\\u0e17\\u0e1e", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e41\\u0e25\\u0e30", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e40\\u0e03\\u0e35\\u0e22\\u0e07", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e43\\u0e2b\\u0e21\\u0e48", 0, status);
RuleBasedBreakIterator* e = (RuleBasedBreakIterator *)BreakIterator::createLineInstance(
Locale("th"), status);
if (U_FAILURE(status))
{
errln("Failed to create the BreakIterator for Thai locale in TestMaiyamok.\n");
return;
}
generalIteratorTest(*e, thaiLineSelection);
delete e;
}
void RBBITest::TestThaiWordBreak() {
UErrorCode status = U_ZERO_ERROR;
BITestData thaiWordSelection(status);
ADD_DATACHUNK(thaiWordSelection, NULL, 0, status); // Break at start of data
ADD_DATACHUNK(thaiWordSelection, "\\u0E1A\\u0E17", 0, status); //2
ADD_DATACHUNK(thaiWordSelection, "\\u0E17\\u0E35\\u0E48", 0, status); //5
ADD_DATACHUNK(thaiWordSelection, "\\u0E51", 0, status); //6
ADD_DATACHUNK(thaiWordSelection, "\\u0E1E\\u0E32\\u0E22\\u0E38", 0, status); //10
ADD_DATACHUNK(thaiWordSelection, "\\u0E44\\u0E0B\\u0E42\\u0E04\\u0E25\\u0E19", 0, status); //16
ADD_DATACHUNK(thaiWordSelection, "\\u000D\\u000A", 0, status); //18
// This is the correct result
//ADD_DATACHUNK(thaiWordSelection, "\\u0E42\\u0E14\\u0E42\\u0E23\\u0E18\\u0E35", 0, status); //24
//ADD_DATACHUNK(thaiWordSelection, "\\u0E2D\\u0E32\\u0E28\\u0E31\\u0E22", 0, status); //29
// and this is what the dictionary does...
ADD_DATACHUNK(thaiWordSelection, "\\u0E42\\u0E14", 0, status); // 20
ADD_DATACHUNK(thaiWordSelection, "\\u0E42\\u0E23\\u0E18\\u0E35\\u0E2D\\u0E32\\u0E28\\u0E31\\u0E22", 0, status); //29
ADD_DATACHUNK(thaiWordSelection, "\\u0E2D\\u0E22\\u0E39\\u0E48", 0, status); //33
// This is the correct result
//ADD_DATACHUNK(thaiWordSelection, "\\u0E17\\u0E48\\u0E32\\u0E21", 0, status); //37
//ADD_DATACHUNK(thaiWordSelection, "\\u0E01\\u0E25\\u0E32\\u0E07", 0, status); //41
// and this is what the dictionary does
ADD_DATACHUNK(thaiWordSelection, "\\u0E17\\u0E48\\u0E32\\u0E21\\u0E01\\u0E25\\u0E32\\u0E07", 0, status); //41
ADD_DATACHUNK(thaiWordSelection, "\\u0E17\\u0E38\\u0E48\\u0E07", 0, status); //45
ADD_DATACHUNK(thaiWordSelection, "\\u0E43\\u0E2B\\u0E0D\\u0E48", 0, status); //49
ADD_DATACHUNK(thaiWordSelection, "\\u0E43\\u0E19", 0, status); //51
// This is the correct result
//ADD_DATACHUNK(thaiWordSelection, "\\u0E41\\u0E04\\u0E19\\u0E0B\\u0E31\\u0E2A", 0, status); //57
//ADD_DATACHUNK(thaiWordSelection, "\\u0E01\\u0E31\\u0E1A", 0, status); //60
// and this is what the dictionary does
ADD_DATACHUNK(thaiWordSelection, "\\u0E41\\u0E04\\u0E19", 0, status); // 54
ADD_DATACHUNK(thaiWordSelection, "\\u0E0B\\u0E31\\u0E2A\\u0E01\\u0E31\\u0E1A", 0, status); //60
ADD_DATACHUNK(thaiWordSelection, "\\u0E25\\u0E38\\u0E07", 0, status); //63
// This is the correct result
//ADD_DATACHUNK(thaiWordSelection, "\\u0E40\\u0E2E\\u0E19\\u0E23\\u0E35", 0, status); //68
//ADD_DATACHUNK(thaiWordSelection, "\\u0E0A\\u0E32\\u0E27", 0, status); //71
//ADD_DATACHUNK(thaiWordSelection, "\\u0E44\\u0E23\\u0E48", 0, status); //74
//ADD_DATACHUNK(thaiWordSelection, "\\u0E41\\u0E25\\u0E30", 0, status); //77
// and this is what the dictionary does
ADD_DATACHUNK(thaiWordSelection, "\\u0E40\\u0E2E", 0, status); // 65
ADD_DATACHUNK(thaiWordSelection, "\\u0E19\\u0E23\\u0E35\\u0E0A\\u0E32\\u0E27\\u0E44\\u0E23\\u0E48\\u0E41\\u0E25\\u0E30", 0, status); //77
RuleBasedBreakIterator* e = (RuleBasedBreakIterator *)BreakIterator::createWordInstance(
Locale("th"), status);
if (U_FAILURE(status))
{
errln("Failed to create the BreakIterator for Thai locale in TestThaiWordBreak.\n");
return;
}
generalIteratorTest(*e, thaiWordSelection);
delete e;
}
//---------------------------------------------
// runIndexedTest
//---------------------------------------------
void RBBITest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* /*par*/ )
{
if (exec) logln("TestSuite RuleBasedBreakIterator: ");
switch (index) {
case 0: name = "TestDefaultRuleBasedCharacterIteration";
if(exec) TestDefaultRuleBasedCharacterIteration(); break;
case 1: name = "TestExtended";
if(exec) TestExtended(); break;
case 2: name = "TestDefaultRuleBasedWordIteration";
if(exec) TestDefaultRuleBasedWordIteration(); break;
case 3: name = "";
break;
case 4: name = "TestHindiCharacterBreak";
if(exec) TestHindiCharacterBreak(); break;
case 5: name = "TestHindiWordBreak";
if(exec) TestHindiWordBreak(); break;
case 6: name = "TestTitleBreak";
if(exec) TestTitleBreak(); break;
case 7: name = "TestStatusReturn";
if(exec) TestStatusReturn(); break;
case 8: name = "TestLineBreakData";
if(exec) TestLineBreakData(); break;
case 9: name = "TestSentenceInvariants";
if(exec) TestSentenceInvariants(); break;
case 10: name = "TestCharacterInvariants";
if(exec) TestCharacterInvariants(); break;
case 11: name = "TestWordInvariants";
if(exec) TestWordInvariants(); break;
case 12: name = "TestEmptyString";
if(exec) TestEmptyString(); break;
case 13: name = "TestGetAvailableLocales";
if(exec) TestGetAvailableLocales(); break;
case 14: name = "TestGetDisplayName";
if(exec) TestGetDisplayName(); break;
case 15: name = "TestEndBehaviour";
if(exec) TestEndBehaviour(); break;
case 16: name = "TestBug4153072";
if(exec) TestBug4153072(); break;
case 17: name = "TestJapaneseLineBreak";
if(exec) TestJapaneseLineBreak(); break;
case 18: name = "TestThaiLineBreak";
if(exec) TestThaiLineBreak(); break;
case 19: name = "TestMixedThaiLineBreak";
if(exec) TestMixedThaiLineBreak(); break;
case 20: name = "TestMaiyamok";
if(exec) TestMaiyamok(); break;
case 21: name = "TestThaiWordBreak";
if(exec) TestThaiWordBreak(); break;
// case 7: name = "TestHindiCharacterWrapping()";
// if(exec) TestHindiCharacterWrapping(); break;
// case 8: name = "TestCustomRuleBasedWordIteration";
// if(exec) TestCustomRuleBasedWordIteration(); break;
// case 9: name = "TestAbbrRuleBasedWordIteration";
// if(exec) TestAbbrRuleBasedWordIteration(); break;
// case 10: name = "TestTeluguRuleBasedCharacterIteration";
// if(exec) TestTeluguRuleBasedCharacterIteration(); break;
// case 11: name = "TestCustomRuleBasedCharacterIteration";
// if(exec) TestCustomRuleBasedCharacterIteration(); break;
default: name = ""; break; //needed to end loop
}
}
//----------------------------------------------------------------------------
//
// generalIteratorTest Given a break iterator and a set of test data,
// Run the tests and report the results.
//
//----------------------------------------------------------------------------
void RBBITest::generalIteratorTest(RuleBasedBreakIterator& bi, BITestData &td)
{
bi.setText(td.fDataToBreak);
testFirstAndNext(bi, td);
testLastAndPrevious(bi, td);
testFollowing(bi, td);
testPreceding(bi, td);
testIsBoundary(bi, td);
doMultipleSelectionTest(bi, td);
}
//
// testFirstAndNext. Run the iterator forwards in the obvious first(), next()
// kind of loop.
//
void RBBITest::testFirstAndNext(RuleBasedBreakIterator& bi, BITestData &td)
{
UErrorCode status = U_ZERO_ERROR;
int32_t p;
int32_t lastP = -1;
int32_t tag;
logln("Test first and next");
bi.setText(td.fDataToBreak);
td.clearResults();
for (p=bi.first(); p!=RuleBasedBreakIterator::DONE; p=bi.next()) {
td.fActualBreakPositions.addElement(p, status); // Save result.
tag = bi.getRuleStatus();
td.fActualTags.addElement(tag, status);
if (p <= lastP) {
// If the iterator is not making forward progress, stop.
// No need to raise an error here, it'll be detected in the normal check of results.
break;
}
lastP = p;
}
td.checkResults("testFirstAndNext", this);
}
//
// TestLastAndPrevious. Run the iterator backwards, starting with last().
//
void RBBITest::testLastAndPrevious(RuleBasedBreakIterator& bi, BITestData &td)
{
UErrorCode status = U_ZERO_ERROR;
int32_t p;
int32_t lastP = 0x7ffffffe;
int32_t tag;
logln("Test first and next");
bi.setText(td.fDataToBreak);
td.clearResults();
for (p=bi.last(); p!=RuleBasedBreakIterator::DONE; p=bi.previous()) {
// Save break position. Insert it at start of vector of results, shoving
// already-saved results further towards the end.
td.fActualBreakPositions.insertElementAt(p, 0, status);
// bi.previous(); // TODO: Why does this fix things up????
// bi.next();
tag = bi.getRuleStatus();
td.fActualTags.insertElementAt(tag, 0, status);
if (p >= lastP) {
// If the iterator is not making progress, stop.
// No need to raise an error here, it'll be detected in the normal check of results.
break;
}
lastP = p;
}
td.checkResults("testLastAndPrevious", this);
}
void RBBITest::testFollowing(RuleBasedBreakIterator& bi, BITestData &td)
{
UErrorCode status = U_ZERO_ERROR;
int32_t p;
int32_t tag;
int32_t lastP = -2; // A value that will never be returned as a break position.
// cannot be -1; that is returned for DONE.
int i;
logln("testFollowing():");
bi.setText(td.fDataToBreak);
td.clearResults();
// Save the starting point, since we won't get that out of following.
p = bi.first();
td.fActualBreakPositions.addElement(p, status); // Save result.
tag = bi.getRuleStatus();
td.fActualTags.addElement(tag, status);
for (i = 0; i <= td.fDataToBreak.length()+1; i++) {
p = bi.following(i);
if (p != lastP) {
if (p == RuleBasedBreakIterator::DONE) {
break;
}
// We've reached a new break position. Save it.
td.fActualBreakPositions.addElement(p, status); // Save result.
tag = bi.getRuleStatus();
td.fActualTags.addElement(tag, status);
lastP = p;
}
}
// The loop normally exits by means of the break in the middle.
// Make sure that the index was at the correct position for the break iterator to have
// returned DONE.
if (i != td.fDataToBreak.length()) {
errln("testFollowing(): iterator returned DONE prematurely.");
}
// Full check of all results.
td.checkResults("testFollowing", this);
}
void RBBITest::testPreceding(RuleBasedBreakIterator& bi, BITestData &td) {
UErrorCode status = U_ZERO_ERROR;
int32_t p;
int32_t tag;
int32_t lastP = 0x7ffffffe;
int i;
logln("testPreceding():");
bi.setText(td.fDataToBreak);
td.clearResults();
p = bi.last();
td.fActualBreakPositions.addElement(p, status);
tag = bi.getRuleStatus();
td.fActualTags.addElement(tag, status);
for (i = td.fDataToBreak.length(); i>=-1; i--) {
p = bi.preceding(i);
if (p != lastP) {
if (p == RuleBasedBreakIterator::DONE) {
break;
}
// We've reached a new break position. Save it.
td.fActualBreakPositions.insertElementAt(p, 0, status);
lastP = p;
tag = bi.getRuleStatus();
td.fActualTags.insertElementAt(tag, 0, status);
}
}
// The loop normally exits by means of the break in the middle.
// Make sure that the index was at the correct position for the break iterator to have
// returned DONE.
if (i != 0) {
errln("testPreceding(): iterator returned DONE prematurely.");
}
// Full check of all results.
td.checkResults("testPreceding", this);
}
void RBBITest::testIsBoundary(RuleBasedBreakIterator& bi, BITestData &td) {
UErrorCode status = U_ZERO_ERROR;
int i;
int32_t tag;
logln("testIsBoundary():");
bi.setText(td.fDataToBreak);
td.clearResults();
for (i = 0; i <= td.fDataToBreak.length(); i++) {
if (bi.isBoundary(i)) {
td.fActualBreakPositions.addElement(i, status); // Save result.
tag = bi.getRuleStatus();
td.fActualTags.addElement(tag, status);
}
}
td.checkResults("testIsBoundary: ", this);
}
void RBBITest::doMultipleSelectionTest(RuleBasedBreakIterator& iterator, BITestData &td)
{
iterator.setText(td.fDataToBreak);
RuleBasedBreakIterator* testIterator =(RuleBasedBreakIterator*)iterator.clone();
int32_t offset = iterator.first();
int32_t testOffset;
int32_t count = 0;
logln("doMultipleSelectionTest text of length: %d", td.fDataToBreak.length());
if (*testIterator != iterator)
errln("clone() or operator!= failed: two clones compared unequal");
do {
testOffset = testIterator->first();
testOffset = testIterator->next(count);
if (offset != testOffset)
errln(UnicodeString("next(n) and next() not returning consistent results: for step ") + count + ", next(n) returned " + testOffset + " and next() had " + offset);
if (offset != RuleBasedBreakIterator::DONE) {
count++;
offset = iterator.next();
if (offset != RuleBasedBreakIterator::DONE && *testIterator == iterator)
errln("operator== failed: Two unequal iterators compared equal.");
}
} while (offset != RuleBasedBreakIterator::DONE);
// now do it backwards...
offset = iterator.last();
count = 0;
do {
testOffset = testIterator->last();
testOffset = testIterator->next(count); // next() with a negative arg is same as previous
if (offset != testOffset)
errln(UnicodeString("next(n) and next() not returning consistent results: for step ") + count + ", next(n) returned " + testOffset + " and next() had " + offset);
if (offset != RuleBasedBreakIterator::DONE) {
count--;
offset = iterator.previous();
}
} while (offset != RuleBasedBreakIterator::DONE);
delete testIterator;
}
//--------------------------------------------------------------------------------------------
//
// Break Iterator Invariants Tests
//
//--------------------------------------------------------------------------------------------
void RBBITest::TestCharacterInvariants()
{
UErrorCode status = U_ZERO_ERROR;
BreakIterator *e = BreakIterator::createCharacterInstance(Locale::getDefault(), status);
if (U_FAILURE(status))
{
errln("Failed to create the BreakIterator for default locale in TestCharacterInvariants.\n");
return;
}
UnicodeString s = *cannedTestChars + CharsToUnicodeString("\\u1100\\u1101\\u1102\\u1160\\u1161\\u1162\\u11a8\\u11a9\\u11aa");
doBreakInvariantTest(*e, s);
s = *cannedTestChars + CharsToUnicodeString("\\u1100\\u1101\\u1102\\u1160\\u1161\\u1162\\u11a8\\u11a9\\u11aa");
doOtherInvariantTest(*e, s);
delete e;
}
void RBBITest::TestWordInvariants()
{
UErrorCode status = U_ZERO_ERROR;
BreakIterator *e = BreakIterator::createWordInstance(Locale::getDefault(), status);
if (U_FAILURE(status))
{
errln("Failed to create the BreakIterator for default locale in TestWordInvariants.\n");
return;
}
UnicodeString s = *cannedTestChars + CharsToUnicodeString("\',.\\u3041\\u3042\\u3043\\u309b\\u309c\\u30a1\\u30a2\\u30a3\\u4e00\\u4e01\\u4e02");
doBreakInvariantTest(*e, s);
s = *cannedTestChars + CharsToUnicodeString("\',.\\u3041\\u3042\\u3043\\u309b\\u309c\\u30a1\\u30a2\\u30a3\\u4e00\\u4e01\\u4e02");
doOtherInvariantTest(*e, s);
delete e;
}
void RBBITest::TestSentenceInvariants()
{
UErrorCode status = U_ZERO_ERROR;
BreakIterator *e = BreakIterator::createSentenceInstance(Locale::getDefault(), status);
if (U_FAILURE(status))
{
errln("Failed to create the BreakIterator for default locale in TestSentenceInvariant.\n");
return;
}
UnicodeString s = *cannedTestChars + CharsToUnicodeString(".,\\u3001\\u3002\\u3041\\u3042\\u3043\\ufeff");
doOtherInvariantTest(*e, s);
delete e;
}
void RBBITest::TestLineInvariants()
{
#if 0 // TestLineInvariants() needs to be updated to reflect TR 14 rules.
UErrorCode status = U_ZERO_ERROR;
BreakIterator *e = BreakIterator::createLineInstance(Locale::getUS(), status);
if (U_FAILURE(status))
{
errln("Failed to create the BreakIterator for default locale in TestLineInvariants.\n");
return;
}
UnicodeString s = CharsToUnicodeString(".,;:\\u3001\\u3002\\u3041\\u3042\\u3043\\u3044\\u3045\\u30a3\\u4e00\\u4e01\\u4e02");
UnicodeString testChars = *cannedTestChars + s;
doBreakInvariantTest(*e, testChars);
doOtherInvariantTest(*e, testChars);
int32_t errCount = 0, testCharsLen, noBreakLen, dashesLen;
int32_t i, j, k;
// in addition to the other invariants, a line-break iterator should make sure that:
// it doesn't break around the non-breaking characters,
// EXCEPT breaking after a space takes precedence over not breaking before
// an non-breaking char. So says TR 14.
UnicodeString noBreak = CharsToUnicodeString("\\u00a0\\u2007\\u2011\\ufeff");
UnicodeString work("aaa");
testCharsLen = testChars.length();
noBreakLen = noBreak.length();
for (i = 0; i < testCharsLen; i++) {
UChar c = testChars[i];
if (c == '\r' || c == '\n' || c == 0x2029 || c == 0x2028 || c == 0x0003 ||
u_charType(c) == U_CONTROL_CHAR) {
continue;
}
work[0] = c;
for (j = 0; j < noBreakLen; j++) {
work[1] = noBreak[j];
for (k = 0; k < testCharsLen; k++) {
work[2] = testChars[k];
e->setText(work);
for (int l = e->first(); l != BreakIterator::DONE; l = e->next()) {
UChar c1 = work[l - 1];
UChar c2 = work[l];
if (c1 == 0x20 && l == 1) {
continue;
}
if (l == 1 || l == 2) {
errln("Got break between U+" + UCharToUnicodeString(c1) +
" and U+" + UCharToUnicodeString(c2));
errCount++;
if (errCount >= 75)
return;
}
}
}
}
}
// it does break after hyphens (Rule 15B from TR 14
// (unless they're followed by a digit, a non-spacing mark,
// a currency symbol, a non-breaking space, or a line or paragraph separator
// or something of class BA, HY, NS, QU, GL, CL, EX, IS or SY from TR14 when the hyphen is /u002d
// This test is sufficiently screwed up that I'm largely disabling it. TODO: fix it. 06/12/2002 AGH
//
UnicodeString dashes = CharsToUnicodeString("-\\u00ad\\u2010\\u2012\\u2013\\u2014");
dashesLen = dashes.length();
for (i = 0; i < testCharsLen; i++) {
work[0] = testChars[i];
for (j = 0; j < dashesLen; j++) {
UChar c1 = work[1] = dashes[j];
for (k = 0; k < testCharsLen; k++) {
UChar c2 = work[2] = testChars[k];
int8_t type = u_charType(c2);
if (type == U_DECIMAL_DIGIT_NUMBER ||
type == U_OTHER_NUMBER ||
type == U_NON_SPACING_MARK ||
type == U_ENCLOSING_MARK ||
type == U_CURRENCY_SYMBOL ||
type == U_SPACE_SEPARATOR ||
type == U_DASH_PUNCTUATION ||
type == U_CONTROL_CHAR ||
type == U_FORMAT_CHAR ||
c2 == '\n' || c2 == '\r' || c2 == 0x2028 || c2 == 0x2029 ||
c2 == 0x0003 || c2 == 0x00a0 || c2 == 0x2007 || c2 == 0x2011 ||
c2 == 0xfeff)
{
continue;
}
// If c1 == hyphen-minus, and ...
if (c1 == 0x002d && (
c2 == 0x0021 || // !
c2 == 0x002c || // ,
c2 == 0x002d || // -
c2 == 0x002e || // . (TR 14 class IS)
c2 == 0x0029 || // )
c2 == 0x003a || // :
c2 == 0x003b || // ; (TR 14 class IS)
c2 == 0x005d || // ]
c2 == 0x007c || // | (TR 14 class BA, rule 15)
c2 == 0x007d || // }
c2 == 0x0903 || // Devanagari sign visarga, combining, what's it doing in this test?
c2 == 0x093E || // Devanagari , combining, what's it doing in this test?
c2 == 0x093F || // Devanagari , combining, what's it doing in this test?
c2 == 0x0940 || // Devanagari , combining, what's it doing in this test?
c2 == 0x0949 || // Devanagari , combining, what's it doing in this test?
c2 == 0x0f3b || // Tibetan closing bracket
c2 == 0x3001 || // CJK closing bracket
c2 == 0x3002 // CJK closing bracket
)) {
continue;
}
e->setText(work);
UBool saw2 = FALSE;
for (int l = e->first(); l != BreakIterator::DONE; l = e->next()) {
if (l == 2) {
saw2 = TRUE;
break;
}
}
if (!saw2) {
// TODO: This test is completely out of sync with the spec. Fix it.
// errln("Didn't get break between U+" + UCharToUnicodeString(work[1]) +
// " and U+" + UCharToUnicodeString(work[2]));
// errCount++;
// if (errCount >= 75)
// return;
}
}
}
}
delete e;
#endif
}
void RBBITest::doBreakInvariantTest(BreakIterator& tb, UnicodeString& testChars)
{
UnicodeString work("aaa");
int32_t errCount = 0, testCharsLen = testChars.length(), breaksLen;
// a break should always occur after CR (unless followed by LF), LF, PS, and LS
UnicodeString breaks = CharsToUnicodeString("\r\n\\u2029\\u2028");
int32_t i, j;
breaksLen = breaks.length();
for (i = 0; i < breaksLen; i++) {
UChar c1 = breaks[i];
work.setCharAt(1, c1);
for (j = 0; j < testCharsLen; j++) {
UChar c0 = testChars[j];
work.setCharAt(0, c0);
for (int k = 0; k < testCharsLen; k++) {
UChar c2 = testChars[k];
work.setCharAt(2, c2);
// if a cr is followed by lf, ps, ls or etx, don't do the check (that's
// not supposed to work)
if (c1 == '\r' && (c2 == '\n' || c2 == 0x2029
|| c2 == 0x2028 || c2 == 0x0003))
continue;
if (u_charType(c1) == U_CONTROL_CHAR &&
(u_charType(c2) == U_NON_SPACING_MARK ||
u_charType(c2) == U_ENCLOSING_MARK ||
u_charType(c2) == U_COMBINING_SPACING_MARK)
) {
// Combining marks don't combine with controls.
// TODO: enhance test to verify that the break actually occurs,
// not just ignore the case.
continue;
}
tb.setText(work);
UBool seen2 = FALSE;
for (int l = tb.first(); l != BreakIterator::DONE; l = tb.next()) {
if (l == 2) {
seen2 = TRUE;
break;
}
}
if (!seen2) {
errln("No Break between \\U%04x and \\U%04x", c1, c2);
errCount++;
if (errCount >= 75)
return;
}
}
}
}
}
void RBBITest::doOtherInvariantTest(BreakIterator& tb, UnicodeString& testChars)
{
UnicodeString work("a\r\na");
int32_t errCount = 0, testCharsLen = testChars.length();
int32_t i, j;
int8_t type;
// a break should never occur between CR and LF
for (i = 0; i < testCharsLen; i++) {
work.setCharAt(0, testChars[i]);
for (j = 0; j < testCharsLen; j++) {
work.setCharAt(3, testChars[j]);
tb.setText(work);
for (int32_t k = tb.first(); k != BreakIterator::DONE; k = tb.next())
if (k == 2) {
errln("Break between CR and LF in string U\\%04x U\\%04x U\\%04x U\\%04x",
work[0], work[1], work[2], work[3]);
errCount++;
if (errCount >= 75)
return;
}
}
}
// a break should never occur before a non-spacing mark, unless the preceding
// character is CR, LF, PS, or LS
// Or the general category == Control.
work.remove();
work += "aaaa";
for (i = 0; i < testCharsLen; i++) {
UChar c1 = testChars[i];
if (c1 == '\n' || c1 == '\r' || c1 == 0x2029 || c1 == 0x2028 || c1 == 0x0003 ||
u_charType(c1) == U_CONTROL_CHAR || u_charType(c1) == U_FORMAT_CHAR) {
continue;
}
work.setCharAt(1, c1);
for (j = 0; j < testCharsLen; j++) {
UChar c2 = testChars[j];
type = u_charType(c2);
if ((type != U_NON_SPACING_MARK) &&
(type != U_ENCLOSING_MARK)) {
continue;
}
work.setCharAt(2, c2);
tb.setText(work);
for (int k = tb.first(); k != BreakIterator::DONE; k = tb.next())
if (k == 2) {
//errln("Break between U+" + UCharToUnicodeString(work[1])
// + " and U+" + UCharToUnicodeString(work[2]));
errln("Unexpected Break between %6x and %6x", c1, c2);
errCount++;
if (errCount >= 75)
return;
}
}
}
}
//---------------------------------------------
//
// other tests
//
//---------------------------------------------
void RBBITest::TestEmptyString()
{
UnicodeString text = "";
UErrorCode status = U_ZERO_ERROR;
BITestData x(status);
ADD_DATACHUNK(x, "", 0, status); // Break at start of data
RuleBasedBreakIterator* bi = (RuleBasedBreakIterator *)BreakIterator::createLineInstance(Locale::getDefault(), status);
if (U_FAILURE(status))
{
errln("Failed to create the BreakIterator for default locale in TestEmptyString.\n");
return;
}
generalIteratorTest(*bi, x);
delete bi;
}
void RBBITest::TestGetAvailableLocales()
{
int32_t locCount = 0;
const Locale* locList = BreakIterator::getAvailableLocales(locCount);
if (locCount == 0)
errln("getAvailableLocales() returned an empty list!");
// Just make sure that it's returning good memory.
for (int32_t i = 0; i < locCount; ++i) {
logln(locList[i].getName());
}
}
//Testing the BreakIterator::getDisplayName() function
void RBBITest::TestGetDisplayName()
{
UnicodeString result;
BreakIterator::getDisplayName(Locale::getUS(), result);
if (Locale::getDefault() == Locale::getUS() && result != "English (United States)")
errln("BreakIterator::getDisplayName() failed: expected \"English (United States)\", got \""
+ result);
BreakIterator::getDisplayName(Locale::getFrance(), Locale::getUS(), result);
if (result != "French (France)")
errln("BreakIterator::getDisplayName() failed: expected \"French (France)\", got \""
+ result);
}
/**
* Test End Behaviour
* @bug 4068137
*/
void RBBITest::TestEndBehaviour()
{
UErrorCode status = U_ZERO_ERROR;
UnicodeString testString("boo.");
BreakIterator *wb = BreakIterator::createWordInstance(Locale::getDefault(), status);
if (U_FAILURE(status))
{
errln("Failed to create the BreakIterator for default locale in TestEndBehaviour.\n");
return;
}
wb->setText(testString);
if (wb->first() != 0)
errln("Didn't get break at beginning of string.");
if (wb->next() != 3)
errln("Didn't get break before period in \"boo.\"");
if (wb->current() != 4 && wb->next() != 4)
errln("Didn't get break at end of string.");
delete wb;
}
/*
* @bug 4153072
*/
void RBBITest::TestBug4153072() {
UErrorCode status = U_ZERO_ERROR;
BreakIterator *iter = BreakIterator::createWordInstance(Locale::getDefault(), status);
if (U_FAILURE(status))
{
errln("Failed to create the BreakIterator for default locale in TestBug4153072\n");
return;
}
UnicodeString str("...Hello, World!...");
int32_t begin = 3;
int32_t end = str.length() - 3;
UBool dummy;
StringCharacterIterator* textIterator = new StringCharacterIterator(str, begin, end, begin);
iter->adoptText(textIterator);
for (int index = -1; index < begin + 1; ++index) {
dummy = iter->isBoundary(index);
if (index < begin && dummy == TRUE) {
errln((UnicodeString)"Didn't handle preceeding correctly with offset = " + index +
" and begin index = " + begin);
}
}
delete iter;
}
/**
* Test Japanese Line Break
* @bug 4095322
*/
void RBBITest::TestJapaneseLineBreak()
{
// Change for Unicode TR 14: Punctuation characters with categories Pi and Pf do not count
// as opening and closing punctuation for line breaking.
// Also, \u30fc and \u30fe are not counted as hyphens. Remove these chars
// from these tests. 6-13-2002
//
UErrorCode status = U_ZERO_ERROR;
UnicodeString testString = CharsToUnicodeString("\\u4e00x\\u4e8c");
UnicodeString precedingChars = CharsToUnicodeString(
//"([{\\u00ab$\\u00a5\\u00a3\\u00a4\\u2018\\u201a\\u201c\\u201e\\u201b\\u201f");
"([{$\\u00a5\\u00a3\\u00a4\\u201a\\u201e");
UnicodeString followingChars = CharsToUnicodeString(
// ")]}\\u00bb!%,.\\u3001\\u3002\\u3063\\u3083\\u3085\\u3087\\u30c3\\u30e3\\u30e5\\u30e7\\u30fc"
")]}!%,.\\u3001\\u3002\\u3063\\u3083\\u3085\\u3087\\u30c3\\u30e3\\u30e5\\u30e7"
// ":;\\u309b\\u309c\\u3005\\u309d\\u309e\\u30fd\\u30fe\\u2019\\u201d\\u00b0\\u2032\\u2033\\u2034"
":;\\u309b\\u309c\\u3005\\u309d\\u309e\\u30fd\\u00b0\\u2032\\u2033\\u2034"
"\\u2030\\u2031\\u2103\\u2109\\u00a2\\u0300\\u0301\\u0302");
BreakIterator *iter = BreakIterator::createLineInstance(Locale::getJapan(), status);
int32_t i;
if (U_FAILURE(status))
{
errln("Failed to create the BreakIterator for Japanese locale in TestJapaneseLineBreak.\n");
return;
}
for (i = 0; i < precedingChars.length(); i++) {
testString.setCharAt(1, precedingChars[i]);
iter->setText(testString);
int32_t j = iter->first();
if (j != 0)
errln("ja line break failure: failed to start at 0");
j = iter->next();
if (j != 1)
errln("ja line break failure: failed to stop before '" + UCharToUnicodeString(precedingChars[i])
+ "' (" + ((int)(precedingChars[i])) + ")");
j = iter->next();
if (j != 3)
errln("ja line break failure: failed to skip position after '" + UCharToUnicodeString(precedingChars[i])
+ "' (" + ((int)(precedingChars[i])) + ")");
}
for (i = 0; i < followingChars.length(); i++) {
testString.setCharAt(1, followingChars[i]);
iter->setText(testString);
int j = iter->first();
if (j != 0)
errln("ja line break failure: failed to start at 0");
j = iter->next();
if (j != 2)
errln("ja line break failure: failed to skip position before '" + UCharToUnicodeString(followingChars[i])
+ "' (" + ((int)(followingChars[i])) + ")");
j = iter->next();
if (j != 3)
errln("ja line break failure: failed to stop after '" + UCharToUnicodeString(followingChars[i])
+ "' (" + ((int)(followingChars[i])) + ")");
}
delete iter;
}
//------------------------------------------------------------------------------
//
// RBBITest::Extended Run RBBI Tests from an external test data file
//
//------------------------------------------------------------------------------
struct TestParams {
BreakIterator *bi;
UnicodeString dataToBreak;
UVector32 *expectedBreaks;
UVector32 *srcLine;
UVector32 *srcCol;
};
void RBBITest::executeTest(TestParams *t) {
int32_t bp;
int32_t prevBP;
int32_t i;
t->bi->setText(t->dataToBreak);
//
// Run the iterator forward
//
prevBP = -1;
for (bp = t->bi->first(); bp != BreakIterator::DONE; bp = t->bi->next()) {
if (prevBP == bp) {
// Fail for lack of forward progress.
errln("Forward Iteration, no forward progress. Break Pos=%4d File line,col=%4d,%4d",
bp, t->srcLine->elementAti(bp), t->srcCol->elementAti(bp));
break;
}
// Check that there were we didn't miss an expected break between the last one
// and this one.
for (i=prevBP+1; i<bp; i++) {
if (t->expectedBreaks->elementAti(i) != 0) {
errln("Forward Itertion, break expected, but not found. Pos=%4d File line,col= %4d,%4d",
i, t->srcLine->elementAti(i), t->srcCol->elementAti(i));
}
}
// Check that the break we did find was expected
if (t->expectedBreaks->elementAti(bp) == 0) {
errln("Forward Itertion, break found, but not expected. Pos=%4d File line,col= %4d,%4d",
bp, t->srcLine->elementAti(bp), t->srcCol->elementAti(bp));
} else {
// The break was expected.
// Check that the {nnn} tag value is correct.
int32_t expectedTagVal = t->expectedBreaks->elementAti(bp);
if (expectedTagVal == -1) {
expectedTagVal = 0;
}
int32_t rs = ((RuleBasedBreakIterator *)t->bi)->getRuleStatus();
if (rs != expectedTagVal) {
errln("Incorrect status for break. Pos=%4d File line,col= %4d,%4d.\n"
" Actual, Expected status = %4d, %4d",
bp, t->srcLine->elementAti(bp), t->srcCol->elementAti(bp), rs, expectedTagVal);
}
}
prevBP = bp;
}
// Verify that there were no missed expected breaks after the last one found
for (i=prevBP+1; i<t->expectedBreaks->size(); i++) {
if (t->expectedBreaks->elementAti(i) != 0) {
errln("Forward Itertion, break expected, but not found. Pos=%4d File line,col= %4d,%4d",
i, t->srcLine->elementAti(i), t->srcCol->elementAti(i));
}
}
//
// Run the iterator backwards, verify that the same breaks are found.
//
prevBP = t->dataToBreak.length()+2; // start with a phony value for the last break pos seen.
for (bp = t->bi->last(); bp != BreakIterator::DONE; bp = t->bi->previous()) {
if (prevBP == bp) {
// Fail for lack of progress.
errln("Reverse Iteration, no progress. Break Pos=%4d File line,col=%4d,%4d",
bp, t->srcLine->elementAti(bp), t->srcCol->elementAti(bp));
break;
}
// Check that there were we didn't miss an expected break between the last one
// and this one. (UVector returns zeros for index out of bounds.)
for (i=prevBP-1; i>bp; i--) {
if (t->expectedBreaks->elementAti(i) != 0) {
errln("Reverse Itertion, break expected, but not found. Pos=%4d File line,col= %4d,%4d",
i, t->srcLine->elementAti(i), t->srcCol->elementAti(i));
}
}
// Check that the break we did find was expected
if (t->expectedBreaks->elementAti(bp) == 0) {
errln("Reverse Itertion, break found, but not expected. Pos=%4d File line,col= %4d,%4d",
bp, t->srcLine->elementAti(bp), t->srcCol->elementAti(bp));
} else {
// The break was expected.
// Check that the {nnn} tag value is correct.
int32_t expectedTagVal = t->expectedBreaks->elementAti(bp);
if (expectedTagVal == -1) {
expectedTagVal = 0;
}
int32_t rs = ((RuleBasedBreakIterator *)t->bi)->getRuleStatus();
if (rs != expectedTagVal) {
errln("Incorrect status for break. Pos=%4d File line,col= %4d,%4d.\n"
" Actual, Expected status = %4d, %4d",
bp, t->srcLine->elementAti(bp), t->srcCol->elementAti(bp), rs, expectedTagVal);
}
}
prevBP = bp;
}
// Verify that there were no missed breaks prior to the last one found
for (i=prevBP-1; i>=0; i--) {
if (t->expectedBreaks->elementAti(i) != 0) {
errln("Forward Itertion, break expected, but not found. Pos=%4d File line,col= %4d,%4d",
i, t->srcLine->elementAti(i), t->srcCol->elementAti(i));
}
}
}
void RBBITest::TestExtended() {
UErrorCode status = U_ZERO_ERROR;
Locale locale = Locale::getDefault();
UnicodeString rules;
TestParams tp;
tp.bi = NULL;
tp.expectedBreaks = new UVector32(status);
tp.srcLine = new UVector32(status);
tp.srcCol = new UVector32(status);
//
// Open and read the test data file.
//
const char *testDataDirectory = loadTestData(status);
UnicodeString tdd(testDataDirectory);
// TODO: Remove regexp dependency
tdd = RegexMatcher("([/\\\\])out[/\\\\]testdata", tdd, 0, status).
replaceFirst("$1rbbitst.txt", status);
int len;
UChar *testFile = ReadAndConvertFile((const char *)CharString(tdd), len, status);
//
// Put the test data into a UnicodeString
//
UnicodeString testString(FALSE, testFile, len);
enum EParseState{
PARSE_COMMENT,
PARSE_TAG,
PARSE_RULE,
PARSE_DATA,
PARSE_NUM
}
parseState = PARSE_TAG;
EParseState savedState = PARSE_TAG;
const UChar CH_LF = 0x0a;
const UChar CH_CR = 0x0d;
const UChar CH_HASH = 0x23;
const UChar CH_PERIOD = 0x2e;
const UChar CH_LT = 0x3c;
const UChar CH_GT = 0x3e;
const UChar CH_BACKSLASH = 0x5c;
const UChar CH_BULLET = 0x2022;
int32_t lineNum = 1;
int32_t colStart = 0;
int32_t column = 0;
int32_t charIdx = 0;
int32_t tagValue = 0; // The numeric value of a <nnn> tag.
for (charIdx = 0; charIdx < len; ) {
UChar c = testString.charAt(charIdx);
charIdx++;
if (c == CH_CR && charIdx<len && testString.charAt(charIdx) == CH_LF) {
// treat CRLF as a unit
c = CH_LF;
charIdx++;
}
if (c == CH_LF || c == CH_CR) {
lineNum++;
colStart = charIdx;
}
column = charIdx - colStart + 1;
switch (parseState) {
case PARSE_COMMENT:
if (c == 0x0a || c == 0x0d) {
parseState = savedState;
}
break;
case PARSE_TAG:
{
if (c == CH_HASH) {
parseState = PARSE_COMMENT;
savedState = PARSE_TAG;
break;
}
if (u_isUWhiteSpace(c)) {
break;
}
if (testString.compare(charIdx-1, 6, "<word>") == 0) {
delete tp.bi;
tp.bi = BreakIterator::createWordInstance(locale, status);
charIdx += 5;
break;
}
if (testString.compare(charIdx-1, 6, "<char>") == 0) {
delete tp.bi;
tp.bi = BreakIterator::createCharacterInstance(locale, status);
charIdx += 5;
break;
}
if (testString.compare(charIdx-1, 6, "<line>") == 0) {
delete tp.bi;
tp.bi = BreakIterator::createLineInstance(locale, status);
charIdx += 5;
break;
}
if (testString.compare(charIdx-1, 6, "<sent>") == 0) {
delete tp.bi;
tp.bi = BreakIterator::createSentenceInstance(locale, status);
charIdx += 5;
break;
}
if (testString.compare(charIdx-1, 7, "<title>") == 0) {
delete tp.bi;
tp.bi = BreakIterator::createTitleInstance(locale, status);
charIdx += 6;
break;
}
if (testString.compare(charIdx-1, 6, "<data>") == 0) {
parseState = PARSE_DATA;
charIdx += 5;
tp.dataToBreak = "";
tp.expectedBreaks->removeAllElements();
tp.srcCol ->removeAllElements();
tp.srcLine->removeAllElements();
break;
}
errln("line %d: Tag expected in test file.", lineNum);
parseState = PARSE_COMMENT;
savedState = PARSE_DATA;
}
break;
case PARSE_DATA:
if (c == CH_BULLET) {
int32_t breakIdx = tp.dataToBreak.length();
tp.expectedBreaks->setSize(breakIdx+1);
tp.expectedBreaks->setElementAt(-1, breakIdx);
tp.srcLine->setSize(breakIdx+1);
tp.srcLine->setElementAt(lineNum, breakIdx);
tp.srcCol ->setSize(breakIdx+1);
tp.srcCol ->setElementAt(column, breakIdx);
break;
}
if (testString.compare(charIdx-1, 7, "</data>") == 0) {
// Add final entry to mappings from break location to source file position.
// Need one extra because last break position returned is after the
// last char in the data, not at the last char.
tp.srcLine->addElement(lineNum, status);
tp.srcCol ->addElement(column, status);
parseState = PARSE_TAG;
charIdx += 7;
// RUN THE TEST!
executeTest(&tp);
break;
}
if (testString.compare(charIdx-1, 2, "<>") == 0) {
charIdx++;
int32_t breakIdx = tp.dataToBreak.length();
tp.expectedBreaks->setSize(breakIdx+1);
tp.expectedBreaks->setElementAt(-1, breakIdx);
tp.srcLine->setSize(breakIdx+1);
tp.srcLine->setElementAt(lineNum, breakIdx);
tp.srcCol ->setSize(breakIdx+1);
tp.srcCol ->setElementAt(column, breakIdx);
break;
}
if (c == CH_LT) {
tagValue = 0;
parseState = PARSE_NUM;
break;
}
if (c == CH_HASH && column==3) { // TODO: why is column off so far?
parseState = PARSE_COMMENT;
savedState = PARSE_DATA;
break;
}
if (c == CH_BACKSLASH) {
// Check for \ at end of line, a line continuation.
// Advance over (discard) the newline
UChar32 cp = testString.char32At(charIdx);
if (cp == CH_CR && charIdx<len && testString.charAt(charIdx+1) == CH_LF) {
// We have a CR LF
// Need an extra increment of the input ptr to move over both of them
charIdx++;
}
if (cp == CH_LF || cp == CH_CR) {
lineNum++;
colStart = charIdx;
charIdx++;
break;
}
// Let unescape handle the back slash.
cp = testString.unescapeAt(charIdx);
if (cp != -1) {
// Escape sequence was recognized. Insert the char
// into the test data.
tp.dataToBreak.append(cp);
while (tp.dataToBreak.length() > tp.srcLine->size()) {
tp.srcLine->addElement(lineNum, status);
tp.srcCol ->addElement(column, status);
}
break;
}
// Not a recognized backslash escape sequence.
// Take the next char as a literal.
// TODO: Should this be an error?
c = testString.charAt(charIdx);
charIdx = testString.moveIndex32(charIdx, 1);
}
// Normal, non-escaped data char.
tp.dataToBreak.append(c);
// Save the mapping from offset in the data to line/column numbers in
// the original input file. Will be used for better error messages only.
// If there's an expected break before this char, the slot in the mapping
// vector will already be set for this char; don't overwrite it.
if (tp.dataToBreak.length() > tp.srcLine->size()) {
tp.srcLine->addElement(lineNum, status);
tp.srcCol ->addElement(column, status);
}
break;
case PARSE_NUM:
// We are parsing an expected numeric tag value, like <1234>,
// within a chunk of data.
if (u_isUWhiteSpace(c)) {
break;
}
if (c == CH_GT) {
// Finished the number. Add the info to the expected break data,
// and switch parse state back to doing plain data.
parseState = PARSE_DATA;
if (tagValue == 0) {
tagValue = -1;
}
int32_t breakIdx = tp.dataToBreak.length();
tp.expectedBreaks->setSize(breakIdx+1);
tp.expectedBreaks->setElementAt(tagValue, breakIdx);
tp.srcLine->setSize(breakIdx+1);
tp.srcLine->setElementAt(lineNum, breakIdx);
tp.srcCol ->setSize(breakIdx+1);
tp.srcCol ->setElementAt(column, breakIdx);
break;
}
if (u_isdigit(c)) {
tagValue = tagValue*10 + u_charDigitValue(c);
break;
}
errln("Syntax Error in test file at line %d, col %d",
lineNum, column);
parseState = PARSE_COMMENT;
break;
}
if (U_FAILURE(status)) {
errln("ICU Error %s while parsing test file at line %d.",
u_errorName(status), lineNum);
status = U_ZERO_ERROR;
}
}
delete tp.bi;
delete tp.expectedBreaks;
delete tp.srcLine;
delete tp.srcCol;
delete [] testFile;
}
//-------------------------------------------------------------------------------
//
// ReadAndConvertFile Read a text data file, convert it to UChars, and
// return the datain one big UChar * buffer, which the caller must delete.
//
// TODO: This is a clone of RegexTest::ReadAndConvertFile.
// Move this function to some common place.
//
//--------------------------------------------------------------------------------
UChar *RBBITest::ReadAndConvertFile(const char *fileName, int &ulen, UErrorCode &status) {
UChar *retPtr = NULL;
char *fileBuf = NULL;
UConverter* conv = NULL;
FILE *f = NULL;
ulen = 0;
if (U_FAILURE(status)) {
return retPtr;
}
//
// Open the file.
//
f = fopen(fileName, "rb");
if (f == 0) {
errln("Error opening test data file %s\n", fileName);
goto cleanUpAndReturn;
}
//
// Read it in
//
int fileSize;
int amt_read;
fseek( f, 0, SEEK_END);
fileSize = ftell(f);
fileBuf = new char[fileSize];
fseek(f, 0, SEEK_SET);
amt_read = fread(fileBuf, 1, fileSize, f);
if (amt_read != fileSize || fileSize <= 0) {
errln("Error reading test data file.");
goto cleanUpAndReturn;
}
//
// Look for a Unicode Signature (BOM) on the data just read
//
int32_t signatureLength;
const char * fileBufC;
const char* encoding;
fileBufC = fileBuf;
encoding = ucnv_detectUnicodeSignature(
fileBuf, fileSize, &signatureLength, &status);
if(encoding!=NULL ){
fileBufC += signatureLength;
fileSize -= signatureLength;
}
//
// Open a converter to take the rule file to UTF-16
//
conv = ucnv_open(encoding, &status);
if (U_FAILURE(status)) {
goto cleanUpAndReturn;
}
//
// Convert the rules to UChar.
// Preflight first to determine required buffer size.
//
ulen = ucnv_toUChars(conv,
NULL, // dest,
0, // destCapacity,
fileBufC,
fileSize,
&status);
if (status == U_BUFFER_OVERFLOW_ERROR) {
// Buffer Overflow is expected from the preflight operation.
status = U_ZERO_ERROR;
retPtr = new UChar[ulen+1];
ucnv_toUChars(conv,
retPtr, // dest,
ulen+1,
fileBufC,
fileSize,
&status);
}
cleanUpAndReturn:
fclose(f);
delete fileBuf;
ucnv_close(conv);
if (U_FAILURE(status)) {
errln("ucnv_toUChars: ICU Error \"%s\"\n", u_errorName(status));
delete retPtr;
retPtr = 0;
ulen = 0;
};
return retPtr;
}
//--------------------------------------------------------------------------------------------
//
// Exhaustive Tests, using Unicode Data Files.
//
//--------------------------------------------------------------------------------------------
//
// Token level scanner for the Unicode Line Break Test Data file.
// Return the next token, as follows:
// >= 0: a UChar32 character, scanned from hex in the file.
// -1: a break position, a division sign in the file.
// -2: end of rule. A new line in the file.
// -3: end of file. No more rules.
// -4: Error
//
// The scanner
// strips comments, ('#' to end of line)
// Recognizes CR, CR/LF and LF as new lines.
// Skips over spaces and Xs (don't break here) in the data.
//
struct ScanState {
int32_t fPeekChar;
UBool fPeeked;
int32_t fLineNum;
FILE *fFile;
ScanState() :fPeeked(FALSE), fLineNum(0), fFile(NULL) {};
};
// Literal characters that are of interest. In hex to keep EBCDIC based machines happy.
// The data itself is latin-1 on all platforms.
static const int32_t chSpace = 0x20;
static const int32_t chTab = 0x09;
static const int32_t chCR = 0x0D;
static const int32_t chLF = 0x0A;
static const int32_t chHash = 0x23;
static const int32_t chMult = 0xD7;
static const int32_t chDivide = 0xF7;
static int32_t nextLBDToken(ScanState *s) {
int32_t c;
// Read characters from the input file until we get something interesting
// to return. The file is in latin-1 encoding.
for (;;) {
// Get the next character to look at,
if (s->fPeeked) {
c = s->fPeekChar;
s->fPeeked = FALSE;
} else {
c = getc(s->fFile);
}
// EOF. Return immediately.
if (c == EOF) {
return -3;
}
// Spaces. Treat the multiply sign as a space - it indicates a no-break position
// in the data, and the test program doesn't want to see them.
// Continue the next char loop, looking for something significant.
if (c == chSpace || c == chTab || c == chMult) {
continue;
}
// Divide sign. Indicates an expected break position.
if (c == chDivide) {
return -1;
}
// New Line Handling. Keep track of line number in the file, which in turn
// requires keeping track of CR/LF as a single new line.
if (c == chCR) {
s->fLineNum++;
s->fPeekChar = getc(s->fFile);
if (s->fPeekChar != chLF) {s->fPeeked = TRUE;};
return -2;
}
if (c == chLF) {
s->fLineNum++;
return -2;
}
// Comments. Consume everything up to the next new line.
if (c == chHash) {
do {
c = getc(s->fFile);
} while (!(c == EOF || c == chCR || c == chLF));
s->fPeekChar = c;
s->fPeeked = TRUE;
return nextLBDToken(s);
}
// Scan a hex character (UChar32) value.
if (u_digit(c, 16) >= 0) {
int32_t v = u_digit(c, 16);
for (;;) {
c = getc(s->fFile);
if (u_digit(c, 16) < 0) {break;};
v <<= 4;
v += u_digit(c, 16);
}
s->fPeekChar = c;
s->fPeeked = TRUE;
return v;
}
// Error. Character was something unexpected.
return -4;
}
}
void RBBITest::TestLineBreakData() {
UErrorCode status = U_ZERO_ERROR;
UnicodeString testString;
UVector expectedBreaks(status);
ScanState ss;
int32_t tok;
BreakIterator *bi = BreakIterator::createLineInstance(Locale::getDefault(), status);
if (U_FAILURE(status)) {
errln("Failure creating break iterator");
return;
}
const char * lbdfName = "LBTest.txt";
// Open the test data file.
// TODO: a proper way to handle this data.
ss.fFile = fopen(lbdfName, "rb");
if (ss.fFile == NULL) {
logln("Unable to open Line Break Test Data file. Skipping test.");
delete bi;
return;
}
// Loop once per line from the test data file.
for (;;) {
// Zero out test data from previous line.
testString.truncate(0);
expectedBreaks.removeAllElements();
// Read one test's (line's) worth of data from the file.
// Loop once per token on the input file line.
for(;;) {
tok = nextLBDToken(&ss);
// If we scanned a character number in the file.
// save it in the test data array.
if (tok >= 0) {
testString.append((UChar32)tok);
continue;
}
// If we scanned a break position in the data, record it.
if (tok == -1) {
expectedBreaks.addElement(testString.length(), status);
continue;
}
// If we scanned a new line, or EOF
// drop out of scan loop and run the test case.
if (tok == -2 || tok == -3) {break;};
// None of above. Error.
errln("Failure: Unrecognized data format, test file line %d", ss.fLineNum);
break;
}
// If this line from the test data file actually contained test data,
// run the test.
if (testString.length() > 0) {
int32_t pos; // Break Position in the test string
int32_t expectedI = 0; // Index of expected break position in vector of same.
int32_t expectedPos; // Expected break position (index into test string)
bi->setText(testString);
pos = bi->first(); // TODO: break iterators always return a match at pos 0.
pos = bi->next(); // Line Break TR says no match at position 0.
// Resolve.
for (; pos != BreakIterator::DONE; ) {
expectedPos = expectedBreaks.elementAti(expectedI);
if (pos < expectedPos) {
errln("Failure: Test file line %d, unexpected break found at position %d",
ss.fLineNum, pos);
break;
}
if (pos > expectedPos) {
errln("Failure: Test file line %d, failed to find break at position %d",
ss.fLineNum, expectedPos);
break;
}
pos = bi->next();
expectedI++;
}
}
// If we've hit EOF on the input file, we're done.
if (tok == -3) {
break;
}
}
fclose(ss.fFile);
delete bi;
}
#endif /* #if !UCONFIG_NO_BREAK_ITERATION */