Google Git
Sign in
skia / external / github.com / unicode-org / icu / refs/tags/release-62-rc / . / icu4c / source / test / intltest / rbbitst.cpp
blob: 5463dad247181dd96ddc4607b3bb168d216af527 [file] [log] [blame]
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/********************************************************************
* COPYRIGHT:
* Copyright (c) 1999-2016, International Business Machines Corporation and
* others. All Rights Reserved.
********************************************************************/
/************************************************************************
* Date Name Description
* 12/15/99 Madhu Creation.
* 01/12/2000 Madhu Updated for changed API and added new tests
************************************************************************/
#include "unicode/utypes.h"
#if !UCONFIG_NO_BREAK_ITERATION
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <utility>
#include <vector>
#include "unicode/brkiter.h"
#include "unicode/localpointer.h"
#include "unicode/numfmt.h"
#include "unicode/rbbi.h"
#if !UCONFIG_NO_REGULAR_EXPRESSIONS
#include "unicode/regex.h"
#endif
#include "unicode/schriter.h"
#include "unicode/uchar.h"
#include "unicode/utf16.h"
#include "unicode/ucnv.h"
#include "unicode/uniset.h"
#include "unicode/uscript.h"
#include "unicode/ustring.h"
#include "unicode/utext.h"
#include "charstr.h"
#include "cmemory.h"
#include "cstr.h"
#include "intltest.h"
#include "rbbitst.h"
#include "rbbidata.h"
#include "utypeinfo.h" // for 'typeid' to work
#include "uvector.h"
#include "uvectr32.h"
#if !UCONFIG_NO_FILTERED_BREAK_ITERATION
#include "unicode/filteredbrk.h"
#endif // !UCONFIG_NO_FILTERED_BREAK_ITERATION
#define TEST_ASSERT(x) {if (!(x)) { \
errln("Failure in file %s, line %d", __FILE__, __LINE__);}}
#define TEST_ASSERT_SUCCESS(errcode) { if (U_FAILURE(errcode)) { \
errcheckln(errcode, "Failure in file %s, line %d, status = \"%s\"", __FILE__, __LINE__, u_errorName(errcode));}}
//---------------------------------------------
// runIndexedTest
//---------------------------------------------
// Note: Before adding new tests to this file, check whether the desired test data can
// simply be added to the file testdata/rbbitest.txt. In most cases it can,
// it's much less work than writing a new test, diagnostic output in the event of failures
// is good, and the test data file will is shared with ICU4J, so eventually the test
// will run there as well, without additional effort.
void RBBITest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* params )
{
if (exec) logln("TestSuite RuleBasedBreakIterator: ");
fTestParams = params;
TESTCASE_AUTO_BEGIN;
#if !UCONFIG_NO_FILE_IO
TESTCASE_AUTO(TestBug4153072);
#endif
#if !UCONFIG_NO_FILE_IO
TESTCASE_AUTO(TestUnicodeFiles);
#endif
TESTCASE_AUTO(TestGetAvailableLocales);
TESTCASE_AUTO(TestGetDisplayName);
#if !UCONFIG_NO_FILE_IO
TESTCASE_AUTO(TestEndBehaviour);
TESTCASE_AUTO(TestWordBreaks);
TESTCASE_AUTO(TestWordBoundary);
TESTCASE_AUTO(TestLineBreaks);
TESTCASE_AUTO(TestSentBreaks);
TESTCASE_AUTO(TestExtended);
#endif
#if !UCONFIG_NO_REGULAR_EXPRESSIONS && !UCONFIG_NO_FILE_IO
TESTCASE_AUTO(TestMonkey);
#endif
#if !UCONFIG_NO_FILE_IO
TESTCASE_AUTO(TestBug3818);
#endif
TESTCASE_AUTO(TestDebug);
#if !UCONFIG_NO_FILE_IO
TESTCASE_AUTO(TestBug5775);
#endif
TESTCASE_AUTO(TestBug9983);
TESTCASE_AUTO(TestDictRules);
TESTCASE_AUTO(TestBug5532);
TESTCASE_AUTO(TestBug7547);
TESTCASE_AUTO(TestBug12797);
TESTCASE_AUTO(TestBug12918);
TESTCASE_AUTO(TestBug12932);
TESTCASE_AUTO(TestEmoji);
TESTCASE_AUTO(TestBug12519);
TESTCASE_AUTO(TestBug12677);
TESTCASE_AUTO(TestTableRedundancies);
TESTCASE_AUTO(TestBug13447);
TESTCASE_AUTO(TestReverse);
TESTCASE_AUTO(TestBug13692);
TESTCASE_AUTO_END;
}
//--------------------------------------------------------------------------------------
//
// RBBITest constructor and destructor
//
//--------------------------------------------------------------------------------------
RBBITest::RBBITest() {
fTestParams = NULL;
}
RBBITest::~RBBITest() {
}
static void printStringBreaks(UText *tstr, int expected[], int expectedCount) {
UErrorCode status = U_ZERO_ERROR;
char name[100];
printf("code alpha extend alphanum type word sent line name\n");
int nextExpectedIndex = 0;
utext_setNativeIndex(tstr, 0);
for (int j = 0; j < utext_nativeLength(tstr); j=utext_getNativeIndex(tstr)) {
if (nextExpectedIndex < expectedCount && j >= expected[nextExpectedIndex] ) {
printf("------------------------------------------------ %d\n", j);
++nextExpectedIndex;
}
UChar32 c = utext_next32(tstr);
u_charName(c, U_UNICODE_CHAR_NAME, name, 100, &status);
printf("%7x %5d %6d %8d %4s %4s %4s %4s %s\n", (int)c,
u_isUAlphabetic(c),
u_hasBinaryProperty(c, UCHAR_GRAPHEME_EXTEND),
u_isalnum(c),
u_getPropertyValueName(UCHAR_GENERAL_CATEGORY,
u_charType(c),
U_SHORT_PROPERTY_NAME),
u_getPropertyValueName(UCHAR_WORD_BREAK,
u_getIntPropertyValue(c,
UCHAR_WORD_BREAK),
U_SHORT_PROPERTY_NAME),
u_getPropertyValueName(UCHAR_SENTENCE_BREAK,
u_getIntPropertyValue(c,
UCHAR_SENTENCE_BREAK),
U_SHORT_PROPERTY_NAME),
u_getPropertyValueName(UCHAR_LINE_BREAK,
u_getIntPropertyValue(c,
UCHAR_LINE_BREAK),
U_SHORT_PROPERTY_NAME),
name);
}
}
static void printStringBreaks(const UnicodeString &ustr, int expected[], int expectedCount) {
UErrorCode status = U_ZERO_ERROR;
UText *tstr = NULL;
tstr = utext_openConstUnicodeString(NULL, &ustr, &status);
if (U_FAILURE(status)) {
printf("printStringBreaks, utext_openConstUnicodeString() returns %s\n", u_errorName(status));
return;
}
printStringBreaks(tstr, expected, expectedCount);
utext_close(tstr);
}
void RBBITest::TestBug3818() {
UErrorCode status = U_ZERO_ERROR;
// Four Thai words...
static const UChar thaiWordData[] = { 0x0E43,0x0E2B,0x0E0D,0x0E48, 0x0E43,0x0E2B,0x0E0D,0x0E48,
0x0E43,0x0E2B,0x0E0D,0x0E48, 0x0E43,0x0E2B,0x0E0D,0x0E48, 0 };
UnicodeString thaiStr(thaiWordData);
BreakIterator* bi = BreakIterator::createWordInstance(Locale("th"), status);
if (U_FAILURE(status) || bi == NULL) {
errcheckln(status, "Fail at file %s, line %d, status = %s", __FILE__, __LINE__, u_errorName(status));
return;
}
bi->setText(thaiStr);
int32_t startOfSecondWord = bi->following(1);
if (startOfSecondWord != 4) {
errln("Fail at file %s, line %d expected start of word at 4, got %d",
__FILE__, __LINE__, startOfSecondWord);
}
startOfSecondWord = bi->following(0);
if (startOfSecondWord != 4) {
errln("Fail at file %s, line %d expected start of word at 4, got %d",
__FILE__, __LINE__, startOfSecondWord);
}
delete bi;
}
//---------------------------------------------
//
// other tests
//
//---------------------------------------------
void RBBITest::TestGetAvailableLocales()
{
int32_t locCount = 0;
const Locale* locList = BreakIterator::getAvailableLocales(locCount);
if (locCount == 0)
dataerrln("getAvailableLocales() returned an empty list!");
// Just make sure that it's returning good memory.
int32_t i;
for (i = 0; i < locCount; ++i) {
logln(locList[i].getName());
}
}
//Testing the BreakIterator::getDisplayName() function
void RBBITest::TestGetDisplayName()
{
UnicodeString result;
BreakIterator::getDisplayName(Locale::getUS(), result);
if (Locale::getDefault() == Locale::getUS() && result != "English (United States)")
dataerrln("BreakIterator::getDisplayName() failed: expected \"English (United States)\", got \""
+ result);
BreakIterator::getDisplayName(Locale::getFrance(), Locale::getUS(), result);
if (result != "French (France)")
dataerrln("BreakIterator::getDisplayName() failed: expected \"French (France)\", got \""
+ result);
}
/**
* Test End Behaviour
* @bug 4068137
*/
void RBBITest::TestEndBehaviour()
{
UErrorCode status = U_ZERO_ERROR;
UnicodeString testString("boo.");
BreakIterator *wb = BreakIterator::createWordInstance(Locale::getDefault(), status);
if (U_FAILURE(status))
{
errcheckln(status, "Failed to create the BreakIterator for default locale in TestEndBehaviour. - %s", u_errorName(status));
return;
}
wb->setText(testString);
if (wb->first() != 0)
errln("Didn't get break at beginning of string.");
if (wb->next() != 3)
errln("Didn't get break before period in \"boo.\"");
if (wb->current() != 4 && wb->next() != 4)
errln("Didn't get break at end of string.");
delete wb;
}
/*
* @bug 4153072
*/
void RBBITest::TestBug4153072() {
UErrorCode status = U_ZERO_ERROR;
BreakIterator *iter = BreakIterator::createWordInstance(Locale::getDefault(), status);
if (U_FAILURE(status))
{
errcheckln(status, "Failed to create the BreakIterator for default locale in TestBug4153072 - %s", u_errorName(status));
return;
}
UnicodeString str("...Hello, World!...");
int32_t begin = 3;
int32_t end = str.length() - 3;
UBool onBoundary;
StringCharacterIterator* textIterator = new StringCharacterIterator(str, begin, end, begin);
iter->adoptText(textIterator);
int index;
// Note: with the switch to UText, there is no way to restrict the
// iteration range to begin at an index other than zero.
// String character iterators created with a non-zero bound are
// treated by RBBI as being empty.
for (index = -1; index < begin + 1; ++index) {
onBoundary = iter->isBoundary(index);
if (index == 0? !onBoundary : onBoundary) {
errln((UnicodeString)"Didn't handle isBoundary correctly with offset = " + index +
" and begin index = " + begin);
}
}
delete iter;
}
//
// Test for problem reported by Ashok Matoria on 9 July 2007
// One.<kSoftHyphen><kSpace>Two.
//
// Sentence break at start (0) and then on calling next() it breaks at
// 'T' of "Two". Now, at this point if I do next() and
// then previous(), it breaks at <kSOftHyphen> instead of 'T' of "Two".
//
void RBBITest::TestBug5775() {
UErrorCode status = U_ZERO_ERROR;
BreakIterator *bi = BreakIterator::createSentenceInstance(Locale::getEnglish(), status);
TEST_ASSERT_SUCCESS(status);
if (U_FAILURE(status)) {
return;
}
// Check for status first for better handling of no data errors.
TEST_ASSERT(bi != NULL);
if (bi == NULL) {
return;
}
UnicodeString s("One.\\u00ad Two.", -1, US_INV);
// 01234 56789
s = s.unescape();
bi->setText(s);
int pos = bi->next();
TEST_ASSERT(pos == 6);
pos = bi->next();
TEST_ASSERT(pos == 10);
pos = bi->previous();
TEST_ASSERT(pos == 6);
delete bi;
}
//------------------------------------------------------------------------------
//
// RBBITest::Extended Run RBBI Tests from an external test data file
//
//------------------------------------------------------------------------------
struct TestParams {
BreakIterator *bi; // Break iterator is set while parsing test source.
// Changed out whenever test data changes break type.
UnicodeString dataToBreak; // Data that is built up while parsing the test.
UVector32 *expectedBreaks; // Expected break positions, matches dataToBreak UnicodeString.
UVector32 *srcLine; // Positions in source file, indexed same as dataToBreak.
UVector32 *srcCol;
UText *textToBreak; // UText, could be UTF8 or UTF16.
UVector32 *textMap; // Map from UTF-16 dataToBreak offsets to UText offsets.
CharString utf8String; // UTF-8 form of text to break.
TestParams(UErrorCode &status) : dataToBreak() {
bi = NULL;
expectedBreaks = new UVector32(status);
srcLine = new UVector32(status);
srcCol = new UVector32(status);
textToBreak = NULL;
textMap = new UVector32(status);
}
~TestParams() {
delete bi;
delete expectedBreaks;
delete srcLine;
delete srcCol;
utext_close(textToBreak);
delete textMap;
}
int32_t getSrcLine(int32_t bp);
int32_t getExpectedBreak(int32_t bp);
int32_t getSrcCol(int32_t bp);
void setUTF16(UErrorCode &status);
void setUTF8(UErrorCode &status);
};
// Append a UnicodeString to a CharString with UTF-8 encoding.
// Substitute any invalid chars.
// Note: this is used with test data that includes a few unpaired surrogates in the UTF-16 that will be substituted.
static void CharStringAppend(CharString &dest, const UnicodeString &src, UErrorCode &status) {
if (U_FAILURE(status)) {
return;
}
int32_t utf8Length;
u_strToUTF8WithSub(NULL, 0, &utf8Length, // Output Buffer, NULL for preflight.
src.getBuffer(), src.length(), // UTF-16 data
0xfffd, NULL, // Substitution char, number of subs.
&status);
if (U_FAILURE(status) && status != U_BUFFER_OVERFLOW_ERROR) {
return;
}
status = U_ZERO_ERROR;
int32_t capacity;
char *buffer = dest.getAppendBuffer(utf8Length, utf8Length, capacity, status);
u_strToUTF8WithSub(buffer, utf8Length, NULL,
src.getBuffer(), src.length(),
0xfffd, NULL, &status);
dest.append(buffer, utf8Length, status);
}
void TestParams::setUTF16(UErrorCode &status) {
textToBreak = utext_openUnicodeString(textToBreak, &dataToBreak, &status);
textMap->removeAllElements();
for (int32_t i=0; i<dataToBreak.length(); i++) {
if (i == dataToBreak.getChar32Start(i)) {
textMap->addElement(i, status);
} else {
textMap->addElement(-1, status);
}
}
textMap->addElement(dataToBreak.length(), status);
U_ASSERT(dataToBreak.length() + 1 == textMap->size());
}
void TestParams::setUTF8(UErrorCode &status) {
if (U_FAILURE(status)) {
return;
}
utf8String.clear();
CharStringAppend(utf8String, dataToBreak, status);
textToBreak = utext_openUTF8(textToBreak, utf8String.data(), utf8String.length(), &status);
if (U_FAILURE(status)) {
return;
}
textMap->removeAllElements();
int32_t utf16Index = 0;
for (;;) {
textMap->addElement(utf16Index, status);
UChar32 c32 = utext_current32(textToBreak);
if (c32 < 0) {
break;
}
utf16Index += U16_LENGTH(c32);
utext_next32(textToBreak);
while (textMap->size() < utext_getNativeIndex(textToBreak)) {
textMap->addElement(-1, status);
}
}
U_ASSERT(utext_nativeLength(textToBreak) + 1 == textMap->size());
}
int32_t TestParams::getSrcLine(int32_t bp) {
if (bp >= textMap->size()) {
bp = textMap->size() - 1;
}
int32_t i = 0;
for(; bp >= 0 ; --bp) {
// Move to a character boundary if we are not on one already.
i = textMap->elementAti(bp);
if (i >= 0) {
break;
}
}
return srcLine->elementAti(i);
}
int32_t TestParams::getExpectedBreak(int32_t bp) {
if (bp >= textMap->size()) {
return 0;
}
int32_t i = textMap->elementAti(bp);
int32_t retVal = 0;
if (i >= 0) {
retVal = expectedBreaks->elementAti(i);
}
return retVal;
}
int32_t TestParams::getSrcCol(int32_t bp) {
if (bp >= textMap->size()) {
bp = textMap->size() - 1;
}
int32_t i = 0;
for(; bp >= 0; --bp) {
// Move bp to a character boundary if we are not on one already.
i = textMap->elementAti(bp);
if (i >= 0) {
break;
}
}
return srcCol->elementAti(i);
}
void RBBITest::executeTest(TestParams *t, UErrorCode &status) {
int32_t bp;
int32_t prevBP;
int32_t i;
TEST_ASSERT_SUCCESS(status);
if (U_FAILURE(status)) {
return;
}
if (t->bi == NULL) {
return;
}
t->bi->setText(t->textToBreak, status);
//
// Run the iterator forward
//
prevBP = -1;
for (bp = t->bi->first(); bp != BreakIterator::DONE; bp = t->bi->next()) {
if (prevBP == bp) {
// Fail for lack of forward progress.
errln("Forward Iteration, no forward progress. Break Pos=%4d File line,col=%4d,%4d",
bp, t->getSrcLine(bp), t->getSrcCol(bp));
break;
}
// Check that there we didn't miss an expected break between the last one
// and this one.
for (i=prevBP+1; i<bp; i++) {
if (t->getExpectedBreak(i) != 0) {
int expected[] = {0, i};
printStringBreaks(t->dataToBreak, expected, 2);
errln("Forward Iteration, break expected, but not found. Pos=%4d File line,col= %4d,%4d",
i, t->getSrcLine(i), t->getSrcCol(i));
}
}
// Check that the break we did find was expected
if (t->getExpectedBreak(bp) == 0) {
int expected[] = {0, bp};
printStringBreaks(t->textToBreak, expected, 2);
errln("Forward Iteration, break found, but not expected. Pos=%4d File line,col= %4d,%4d",
bp, t->getSrcLine(bp), t->getSrcCol(bp));
} else {
// The break was expected.
// Check that the {nnn} tag value is correct.
int32_t expectedTagVal = t->getExpectedBreak(bp);
if (expectedTagVal == -1) {
expectedTagVal = 0;
}
int32_t line = t->getSrcLine(bp);
int32_t rs = t->bi->getRuleStatus();
if (rs != expectedTagVal) {
errln("Incorrect status for forward break. Pos=%4d File line,col= %4d,%4d.\n"
" Actual, Expected status = %4d, %4d",
bp, line, t->getSrcCol(bp), rs, expectedTagVal);
}
}
prevBP = bp;
}
// Verify that there were no missed expected breaks after the last one found
for (i=prevBP+1; i<utext_nativeLength(t->textToBreak); i++) {
if (t->getExpectedBreak(i) != 0) {
errln("Forward Iteration, break expected, but not found. Pos=%4d File line,col= %4d,%4d",
i, t->getSrcLine(i), t->getSrcCol(i));
}
}
//
// Run the iterator backwards, verify that the same breaks are found.
//
prevBP = utext_nativeLength(t->textToBreak)+2; // start with a phony value for the last break pos seen.
bp = t->bi->last();
while (bp != BreakIterator::DONE) {
if (prevBP == bp) {
// Fail for lack of progress.
errln("Reverse Iteration, no progress. Break Pos=%4d File line,col=%4d,%4d",
bp, t->getSrcLine(bp), t->getSrcCol(bp));
break;
}
// Check that we didn't miss an expected break between the last one
// and this one. (UVector returns zeros for index out of bounds.)
for (i=prevBP-1; i>bp; i--) {
if (t->getExpectedBreak(i) != 0) {
errln("Reverse Iteration, break expected, but not found. Pos=%4d File line,col= %4d,%4d",
i, t->getSrcLine(i), t->getSrcCol(i));
}
}
// Check that the break we did find was expected
if (t->getExpectedBreak(bp) == 0) {
errln("Reverse Itertion, break found, but not expected. Pos=%4d File line,col= %4d,%4d",
bp, t->getSrcLine(bp), t->getSrcCol(bp));
} else {
// The break was expected.
// Check that the {nnn} tag value is correct.
int32_t expectedTagVal = t->getExpectedBreak(bp);
if (expectedTagVal == -1) {
expectedTagVal = 0;
}
int line = t->getSrcLine(bp);
int32_t rs = t->bi->getRuleStatus();
if (rs != expectedTagVal) {
errln("Incorrect status for reverse break. Pos=%4d File line,col= %4d,%4d.\n"
" Actual, Expected status = %4d, %4d",
bp, line, t->getSrcCol(bp), rs, expectedTagVal);
}
}
prevBP = bp;
bp = t->bi->previous();
}
// Verify that there were no missed breaks prior to the last one found
for (i=prevBP-1; i>=0; i--) {
if (t->getExpectedBreak(i) != 0) {
errln("Forward Itertion, break expected, but not found. Pos=%4d File line,col= %4d,%4d",
i, t->getSrcLine(i), t->getSrcCol(i));
}
}
// Check isBoundary()
for (i=0; i < utext_nativeLength(t->textToBreak); i++) {
UBool boundaryExpected = (t->getExpectedBreak(i) != 0);
UBool boundaryFound = t->bi->isBoundary(i);
if (boundaryExpected != boundaryFound) {
errln("isBoundary(%d) incorrect. File line,col= %4d,%4d\n"
" Expected, Actual= %s, %s",
i, t->getSrcLine(i), t->getSrcCol(i),
boundaryExpected ? "true":"false", boundaryFound? "true" : "false");
}
}
// Check following()
for (i=0; i < utext_nativeLength(t->textToBreak); i++) {
int32_t actualBreak = t->bi->following(i);
int32_t expectedBreak = BreakIterator::DONE;
for (int32_t j=i+1; j <= utext_nativeLength(t->textToBreak); j++) {
if (t->getExpectedBreak(j) != 0) {
expectedBreak = j;
break;
}
}
if (expectedBreak != actualBreak) {
errln("following(%d) incorrect. File line,col= %4d,%4d\n"
" Expected, Actual= %d, %d",
i, t->getSrcLine(i), t->getSrcCol(i), expectedBreak, actualBreak);
}
}
// Check preceding()
for (i=utext_nativeLength(t->textToBreak); i>=0; i--) {
int32_t actualBreak = t->bi->preceding(i);
int32_t expectedBreak = BreakIterator::DONE;
// For UTF-8 & UTF-16 supplementals, all code units of a character are equivalent.
// preceding(trailing byte) will return the index of some preceding code point,
// not the lead byte of the current code point, even though that has a smaller index.
// Therefore, start looking at the expected break data not at i-1, but at
// the start of code point index - 1.
utext_setNativeIndex(t->textToBreak, i);
int32_t j = utext_getNativeIndex(t->textToBreak) - 1;
for (; j >= 0; j--) {
if (t->getExpectedBreak(j) != 0) {
expectedBreak = j;
break;
}
}
if (expectedBreak != actualBreak) {
errln("preceding(%d) incorrect. File line,col= %4d,%4d\n"
" Expected, Actual= %d, %d",
i, t->getSrcLine(i), t->getSrcCol(i), expectedBreak, actualBreak);
}
}
}
void RBBITest::TestExtended() {
// Skip test for now when UCONFIG_NO_FILTERED_BREAK_ITERATION is set. This
// data driven test closely entangles filtered and regular data.
#if !UCONFIG_NO_REGULAR_EXPRESSIONS && !UCONFIG_NO_FILTERED_BREAK_ITERATION
UErrorCode status = U_ZERO_ERROR;
Locale locale("");
TestParams tp(status);
RegexMatcher localeMatcher(UnicodeString(u"<locale *([\\p{L}\\p{Nd}_@&=-]*) *>"), 0, status);
if (U_FAILURE(status)) {
dataerrln("Failure in file %s, line %d, status = \"%s\"", __FILE__, __LINE__, u_errorName(status));
}
//
// Open and read the test data file.
//
const char *testDataDirectory = IntlTest::getSourceTestData(status);
CharString testFileName(testDataDirectory, -1, status);
testFileName.append("rbbitst.txt", -1, status);
int len;
UChar *testFile = ReadAndConvertFile(testFileName.data(), len, "UTF-8", status);
if (U_FAILURE(status)) {
errln("%s:%d Error %s opening file rbbitst.txt", __FILE__, __LINE__, u_errorName(status));
return;
}
bool skipTest = false; // Skip this test?
//
// Put the test data into a UnicodeString
//
UnicodeString testString(FALSE, testFile, len);
enum EParseState{
PARSE_COMMENT,
PARSE_TAG,
PARSE_DATA,
PARSE_NUM,
PARSE_RULES
}
parseState = PARSE_TAG;
EParseState savedState = PARSE_TAG;
int32_t lineNum = 1;
int32_t colStart = 0;
int32_t column = 0;
int32_t charIdx = 0;
int32_t tagValue = 0; // The numeric value of a <nnn> tag.
UnicodeString rules; // Holds rules from a <rules> ... </rules> block
int32_t rulesFirstLine; // Line number of the start of current <rules> block
for (charIdx = 0; charIdx < len; ) {
status = U_ZERO_ERROR;
UChar c = testString.charAt(charIdx);
charIdx++;
if (c == u'\r' && charIdx<len && testString.charAt(charIdx) == u'\n') {
// treat CRLF as a unit
c = u'\n';
charIdx++;
}
if (c == u'\n' || c == u'\r') {
lineNum++;
colStart = charIdx;
}
column = charIdx - colStart + 1;
switch (parseState) {
case PARSE_COMMENT:
if (c == u'\n' || c == u'\r') {
parseState = savedState;
}
break;
case PARSE_TAG:
{
if (c == u'#') {
parseState = PARSE_COMMENT;
savedState = PARSE_TAG;
break;
}
if (u_isUWhiteSpace(c)) {
break;
}
if (testString.compare(charIdx-1, 6, u"<word>") == 0) {
delete tp.bi;
tp.bi = BreakIterator::createWordInstance(locale, status);
skipTest = false;
charIdx += 5;
break;
}
if (testString.compare(charIdx-1, 6, u"<char>") == 0) {
delete tp.bi;
tp.bi = BreakIterator::createCharacterInstance(locale, status);
skipTest = false;
charIdx += 5;
break;
}
if (testString.compare(charIdx-1, 6, u"<line>") == 0) {
delete tp.bi;
tp.bi = BreakIterator::createLineInstance(locale, status);
skipTest = false;
charIdx += 5;
break;
}
if (testString.compare(charIdx-1, 6, u"<sent>") == 0) {
delete tp.bi;
tp.bi = BreakIterator::createSentenceInstance(locale, status);
skipTest = false;
charIdx += 5;
break;
}
if (testString.compare(charIdx-1, 7, u"<title>") == 0) {
delete tp.bi;
tp.bi = BreakIterator::createTitleInstance(locale, status);
charIdx += 6;
break;
}
if (testString.compare(charIdx-1, 7, u"<rules>") == 0 ||
testString.compare(charIdx-1, 10, u"<badrules>") == 0) {
charIdx = testString.indexOf(u'>', charIdx) + 1;
parseState = PARSE_RULES;
rules.remove();
rulesFirstLine = lineNum;
break;
}
// <locale loc_name>
localeMatcher.reset(testString);
if (localeMatcher.lookingAt(charIdx-1, status)) {
UnicodeString localeName = localeMatcher.group(1, status);
char localeName8[100];
localeName.extract(0, localeName.length(), localeName8, sizeof(localeName8), 0);
locale = Locale::createFromName(localeName8);
charIdx += localeMatcher.group(0, status).length() - 1;
TEST_ASSERT_SUCCESS(status);
break;
}
if (testString.compare(charIdx-1, 6, u"<data>") == 0) {
parseState = PARSE_DATA;
charIdx += 5;
tp.dataToBreak = "";
tp.expectedBreaks->removeAllElements();
tp.srcCol ->removeAllElements();
tp.srcLine->removeAllElements();
break;
}
errln("line %d: Tag expected in test file.", lineNum);
parseState = PARSE_COMMENT;
savedState = PARSE_DATA;
goto end_test; // Stop the test.
}
break;
case PARSE_RULES:
if (testString.compare(charIdx-1, 8, u"</rules>") == 0) {
charIdx += 7;
parseState = PARSE_TAG;
delete tp.bi;
UParseError pe;
tp.bi = new RuleBasedBreakIterator(rules, pe, status);
skipTest = U_FAILURE(status);
if (U_FAILURE(status)) {
errln("file rbbitst.txt: %d - Error %s creating break iterator from rules.",
rulesFirstLine + pe.line - 1, u_errorName(status));
}
} else if (testString.compare(charIdx-1, 11, u"</badrules>") == 0) {
charIdx += 10;
parseState = PARSE_TAG;
UErrorCode ec = U_ZERO_ERROR;
UParseError pe;
RuleBasedBreakIterator bi(rules, pe, ec);
if (U_SUCCESS(ec)) {
errln("file rbbitst.txt: %d - Expected, but did not get, a failure creating break iterator from rules.",
rulesFirstLine + pe.line - 1);
}
} else {
rules.append(c);
}
break;
case PARSE_DATA:
if (c == u'•') {
int32_t breakIdx = tp.dataToBreak.length();
tp.expectedBreaks->setSize(breakIdx+1);
tp.expectedBreaks->setElementAt(-1, breakIdx);
tp.srcLine->setSize(breakIdx+1);
tp.srcLine->setElementAt(lineNum, breakIdx);
tp.srcCol ->setSize(breakIdx+1);
tp.srcCol ->setElementAt(column, breakIdx);
break;
}
if (testString.compare(charIdx-1, 7, u"</data>") == 0) {
// Add final entry to mappings from break location to source file position.
// Need one extra because last break position returned is after the
// last char in the data, not at the last char.
tp.srcLine->addElement(lineNum, status);
tp.srcCol ->addElement(column, status);
parseState = PARSE_TAG;
charIdx += 6;
if (!skipTest) {
// RUN THE TEST!
status = U_ZERO_ERROR;
tp.setUTF16(status);
executeTest(&tp, status);
TEST_ASSERT_SUCCESS(status);
// Run again, this time with UTF-8 text wrapped in a UText.
status = U_ZERO_ERROR;
tp.setUTF8(status);
TEST_ASSERT_SUCCESS(status);
executeTest(&tp, status);
}
break;
}
if (testString.compare(charIdx-1, 3, u"\\N{") == 0) {
// Named character, e.g. \N{COMBINING GRAVE ACCENT}
// Get the code point from the name and insert it into the test data.
// (Damn, no API takes names in Unicode !!!
// we've got to take it back to char *)
int32_t nameEndIdx = testString.indexOf(u'}', charIdx);
int32_t nameLength = nameEndIdx - (charIdx+2);
char charNameBuf[200];
UChar32 theChar = -1;
if (nameEndIdx != -1) {
UErrorCode status = U_ZERO_ERROR;
testString.extract(charIdx+2, nameLength, charNameBuf, sizeof(charNameBuf));
charNameBuf[sizeof(charNameBuf)-1] = 0;
theChar = u_charFromName(U_UNICODE_CHAR_NAME, charNameBuf, &status);
if (U_FAILURE(status)) {
theChar = -1;
}
}
if (theChar == -1) {
errln("Error in named character in test file at line %d, col %d",
lineNum, column);
} else {
// Named code point was recognized. Insert it
// into the test data.
tp.dataToBreak.append(theChar);
while (tp.dataToBreak.length() > tp.srcLine->size()) {
tp.srcLine->addElement(lineNum, status);
tp.srcCol ->addElement(column, status);
}
}
if (nameEndIdx > charIdx) {
charIdx = nameEndIdx+1;
}
break;
}
if (testString.compare(charIdx-1, 2, u"<>") == 0) {
charIdx++;
int32_t breakIdx = tp.dataToBreak.length();
tp.expectedBreaks->setSize(breakIdx+1);
tp.expectedBreaks->setElementAt(-1, breakIdx);
tp.srcLine->setSize(breakIdx+1);
tp.srcLine->setElementAt(lineNum, breakIdx);
tp.srcCol ->setSize(breakIdx+1);
tp.srcCol ->setElementAt(column, breakIdx);
break;
}
if (c == u'<') {
tagValue = 0;
parseState = PARSE_NUM;
break;
}
if (c == u'#' && column==3) { // TODO: why is column off so far?
parseState = PARSE_COMMENT;
savedState = PARSE_DATA;
break;
}
if (c == u'\\') {
// Check for \ at end of line, a line continuation.
// Advance over (discard) the newline
UChar32 cp = testString.char32At(charIdx);
if (cp == u'\r' && charIdx<len && testString.charAt(charIdx+1) == u'\n') {
// We have a CR LF
// Need an extra increment of the input ptr to move over both of them
charIdx++;
}
if (cp == u'\n' || cp == u'\r') {
lineNum++;
colStart = charIdx;
charIdx++;
break;
}
// Let unescape handle the back slash.
cp = testString.unescapeAt(charIdx);
if (cp != -1) {
// Escape sequence was recognized. Insert the char
// into the test data.
tp.dataToBreak.append(cp);
while (tp.dataToBreak.length() > tp.srcLine->size()) {
tp.srcLine->addElement(lineNum, status);
tp.srcCol ->addElement(column, status);
}
break;
}
// Not a recognized backslash escape sequence.
// Take the next char as a literal.
// TODO: Should this be an error?
c = testString.charAt(charIdx);
charIdx = testString.moveIndex32(charIdx, 1);
}
// Normal, non-escaped data char.
tp.dataToBreak.append(c);
// Save the mapping from offset in the data to line/column numbers in
// the original input file. Will be used for better error messages only.
// If there's an expected break before this char, the slot in the mapping
// vector will already be set for this char; don't overwrite it.
if (tp.dataToBreak.length() > tp.srcLine->size()) {
tp.srcLine->addElement(lineNum, status);
tp.srcCol ->addElement(column, status);
}
break;
case PARSE_NUM:
// We are parsing an expected numeric tag value, like <1234>,
// within a chunk of data.
if (u_isUWhiteSpace(c)) {
break;
}
if (c == u'>') {
// Finished the number. Add the info to the expected break data,
// and switch parse state back to doing plain data.
parseState = PARSE_DATA;
if (tagValue == 0) {
tagValue = -1;
}
int32_t breakIdx = tp.dataToBreak.length();
tp.expectedBreaks->setSize(breakIdx+1);
tp.expectedBreaks->setElementAt(tagValue, breakIdx);
tp.srcLine->setSize(breakIdx+1);
tp.srcLine->setElementAt(lineNum, breakIdx);
tp.srcCol ->setSize(breakIdx+1);
tp.srcCol ->setElementAt(column, breakIdx);
break;
}
if (u_isdigit(c)) {
tagValue = tagValue*10 + u_charDigitValue(c);
break;
}
errln("Syntax Error in test file at line %d, col %d",
lineNum, column);
parseState = PARSE_COMMENT;
goto end_test; // Stop the test
break;
}
if (U_FAILURE(status)) {
dataerrln("ICU Error %s while parsing test file at line %d.",
u_errorName(status), lineNum);
status = U_ZERO_ERROR;
goto end_test; // Stop the test
}
}
// Reached end of test file. Raise an error if parseState indicates that we are
// within a block that should have been terminated.
if (parseState == PARSE_RULES) {
errln("rbbitst.txt:%d <rules> block beginning at line %d is not closed.",
lineNum, rulesFirstLine);
}
if (parseState == PARSE_DATA) {
errln("rbbitst.txt:%d <data> block not closed.", lineNum);
}
end_test:
delete [] testFile;
#endif
}
//-------------------------------------------------------------------------------
//
// TestDictRules create a break iterator from source rules that includes a
// dictionary range. Regression for bug #7130. Source rules
// do not declare a break iterator type (word, line, sentence, etc.
// but the dictionary code, without a type, would loop.
//
//-------------------------------------------------------------------------------
void RBBITest::TestDictRules() {
const char *rules = "$dictionary = [a-z]; \n"
"!!forward; \n"
"$dictionary $dictionary; \n"
"!!reverse; \n"
"$dictionary $dictionary; \n";
const char *text = "aa";
UErrorCode status = U_ZERO_ERROR;
UParseError parseError;
RuleBasedBreakIterator bi(rules, parseError, status);
if (U_SUCCESS(status)) {
UnicodeString utext = text;
bi.setText(utext);
int32_t position;
int32_t loops;
for (loops = 0; loops<10; loops++) {
position = bi.next();
if (position == RuleBasedBreakIterator::DONE) {
break;
}
}
TEST_ASSERT(loops == 1);
} else {
dataerrln("Error creating RuleBasedBreakIterator: %s", u_errorName(status));
}
}
//-------------------------------------------------------------------------------
//
// ReadAndConvertFile Read a text data file, convert it to UChars, and
// return the data in one big UChar * buffer, which the caller must delete.
//
// parameters:
// fileName: the name of the file, with no directory part. The test data directory
// is assumed.
// ulen an out parameter, receives the actual length (in UChars) of the file data.
// encoding The file encoding. If the file contains a BOM, that will override the encoding
// specified here. The BOM, if it exists, will be stripped from the returned data.
// Pass NULL for the system default encoding.
// status
// returns:
// The file data, converted to UChar.
// The caller must delete this when done with
// delete [] theBuffer;
//
// TODO: This is a clone of RegexTest::ReadAndConvertFile.
// Move this function to some common place.
//
//--------------------------------------------------------------------------------
UChar *RBBITest::ReadAndConvertFile(const char *fileName, int &ulen, const char *encoding, UErrorCode &status) {
UChar *retPtr = NULL;
char *fileBuf = NULL;
UConverter* conv = NULL;
FILE *f = NULL;
ulen = 0;
if (U_FAILURE(status)) {
return retPtr;
}
//
// Open the file.
//
f = fopen(fileName, "rb");
if (f == 0) {
dataerrln("Error opening test data file %s\n", fileName);
status = U_FILE_ACCESS_ERROR;
return NULL;
}
//
// Read it in
//
int fileSize;
int amt_read;
fseek( f, 0, SEEK_END);
fileSize = ftell(f);
fileBuf = new char[fileSize];
fseek(f, 0, SEEK_SET);
amt_read = fread(fileBuf, 1, fileSize, f);
if (amt_read != fileSize || fileSize <= 0) {
errln("Error reading test data file.");
goto cleanUpAndReturn;
}
//
// Look for a Unicode Signature (BOM) on the data just read
//
int32_t signatureLength;
const char * fileBufC;
const char* bomEncoding;
fileBufC = fileBuf;
bomEncoding = ucnv_detectUnicodeSignature(
fileBuf, fileSize, &signatureLength, &status);
if(bomEncoding!=NULL ){
fileBufC += signatureLength;
fileSize -= signatureLength;
encoding = bomEncoding;
}
//
// Open a converter to take the rule file to UTF-16
//
conv = ucnv_open(encoding, &status);
if (U_FAILURE(status)) {
goto cleanUpAndReturn;
}
//
// Convert the rules to UChar.
// Preflight first to determine required buffer size.
//
ulen = ucnv_toUChars(conv,
NULL, // dest,
0, // destCapacity,
fileBufC,
fileSize,
&status);
if (status == U_BUFFER_OVERFLOW_ERROR) {
// Buffer Overflow is expected from the preflight operation.
status = U_ZERO_ERROR;
retPtr = new UChar[ulen+1];
ucnv_toUChars(conv,
retPtr, // dest,
ulen+1,
fileBufC,
fileSize,
&status);
}
cleanUpAndReturn:
fclose(f);
delete []fileBuf;
ucnv_close(conv);
if (U_FAILURE(status)) {
errln("ucnv_toUChars: ICU Error \"%s\"\n", u_errorName(status));
delete []retPtr;
retPtr = 0;
ulen = 0;
};
return retPtr;
}
//--------------------------------------------------------------------------------------------
//
// Run tests from each of the boundary test data files distributed by the Unicode Consortium
//
//-------------------------------------------------------------------------------------------
void RBBITest::TestUnicodeFiles() {
RuleBasedBreakIterator *bi;
UErrorCode status = U_ZERO_ERROR;
bi = (RuleBasedBreakIterator *)BreakIterator::createCharacterInstance(Locale::getEnglish(), status);
TEST_ASSERT_SUCCESS(status);
if (U_SUCCESS(status)) {
runUnicodeTestData("GraphemeBreakTest.txt", bi);
}
delete bi;
bi = (RuleBasedBreakIterator *)BreakIterator::createWordInstance(Locale::getEnglish(), status);
TEST_ASSERT_SUCCESS(status);
if (U_SUCCESS(status)) {
runUnicodeTestData("WordBreakTest.txt", bi);
}
delete bi;
bi = (RuleBasedBreakIterator *)BreakIterator::createSentenceInstance(Locale::getEnglish(), status);
TEST_ASSERT_SUCCESS(status);
if (U_SUCCESS(status)) {
runUnicodeTestData("SentenceBreakTest.txt", bi);
}
delete bi;
bi = (RuleBasedBreakIterator *)BreakIterator::createLineInstance(Locale::getEnglish(), status);
TEST_ASSERT_SUCCESS(status);
if (U_SUCCESS(status)) {
runUnicodeTestData("LineBreakTest.txt", bi);
}
delete bi;
}
// Check for test cases from the Unicode test data files that are known to fail
// and should be skipped because ICU is not yet able to fully implement the spec.
// See ticket #7270.
UBool RBBITest::testCaseIsKnownIssue(const UnicodeString &testCase, const char *fileName) {
static struct TestCase {
const char *fFileName;
const UChar *fString;
} badTestCases[] = { // Line Numbers from Unicode 7.0.0 file.
{"LineBreakTest.txt", u"\u200B\u0020}"}, // Line 5198
{"LineBreakTest.txt", u"\u200B\u0020)"}, // Line 5202
{"LineBreakTest.txt", u"\u200B\u0020!"}, // Line 5214
{"LineBreakTest.txt", u"\u200B\u0020,"}, // Line 5246
{"LineBreakTest.txt", u"\u200B\u0020/"}, // Line 5298
{"LineBreakTest.txt", u"\u200B\u0020\u2060"}, // Line 5302
// Line Numbers from pre-release verion of GraphemeBreakTest-10.0.0.txt
{"GraphemeBreakTest.txt", u"\u200D\u2640"}, // Line 656, old GB 11 test ZWJ x GAZ
{"GraphemeBreakTest.txt", u"\u200D\U0001F466"}, // Line 658, old GB 11 test ZWJ x EBG
{"GraphemeBreakTest.txt", u"\u200D\U0001F466\U0001F3FB"}, // Line 842, old GB 11 test ZWJ x EBG x EModifier
// Line Numbers from pre-release verion of WordBreakTest-10.0.0.txt
{"WordBreakTest.txt", u"\u200D\u261D"}, // Line 1356, ZWJ x EmojiNRK
{"WordBreakTest.txt", u"\u200D\U0001F3FB"}, // Line 1358, ZWJ x EmojiNRK
};
for (int n=0; n<UPRV_LENGTHOF(badTestCases); n++) {
const TestCase &badCase = badTestCases[n];
if (!strcmp(fileName, badCase.fFileName) &&
testCase == UnicodeString(badCase.fString)) {
return logKnownIssue("7270");
}
}
return FALSE;
}
//--------------------------------------------------------------------------------------------
//
// Run tests from one of the boundary test data files distributed by the Unicode Consortium
//
//-------------------------------------------------------------------------------------------
void RBBITest::runUnicodeTestData(const char *fileName, RuleBasedBreakIterator *bi) {
#if !UCONFIG_NO_REGULAR_EXPRESSIONS
UErrorCode status = U_ZERO_ERROR;
//
// Open and read the test data file, put it into a UnicodeString.
//
const char *testDataDirectory = IntlTest::getSourceTestData(status);
char testFileName[1000];
if (testDataDirectory == NULL || strlen(testDataDirectory) >= sizeof(testFileName)) {
dataerrln("Can't open test data. Path too long.");
return;
}
strcpy(testFileName, testDataDirectory);
strcat(testFileName, fileName);
logln("Opening data file %s\n", fileName);
int len;
UChar *testFile = ReadAndConvertFile(testFileName, len, "UTF-8", status);
if (status != U_FILE_ACCESS_ERROR) {
TEST_ASSERT_SUCCESS(status);
TEST_ASSERT(testFile != NULL);
}
if (U_FAILURE(status) || testFile == NULL) {
return; /* something went wrong, error already output */
}
UnicodeString testFileAsString(TRUE, testFile, len);
//
// Parse the test data file using a regular expression.
// Each kind of token is recognized in its own capture group; what type of item was scanned
// is identified by which group had a match.
//
// Caputure Group # 1 2 3 4 5
// Parses this item: divide x hex digits comment \n unrecognized \n
//
UnicodeString tokenExpr("[ \t]*(?:(\\u00F7)|(\\u00D7)|([0-9a-fA-F]+)|((?:#.*?)?$.)|(.*?$.))", -1, US_INV);
RegexMatcher tokenMatcher(tokenExpr, testFileAsString, UREGEX_MULTILINE | UREGEX_DOTALL, status);
UnicodeString testString;
UVector32 breakPositions(status);
int lineNumber = 1;
TEST_ASSERT_SUCCESS(status);
if (U_FAILURE(status)) {
return;
}
//
// Scan through each test case, building up the string to be broken in testString,
// and the positions that should be boundaries in the breakPositions vector.
//
int spin = 0;
while (tokenMatcher.find()) {
if(tokenMatcher.hitEnd()) {
/* Shouldnt Happen(TM). This means we didn't find the symbols we were looking for.
This occurred when the text file was corrupt (wasn't marked as UTF-8)
and caused an infinite loop here on EBCDIC systems!
*/
fprintf(stderr,"FAIL: hit end of file %s for the %8dth time- corrupt data file?\r", fileName, ++spin);
// return;
}
if (tokenMatcher.start(1, status) >= 0) {
// Scanned a divide sign, indicating a break position in the test data.
if (testString.length()>0) {
breakPositions.addElement(testString.length(), status);
}
}
else if (tokenMatcher.start(2, status) >= 0) {
// Scanned an 'x', meaning no break at this position in the test data
// Nothing to be done here.
}
else if (tokenMatcher.start(3, status) >= 0) {
// Scanned Hex digits. Convert them to binary, append to the character data string.
const UnicodeString &hexNumber = tokenMatcher.group(3, status);
int length = hexNumber.length();
if (length<=8) {
char buf[10];
hexNumber.extract (0, length, buf, sizeof(buf), US_INV);
UChar32 c = (UChar32)strtol(buf, NULL, 16);
if (c<=0x10ffff) {
testString.append(c);
} else {
errln("Error: Unicode Character value out of range. \'%s\', line %d.\n",
fileName, lineNumber);
}
} else {
errln("Syntax Error: Hex Unicode Character value must have no more than 8 digits at \'%s\', line %d.\n",
fileName, lineNumber);
}
}
else if (tokenMatcher.start(4, status) >= 0) {
// Scanned to end of a line, possibly skipping over a comment in the process.
// If the line from the file contained test data, run the test now.
if (testString.length() > 0 && !testCaseIsKnownIssue(testString, fileName)) {
checkUnicodeTestCase(fileName, lineNumber, testString, &breakPositions, bi);
}
// Clear out this test case.
// The string and breakPositions vector will be refilled as the next
// test case is parsed.
testString.remove();
breakPositions.removeAllElements();
lineNumber++;
} else {
// Scanner catchall. Something unrecognized appeared on the line.
char token[16];
UnicodeString uToken = tokenMatcher.group(0, status);
uToken.extract(0, uToken.length(), token, (uint32_t)sizeof(token));
token[sizeof(token)-1] = 0;
errln("Syntax error in test data file \'%s\', line %d. Scanning \"%s\"\n", fileName, lineNumber, token);
// Clean up, in preparation for continuing with the next line.
testString.remove();
breakPositions.removeAllElements();
lineNumber++;
}
TEST_ASSERT_SUCCESS(status);
if (U_FAILURE(status)) {
break;
}
}
delete [] testFile;
#endif // !UCONFIG_NO_REGULAR_EXPRESSIONS
}
//--------------------------------------------------------------------------------------------
//
// checkUnicodeTestCase() Run one test case from one of the Unicode Consortium
// test data files. Do only a simple, forward-only check -
// this test is mostly to check that ICU and the Unicode
// data agree with each other.
//
//--------------------------------------------------------------------------------------------
void RBBITest::checkUnicodeTestCase(const char *testFileName, int lineNumber,
const UnicodeString &testString, // Text data to be broken
UVector32 *breakPositions, // Positions where breaks should be found.
RuleBasedBreakIterator *bi) {
int32_t pos; // Break Position in the test string
int32_t expectedI = 0; // Index of expected break position in the vector of expected results.
int32_t expectedPos; // Expected break position (index into test string)
bi->setText(testString);
pos = bi->first();
pos = bi->next();
while (pos != BreakIterator::DONE) {
if (expectedI >= breakPositions->size()) {
errln("Test file \"%s\", line %d, unexpected break found at position %d",
testFileName, lineNumber, pos);
break;
}
expectedPos = breakPositions->elementAti(expectedI);
if (pos < expectedPos) {
errln("Test file \"%s\", line %d, unexpected break found at position %d",
testFileName, lineNumber, pos);
break;
}
if (pos > expectedPos) {
errln("Test file \"%s\", line %d, failed to find expected break at position %d",
testFileName, lineNumber, expectedPos);
break;
}
pos = bi->next();
expectedI++;
}
if (pos==BreakIterator::DONE && expectedI<breakPositions->size()) {
errln("Test file \"%s\", line %d, failed to find expected break at position %d",
testFileName, lineNumber, breakPositions->elementAti(expectedI));
}
}
#if !UCONFIG_NO_REGULAR_EXPRESSIONS
//---------------------------------------------------------------------------------------
//
// classs RBBIMonkeyKind
//
// Monkey Test for Break Iteration
// Abstract interface class. Concrete derived classes independently
// implement the break rules for different iterator types.
//
// The Monkey Test itself uses doesn't know which type of break iterator it is
// testing, but works purely in terms of the interface defined here.
//
//---------------------------------------------------------------------------------------
class RBBIMonkeyKind {
public:
// Return a UVector of UnicodeSets, representing the character classes used
// for this type of iterator.
virtual UVector *charClasses() = 0;
// Set the test text on which subsequent calls to next() will operate
virtual void setText(const UnicodeString &s) = 0;
// Find the next break postion, starting from the prev break position, or from zero.
// Return -1 after reaching end of string.
virtual int32_t next(int32_t i) = 0;
virtual ~RBBIMonkeyKind();
UErrorCode deferredStatus;
protected:
RBBIMonkeyKind();
private:
};
RBBIMonkeyKind::RBBIMonkeyKind() {
deferredStatus = U_ZERO_ERROR;
}
RBBIMonkeyKind::~RBBIMonkeyKind() {
}
//----------------------------------------------------------------------------------------
//
// Random Numbers. Similar to standard lib rand() and srand()
// Not using library to
// 1. Get same results on all platforms.
// 2. Get access to current seed, to more easily reproduce failures.
//
//---------------------------------------------------------------------------------------
static uint32_t m_seed = 1;
static uint32_t m_rand()
{
m_seed = m_seed * 1103515245 + 12345;
return (uint32_t)(m_seed/65536) % 32768;
}
//------------------------------------------------------------------------------------------
//
// class RBBICharMonkey Character (Grapheme Cluster) specific implementation
// of RBBIMonkeyKind.
//
//------------------------------------------------------------------------------------------
class RBBICharMonkey: public RBBIMonkeyKind {
public:
RBBICharMonkey();
virtual ~RBBICharMonkey();
virtual UVector *charClasses();
virtual void setText(const UnicodeString &s);
virtual int32_t next(int32_t i);
private:
UVector *fSets;
UnicodeSet *fCRLFSet;
UnicodeSet *fControlSet;
UnicodeSet *fExtendSet;
UnicodeSet *fZWJSet;
UnicodeSet *fRegionalIndicatorSet;
UnicodeSet *fPrependSet;
UnicodeSet *fSpacingSet;
UnicodeSet *fLSet;
UnicodeSet *fVSet;
UnicodeSet *fTSet;
UnicodeSet *fLVSet;
UnicodeSet *fLVTSet;
UnicodeSet *fHangulSet;
UnicodeSet *fExtendedPictSet;
UnicodeSet *fAnySet;
const UnicodeString *fText;
};
RBBICharMonkey::RBBICharMonkey() {
UErrorCode status = U_ZERO_ERROR;
fText = NULL;
fCRLFSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\r\\n]"), status);
fControlSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[[\\p{Grapheme_Cluster_Break = Control}]]"), status);
fExtendSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[[\\p{Grapheme_Cluster_Break = Extend}]]"), status);
fZWJSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = ZWJ}]"), status);
fRegionalIndicatorSet =
new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = Regional_Indicator}]"), status);
fPrependSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = Prepend}]"), status);
fSpacingSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = SpacingMark}]"), status);
fLSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = L}]"), status);
fVSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = V}]"), status);
fTSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = T}]"), status);
fLVSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = LV}]"), status);
fLVTSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = LVT}]"), status);
fHangulSet = new UnicodeSet();
fHangulSet->addAll(*fLSet);
fHangulSet->addAll(*fVSet);
fHangulSet->addAll(*fTSet);
fHangulSet->addAll(*fLVSet);
fHangulSet->addAll(*fLVTSet);
fExtendedPictSet = new UnicodeSet(u"[:Extended_Pictographic:]", status);
fAnySet = new UnicodeSet(0, 0x10ffff);
fSets = new UVector(status);
fSets->addElement(fCRLFSet, status);
fSets->addElement(fControlSet, status);
fSets->addElement(fExtendSet, status);
fSets->addElement(fRegionalIndicatorSet, status);
if (!fPrependSet->isEmpty()) {
fSets->addElement(fPrependSet, status);
}
fSets->addElement(fSpacingSet, status);
fSets->addElement(fHangulSet, status);
fSets->addElement(fAnySet, status);
fSets->addElement(fZWJSet, status);
fSets->addElement(fExtendedPictSet, status);
if (U_FAILURE(status)) {
deferredStatus = status;
}
}
void RBBICharMonkey::setText(const UnicodeString &s) {
fText = &s;
}
int32_t RBBICharMonkey::next(int32_t prevPos) {
int p0, p1, p2, p3; // Indices of the significant code points around the
// break position being tested. The candidate break
// location is before p2.
int breakPos = -1;
UChar32 c0, c1, c2, c3; // The code points at p0, p1, p2 & p3.
UChar32 cBase; // for (X Extend*) patterns, the X character.
if (U_FAILURE(deferredStatus)) {
return -1;
}
// Previous break at end of string. return DONE.
if (prevPos >= fText->length()) {
return -1;
}
p0 = p1 = p2 = p3 = prevPos;
c3 = fText->char32At(prevPos);
c0 = c1 = c2 = cBase = 0;
(void)p0; // suppress set but not used warning.
(void)c0;
// Loop runs once per "significant" character position in the input text.
for (;;) {
// Move all of the positions forward in the input string.
p0 = p1; c0 = c1;
p1 = p2; c1 = c2;
p2 = p3; c2 = c3;
// Advancd p3 by one codepoint
p3 = fText->moveIndex32(p3, 1);
c3 = fText->char32At(p3);
if (p1 == p2) {
// Still warming up the loop. (won't work with zero length strings, but we don't care)
continue;
}
if (p2 == fText->length()) {
// Reached end of string. Always a break position.
break;
}
// Rule GB3 CR x LF
// No Extend or Format characters may appear between the CR and LF,
// which requires the additional check for p2 immediately following p1.
//
if (c1==0x0D && c2==0x0A && p1==(p2-1)) {
continue;
}
// Rule (GB4). ( Control | CR | LF ) <break>
if (fControlSet->contains(c1) ||
c1 == 0x0D ||
c1 == 0x0A) {
break;
}
// Rule (GB5) <break> ( Control | CR | LF )
//
if (fControlSet->contains(c2) ||
c2 == 0x0D ||
c2 == 0x0A) {
break;
}
// Rule (GB6) L x ( L | V | LV | LVT )
if (fLSet->contains(c1) &&
(fLSet->contains(c2) ||
fVSet->contains(c2) ||
fLVSet->contains(c2) ||
fLVTSet->contains(c2))) {
continue;
}
// Rule (GB7) ( LV | V ) x ( V | T )
if ((fLVSet->contains(c1) || fVSet->contains(c1)) &&
(fVSet->contains(c2) || fTSet->contains(c2))) {
continue;
}
// Rule (GB8) ( LVT | T) x T
if ((fLVTSet->contains(c1) || fTSet->contains(c1)) &&
fTSet->contains(c2)) {
continue;
}
// Rule (GB9) x (Extend | ZWJ)
if (fExtendSet->contains(c2) || fZWJSet->contains(c2)) {
if (!fExtendSet->contains(c1)) {
cBase = c1;
}
continue;
}
// Rule (GB9a) x SpacingMark
if (fSpacingSet->contains(c2)) {
continue;
}
// Rule (GB9b) Prepend x
if (fPrependSet->contains(c1)) {
continue;
}
// Rule (GB11) Extended_Pictographic Extend * ZWJ x Extended_Pictographic
if (fExtendedPictSet->contains(cBase) && fZWJSet->contains(c1) && fExtendedPictSet->contains(c2)) {
continue;
}
// Rule (GB12-13) Regional_Indicator x Regional_Indicator
// Note: The first if condition is a little tricky. We only need to force
// a break if there are three or more contiguous RIs. If there are
// only two, a break following will occur via other rules, and will include
// any trailing extend characters, which is needed behavior.
if (fRegionalIndicatorSet->contains(c0) && fRegionalIndicatorSet->contains(c1)
&& fRegionalIndicatorSet->contains(c2)) {
break;
}
if (fRegionalIndicatorSet->contains(c1) && fRegionalIndicatorSet->contains(c2)) {
continue;
}
// Rule (GB999) Any <break> Any
break;
}
breakPos = p2;
return breakPos;
}
UVector *RBBICharMonkey::charClasses() {
return fSets;
}
RBBICharMonkey::~RBBICharMonkey() {
delete fSets;
delete fCRLFSet;
delete fControlSet;
delete fExtendSet;
delete fRegionalIndicatorSet;
delete fPrependSet;
delete fSpacingSet;
delete fLSet;
delete fVSet;
delete fTSet;
delete fLVSet;
delete fLVTSet;
delete fHangulSet;
delete fAnySet;
delete fZWJSet;
delete fExtendedPictSet;
}
//------------------------------------------------------------------------------------------
//
// class RBBIWordMonkey Word Break specific implementation
// of RBBIMonkeyKind.
//
//------------------------------------------------------------------------------------------
class RBBIWordMonkey: public RBBIMonkeyKind {
public:
RBBIWordMonkey();
virtual ~RBBIWordMonkey();
virtual UVector *charClasses();
virtual void setText(const UnicodeString &s);
virtual int32_t next(int32_t i);
private:
UVector *fSets;
UnicodeSet *fCRSet;
UnicodeSet *fLFSet;
UnicodeSet *fNewlineSet;
UnicodeSet *fRegionalIndicatorSet;
UnicodeSet *fKatakanaSet;
UnicodeSet *fHebrew_LetterSet;
UnicodeSet *fALetterSet;
UnicodeSet *fSingle_QuoteSet;
UnicodeSet *fDouble_QuoteSet;
UnicodeSet *fMidNumLetSet;
UnicodeSet *fMidLetterSet;
UnicodeSet *fMidNumSet;
UnicodeSet *fNumericSet;
UnicodeSet *fFormatSet;
UnicodeSet *fOtherSet;
UnicodeSet *fExtendSet;
UnicodeSet *fExtendNumLetSet;
UnicodeSet *fWSegSpaceSet;
UnicodeSet *fDictionarySet;
UnicodeSet *fZWJSet;
UnicodeSet *fExtendedPictSet;
const UnicodeString *fText;
};
RBBIWordMonkey::RBBIWordMonkey()
{
UErrorCode status = U_ZERO_ERROR;
fSets = new UVector(status);
fCRSet = new UnicodeSet(u"[\\p{Word_Break = CR}]", status);
fLFSet = new UnicodeSet(u"[\\p{Word_Break = LF}]", status);
fNewlineSet = new UnicodeSet(u"[\\p{Word_Break = Newline}]", status);
fKatakanaSet = new UnicodeSet(u"[\\p{Word_Break = Katakana}]", status);
fRegionalIndicatorSet = new UnicodeSet(u"[\\p{Word_Break = Regional_Indicator}]", status);
fHebrew_LetterSet = new UnicodeSet(u"[\\p{Word_Break = Hebrew_Letter}]", status);
fALetterSet = new UnicodeSet(u"[\\p{Word_Break = ALetter}]", status);
fSingle_QuoteSet = new UnicodeSet(u"[\\p{Word_Break = Single_Quote}]", status);
fDouble_QuoteSet = new UnicodeSet(u"[\\p{Word_Break = Double_Quote}]", status);
fMidNumLetSet = new UnicodeSet(u"[\\p{Word_Break = MidNumLet}]", status);
fMidLetterSet = new UnicodeSet(u"[\\p{Word_Break = MidLetter}]", status);
fMidNumSet = new UnicodeSet(u"[\\p{Word_Break = MidNum}]", status);
fNumericSet = new UnicodeSet(u"[\\p{Word_Break = Numeric}]", status);
fFormatSet = new UnicodeSet(u"[\\p{Word_Break = Format}]", status);
fExtendNumLetSet = new UnicodeSet(u"[\\p{Word_Break = ExtendNumLet}]", status);
fExtendSet = new UnicodeSet(u"[\\p{Word_Break = Extend}]", status);
fWSegSpaceSet = new UnicodeSet(u"[\\p{Word_Break = WSegSpace}]", status);
fZWJSet = new UnicodeSet(u"[\\p{Word_Break = ZWJ}]", status);
fExtendedPictSet = new UnicodeSet(u"[:Extended_Pictographic:]", status);
fDictionarySet = new UnicodeSet(u"[[\\uac00-\\ud7a3][:Han:][:Hiragana:]]", status);
fDictionarySet->addAll(*fKatakanaSet);
fDictionarySet->addAll(UnicodeSet(u"[\\p{LineBreak = Complex_Context}]", status));
fALetterSet->removeAll(*fDictionarySet);
fOtherSet = new UnicodeSet();
if(U_FAILURE(status)) {
IntlTest::gTest->errln("%s:%d %s", __FILE__, __LINE__, u_errorName(status));
deferredStatus = status;
return;
}
fOtherSet->complement();
fOtherSet->removeAll(*fCRSet);
fOtherSet->removeAll(*fLFSet);
fOtherSet->removeAll(*fNewlineSet);
fOtherSet->removeAll(*fKatakanaSet);
fOtherSet->removeAll(*fHebrew_LetterSet);
fOtherSet->removeAll(*fALetterSet);
fOtherSet->removeAll(*fSingle_QuoteSet);
fOtherSet->removeAll(*fDouble_QuoteSet);
fOtherSet->removeAll(*fMidLetterSet);
fOtherSet->removeAll(*fMidNumSet);
fOtherSet->removeAll(*fNumericSet);
fOtherSet->removeAll(*fExtendNumLetSet);
fOtherSet->removeAll(*fWSegSpaceSet);
fOtherSet->removeAll(*fFormatSet);
fOtherSet->removeAll(*fExtendSet);
fOtherSet->removeAll(*fRegionalIndicatorSet);
fOtherSet->removeAll(*fZWJSet);
fOtherSet->removeAll(*fExtendedPictSet);
// Inhibit dictionary characters from being tested at all.
fOtherSet->removeAll(*fDictionarySet);
fSets->addElement(fCRSet, status);
fSets->addElement(fLFSet, status);
fSets->addElement(fNewlineSet, status);
fSets->addElement(fRegionalIndicatorSet, status);
fSets->addElement(fHebrew_LetterSet, status);
fSets->addElement(fALetterSet, status);
fSets->addElement(fSingle_QuoteSet, status);
fSets->addElement(fDouble_QuoteSet, status);
//fSets->addElement(fKatakanaSet, status); // Omit Katakana from fSets, which omits Katakana characters
// from the test data. They are all in the dictionary set,
// which this (old, to be retired) monkey test cannot handle.
fSets->addElement(fMidLetterSet, status);
fSets->addElement(fMidNumLetSet, status);
fSets->addElement(fMidNumSet, status);
fSets->addElement(fNumericSet, status);
fSets->addElement(fFormatSet, status);
fSets->addElement(fExtendSet, status);
fSets->addElement(fOtherSet, status);
fSets->addElement(fExtendNumLetSet, status);
fSets->addElement(fWSegSpaceSet, status);
fSets->addElement(fZWJSet, status);
fSets->addElement(fExtendedPictSet, status);
if (U_FAILURE(status)) {
deferredStatus = status;
}
}
void RBBIWordMonkey::setText(const UnicodeString &s) {
fText = &s;
}
int32_t RBBIWordMonkey::next(int32_t prevPos) {
int p0, p1, p2, p3; // Indices of the significant code points around the
// break position being tested. The candidate break
// location is before p2.
int breakPos = -1;
UChar32 c0, c1, c2, c3; // The code points at p0, p1, p2 & p3.
if (U_FAILURE(deferredStatus)) {
return -1;
}
// Prev break at end of string. return DONE.
if (prevPos >= fText->length()) {
return -1;
}
p0 = p1 = p2 = p3 = prevPos;
c3 = fText->char32At(prevPos);
c0 = c1 = c2 = 0;
(void)p0; // Suppress set but not used warning.
// Loop runs once per "significant" character position in the input text.
for (;;) {
// Move all of the positions forward in the input string.
p0 = p1; c0 = c1;
p1 = p2; c1 = c2;
p2 = p3; c2 = c3;
// Advancd p3 by X(Extend | Format)* Rule 4
// But do not advance over Extend & Format following a new line. (Unicode 5.1 change)
do {
p3 = fText->moveIndex32(p3, 1);
c3 = fText->char32At(p3);
if (fCRSet->contains(c2) || fLFSet->contains(c2) || fNewlineSet->contains(c2)) {
break;
};
}
while (fFormatSet->contains(c3) || fExtendSet->contains(c3) || fZWJSet->contains(c3));
if (p1 == p2) {
// Still warming up the loop. (won't work with zero length strings, but we don't care)
continue;
}
if (p2 == fText->length()) {
// Reached end of string. Always a break position.
break;
}
// Rule (3) CR x LF
// No Extend or Format characters may appear between the CR and LF,
// which requires the additional check for p2 immediately following p1.
//
if (c1==0x0D && c2==0x0A) {
continue;
}
// Rule (3a) Break before and after newlines (including CR and LF)
//
if (fCRSet->contains(c1) || fLFSet->contains(c1) || fNewlineSet->contains(c1)) {
break;
};
if (fCRSet->contains(c2) || fLFSet->contains(c2) || fNewlineSet->contains(c2)) {
break;
};
// Rule (3c) ZWJ x Extended_Pictographic
// Not ignoring extend chars, so peek into input text to
// get the potential ZWJ, the character immediately preceding c2.
// Sloppy UChar32 indexing: p2-1 may reference trail half
// but char32At will get the full code point.
if (fZWJSet->contains(fText->char32At(p2-1)) && fExtendedPictSet->contains(c2)) {
continue;
}
// Rule (3d) Keep horizontal whitespace together.
if (fWSegSpaceSet->contains(fText->char32At(p2-1)) && fWSegSpaceSet->contains(c2)) {
continue;
}
// Rule (5). (ALetter | Hebrew_Letter) x (ALetter | Hebrew_Letter)
if ((fALetterSet->contains(c1) || fHebrew_LetterSet->contains(c1)) &&
(fALetterSet->contains(c2) || fHebrew_LetterSet->contains(c2))) {
continue;
}
// Rule (6) (ALetter | Hebrew_Letter) x (MidLetter | MidNumLet | Single_Quote) (ALetter | Hebrew_Letter)
//
if ( (fALetterSet->contains(c1) || fHebrew_LetterSet->contains(c1)) &&
(fMidLetterSet->contains(c2) || fMidNumLetSet->contains(c2) || fSingle_QuoteSet->contains(c2)) &&
(fALetterSet->contains(c3) || fHebrew_LetterSet->contains(c3))) {
continue;
}
// Rule (7) (ALetter | Hebrew_Letter) (MidLetter | MidNumLet | Single_Quote) x (ALetter | Hebrew_Letter)
if ((fALetterSet->contains(c0) || fHebrew_LetterSet->contains(c0)) &&
(fMidLetterSet->contains(c1) || fMidNumLetSet->contains(c1) || fSingle_QuoteSet->contains(c1)) &&
(fALetterSet->contains(c2) || fHebrew_LetterSet->contains(c2))) {
continue;
}
// Rule (7a) Hebrew_Letter x Single_Quote
if (fHebrew_LetterSet->contains(c1) && fSingle_QuoteSet->contains(c2)) {
continue;
}
// Rule (7b) Hebrew_Letter x Double_Quote Hebrew_Letter
if (fHebrew_LetterSet->contains(c1) && fDouble_QuoteSet->contains(c2) && fHebrew_LetterSet->contains(c3)) {
continue;
}
// Rule (7c) Hebrew_Letter Double_Quote x Hebrew_Letter
if (fHebrew_LetterSet->contains(c0) && fDouble_QuoteSet->contains(c1) && fHebrew_LetterSet->contains(c2)) {
continue;
}
// Rule (8) Numeric x Numeric
if (fNumericSet->contains(c1) &&
fNumericSet->contains(c2)) {
continue;
}
// Rule (9) (ALetter | Hebrew_Letter) x Numeric
if ((fALetterSet->contains(c1) || fHebrew_LetterSet->contains(c1)) &&
fNumericSet->contains(c2)) {
continue;
}
// Rule (10) Numeric x (ALetter | Hebrew_Letter)
if (fNumericSet->contains(c1) &&
(fALetterSet->contains(c2) || fHebrew_LetterSet->contains(c2))) {
continue;
}
// Rule (11) Numeric (MidNum | MidNumLet | Single_Quote) x Numeric
if (fNumericSet->contains(c0) &&
(fMidNumSet->contains(c1) || fMidNumLetSet->contains(c1) || fSingle_QuoteSet->contains(c1)) &&
fNumericSet->contains(c2)) {
continue;
}
// Rule (12) Numeric x (MidNum | MidNumLet | SingleQuote) Numeric
if (fNumericSet->contains(c1) &&
(fMidNumSet->contains(c2) || fMidNumLetSet->contains(c2) || fSingle_QuoteSet->contains(c2)) &&
fNumericSet->contains(c3)) {
continue;
}
// Rule (13) Katakana x Katakana
// Note: matches UAX 29 rules, but doesn't come into play for ICU because
// all Katakana are handled by the dictionary breaker.
if (fKatakanaSet->contains(c1) &&
fKatakanaSet->contains(c2)) {
continue;
}
// Rule 13a (ALetter | Hebrew_Letter | Numeric | KataKana | ExtendNumLet) x ExtendNumLet
if ((fALetterSet->contains(c1) || fHebrew_LetterSet->contains(c1) ||fNumericSet->contains(c1) ||
fKatakanaSet->contains(c1) || fExtendNumLetSet->contains(c1)) &&
fExtendNumLetSet->contains(c2)) {
continue;
}
// Rule 13b ExtendNumLet x (ALetter | Hebrew_Letter | Numeric | Katakana)
if (fExtendNumLetSet->contains(c1) &&
(fALetterSet->contains(c2) || fHebrew_LetterSet->contains(c2) ||
fNumericSet->contains(c2) || fKatakanaSet->contains(c2))) {
continue;
}
// Rule 15 - 17 Group pairs of Regional Indicators.
if (fRegionalIndicatorSet->contains(c0) && fRegionalIndicatorSet->contains(c1)) {
break;
}
if (fRegionalIndicatorSet->contains(c1) && fRegionalIndicatorSet->contains(c2)) {
continue;
}
// Rule 999. Break found here.
break;
}
breakPos = p2;
return breakPos;
}
UVector *RBBIWordMonkey::charClasses() {
return fSets;
}
RBBIWordMonkey::~RBBIWordMonkey() {
delete fSets;
delete fCRSet;
delete fLFSet;
delete fNewlineSet;
delete fKatakanaSet;
delete fHebrew_LetterSet;
delete fALetterSet;
delete fSingle_QuoteSet;
delete fDouble_QuoteSet;
delete fMidNumLetSet;
delete fMidLetterSet;
delete fMidNumSet;
delete fNumericSet;
delete fFormatSet;
delete fExtendSet;
delete fExtendNumLetSet;
delete fWSegSpaceSet;
delete fRegionalIndicatorSet;
delete fDictionarySet;
delete fOtherSet;
delete fZWJSet;
delete fExtendedPictSet;
}
//------------------------------------------------------------------------------------------
//
// class RBBISentMonkey Sentence Break specific implementation
// of RBBIMonkeyKind.
//
//------------------------------------------------------------------------------------------
class RBBISentMonkey: public RBBIMonkeyKind {
public:
RBBISentMonkey();
virtual ~RBBISentMonkey();
virtual UVector *charClasses();
virtual void setText(const UnicodeString &s);
virtual int32_t next(int32_t i);
private:
int moveBack(int posFrom);
int moveForward(int posFrom);
UChar32 cAt(int pos);
UVector *fSets;
UnicodeSet *fSepSet;
UnicodeSet *fFormatSet;
UnicodeSet *fSpSet;
UnicodeSet *fLowerSet;
UnicodeSet *fUpperSet;
UnicodeSet *fOLetterSet;
UnicodeSet *fNumericSet;
UnicodeSet *fATermSet;
UnicodeSet *fSContinueSet;
UnicodeSet *fSTermSet;
UnicodeSet *fCloseSet;
UnicodeSet *fOtherSet;
UnicodeSet *fExtendSet;
const UnicodeString *fText;
};
RBBISentMonkey::RBBISentMonkey()
{
UErrorCode status = U_ZERO_ERROR;
fSets = new UVector(status);
// Separator Set Note: Beginning with Unicode 5.1, CR and LF were removed from the separator
// set and made into character classes of their own. For the monkey impl,
// they remain in SEP, since Sep always appears with CR and LF in the rules.
fSepSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Sep} \\u000a \\u000d]"), status);
fFormatSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Format}]"), status);
fSpSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Sp}]"), status);
fLowerSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Lower}]"), status);
fUpperSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Upper}]"), status);
fOLetterSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = OLetter}]"), status);
fNumericSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Numeric}]"), status);
fATermSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = ATerm}]"), status);
fSContinueSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = SContinue}]"), status);
fSTermSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = STerm}]"), status);
fCloseSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Close}]"), status);
fExtendSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Extend}]"), status);
fOtherSet = new UnicodeSet();
if(U_FAILURE(status)) {
deferredStatus = status;
return;
}
fOtherSet->complement();
fOtherSet->removeAll(*fSepSet);
fOtherSet->removeAll(*fFormatSet);
fOtherSet->removeAll(*fSpSet);
fOtherSet->removeAll(*fLowerSet);
fOtherSet->removeAll(*fUpperSet);
fOtherSet->removeAll(*fOLetterSet);
fOtherSet->removeAll(*fNumericSet);
fOtherSet->removeAll(*fATermSet);
fOtherSet->removeAll(*fSContinueSet);
fOtherSet->removeAll(*fSTermSet);
fOtherSet->removeAll(*fCloseSet);
fOtherSet->removeAll(*fExtendSet);
fSets->addElement(fSepSet, status);
fSets->addElement(fFormatSet, status);
fSets->addElement(fSpSet, status);
fSets->addElement(fLowerSet, status);
fSets->addElement(fUpperSet, status);
fSets->addElement(fOLetterSet, status);
fSets->addElement(fNumericSet, status);
fSets->addElement(fATermSet, status);
fSets->addElement(fSContinueSet, status);
fSets->addElement(fSTermSet, status);
fSets->addElement(fCloseSet, status);
fSets->addElement(fOtherSet, status);
fSets->addElement(fExtendSet, status);
if (U_FAILURE(status)) {
deferredStatus = status;
}
}
void RBBISentMonkey::setText(const UnicodeString &s) {
fText = &s;
}
UVector *RBBISentMonkey::charClasses() {
return fSets;
}
// moveBack() Find the "significant" code point preceding the index i.
// Skips over ($Extend | $Format)* .
//
int RBBISentMonkey::moveBack(int i) {
if (i <= 0) {
return -1;
}
UChar32 c;
int32_t j = i;
do {
j = fText->moveIndex32(j, -1);
c = fText->char32At(j);
}
while (j>0 &&(fFormatSet->contains(c) || fExtendSet->contains(c)));
return j;
}
int RBBISentMonkey::moveForward(int i) {
if (i>=fText->length()) {
return fText->length();
}
UChar32 c;
int32_t j = i;
do {
j = fText->moveIndex32(j, 1);
c = cAt(j);
}
while (fFormatSet->contains(c) || fExtendSet->contains(c));
return j;
}
UChar32 RBBISentMonkey::cAt(int pos) {
if (pos<0 || pos>=fText->length()) {
return -1;
} else {
return fText->char32At(pos);
}
}
int32_t RBBISentMonkey::next(int32_t prevPos) {
int p0, p1, p2, p3; // Indices of the significant code points around the
// break position being tested. The candidate break
// location is before p2.
int breakPos = -1;
UChar32 c0, c1, c2, c3; // The code points at p0, p1, p2 & p3.
UChar32 c;
if (U_FAILURE(deferredStatus)) {
return -1;
}
// Prev break at end of string. return DONE.
if (prevPos >= fText->length()) {
return -1;
}
p0 = p1 = p2 = p3 = prevPos;
c3 = fText->char32At(prevPos);
c0 = c1 = c2 = 0;
(void)p0; // Suppress set but not used warning.
// Loop runs once per "significant" character position in the input text.
for (;;) {
// Move all of the positions forward in the input string.
p0 = p1; c0 = c1;
p1 = p2; c1 = c2;
p2 = p3; c2 = c3;
// Advancd p3 by X(Extend | Format)* Rule 4
p3 = moveForward(p3);
c3 = cAt(p3);
// Rule (3) CR x LF
if (c1==0x0d && c2==0x0a && p2==(p1+1)) {
continue;
}
// Rule (4). Sep <break>
if (fSepSet->contains(c1)) {
p2 = p1+1; // Separators don't combine with Extend or Format.
break;
}
if (p2 >= fText->length()) {
// Reached end of string. Always a break position.
break;
}
if (p2 == prevPos) {
// Still warming up the loop. (won't work with zero length strings, but we don't care)
continue;
}
// Rule (6). ATerm x Numeric
if (fATermSet->contains(c1) && fNumericSet->contains(c2)) {
continue;
}
// Rule (7). (Upper | Lower) ATerm x Uppper
if ((fUpperSet->contains(c0) || fLowerSet->contains(c0)) &&
fATermSet->contains(c1) && fUpperSet->contains(c2)) {
continue;
}
// Rule (8) ATerm Close* Sp* x (not (OLettter | Upper | Lower | Sep | STerm | ATerm))* Lower
// Note: STerm | ATerm are added to the negated part of the expression by a
// note to the Unicode 5.0 documents.
int p8 = p1;
while (fSpSet->contains(cAt(p8))) {
p8 = moveBack(p8);
}
while (fCloseSet->contains(cAt(p8))) {
p8 = moveBack(p8);
}
if (fATermSet->contains(cAt(p8))) {
p8=p2;
for (;;) {
c = cAt(p8);
if (c==-1 || fOLetterSet->contains(c) || fUpperSet->contains(c) ||
fLowerSet->contains(c) || fSepSet->contains(c) ||
fATermSet->contains(c) || fSTermSet->contains(c)) {
break;
}
p8 = moveForward(p8);
}
if (fLowerSet->contains(cAt(p8))) {
continue;
}
}
// Rule 8a (STerm | ATerm) Close* Sp* x (SContinue | STerm | ATerm);
if (fSContinueSet->contains(c2) || fSTermSet->contains(c2) || fATermSet->contains(c2)) {
p8 = p1;
while (fSpSet->contains(cAt(p8))) {
p8 = moveBack(p8);
}
while (fCloseSet->contains(cAt(p8))) {
p8 = moveBack(p8);
}
c = cAt(p8);
if (fSTermSet->contains(c) || fATermSet->contains(c)) {
continue;
}
}
// Rule (9) (STerm | ATerm) Close* x (Close | Sp | Sep | CR | LF)
int p9 = p1;
while (fCloseSet->contains(cAt(p9))) {
p9 = moveBack(p9);
}
c = cAt(p9);
if ((fSTermSet->contains(c) || fATermSet->contains(c))) {
if (fCloseSet->contains(c2) || fSpSet->contains(c2) || fSepSet->contains(c2)) {
continue;
}
}
// Rule (10) (Sterm | ATerm) Close* Sp* x (Sp | Sep | CR | LF)
int p10 = p1;
while (fSpSet->contains(cAt(p10))) {
p10 = moveBack(p10);
}
while (fCloseSet->contains(cAt(p10))) {
p10 = moveBack(p10);
}
if (fSTermSet->contains(cAt(p10)) || fATermSet->contains(cAt(p10))) {
if (fSpSet->contains(c2) || fSepSet->contains(c2)) {
continue;
}
}
// Rule (11) (STerm | ATerm) Close* Sp* (Sep | CR | LF)? <break>
int p11 = p1;
if (fSepSet->contains(cAt(p11))) {
p11 = moveBack(p11);
}
while (fSpSet->contains(cAt(p11))) {
p11 = moveBack(p11);
}
while (fCloseSet->contains(cAt(p11))) {
p11 = moveBack(p11);
}
if (fSTermSet->contains(cAt(p11)) || fATermSet->contains(cAt(p11))) {
break;
}
// Rule (12) Any x Any
continue;
}
breakPos = p2;
return breakPos;
}
RBBISentMonkey::~RBBISentMonkey() {
delete fSets;
delete fSepSet;
delete fFormatSet;
delete fSpSet;
delete fLowerSet;
delete fUpperSet;
delete fOLetterSet;
delete fNumericSet;
delete fATermSet;
delete fSContinueSet;
delete fSTermSet;
delete fCloseSet;
delete fOtherSet;
delete fExtendSet;
}
//-------------------------------------------------------------------------------------------
//
// RBBILineMonkey
//
//-------------------------------------------------------------------------------------------
class RBBILineMonkey: public RBBIMonkeyKind {
public:
RBBILineMonkey();
virtual ~RBBILineMonkey();
virtual UVector *charClasses();
virtual void setText(const UnicodeString &s);
virtual int32_t next(int32_t i);
virtual void rule9Adjust(int32_t pos, UChar32 *posChar, int32_t *nextPos, UChar32 *nextChar);
private:
UVector *fSets;
UnicodeSet *fBK;
UnicodeSet *fCR;
UnicodeSet *fLF;
UnicodeSet *fCM;
UnicodeSet *fNL;
UnicodeSet *fSG;
UnicodeSet *fWJ;
UnicodeSet *fZW;
UnicodeSet *fGL;
UnicodeSet *fCB;
UnicodeSet *fSP;
UnicodeSet *fB2;
UnicodeSet *fBA;
UnicodeSet *fBB;
UnicodeSet *fHY;
UnicodeSet *fH2;
UnicodeSet *fH3;
UnicodeSet *fCL;
UnicodeSet *fCP;
UnicodeSet *fEX;
UnicodeSet *fIN;
UnicodeSet *fJL;
UnicodeSet *fJV;
UnicodeSet *fJT;
UnicodeSet *fNS;
UnicodeSet *fOP;
UnicodeSet *fQU;
UnicodeSet *fIS;
UnicodeSet *fNU;
UnicodeSet *fPO;
UnicodeSet *fPR;
UnicodeSet *fSY;
UnicodeSet *fAI;
UnicodeSet *fAL;
UnicodeSet *fCJ;
UnicodeSet *fHL;
UnicodeSet *fID;
UnicodeSet *fRI;
UnicodeSet *fXX;
UnicodeSet *fEB;
UnicodeSet *fEM;
UnicodeSet *fZJ;
BreakIterator *fCharBI;
const UnicodeString *fText;
RegexMatcher *fNumberMatcher;
};
RBBILineMonkey::RBBILineMonkey() :
RBBIMonkeyKind(),
fSets(NULL),
fCharBI(NULL),
fText(NULL),
fNumberMatcher(NULL)
{
if (U_FAILURE(deferredStatus)) {
return;
}
UErrorCode status = U_ZERO_ERROR;
fSets = new UVector(status);
fBK = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_Break=BK}]"), status);
fCR = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=CR}]"), status);
fLF = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=LF}]"), status);
fCM = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=CM}]"), status);
fNL = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=NL}]"), status);
fWJ = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=WJ}]"), status);
fZW = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=ZW}]"), status);
fGL = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=GL}]"), status);
fCB = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=CB}]"), status);
fSP = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=SP}]"), status);
fB2 = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=B2}]"), status);
fBA = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=BA}]"), status);
fBB = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=BB}]"), status);
fHY = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=HY}]"), status);
fH2 = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=H2}]"), status);
fH3 = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=H3}]"), status);
fCL = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=CL}]"), status);
fCP = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=CP}]"), status);
fEX = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=EX}]"), status);
fIN = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=IN}]"), status);
fJL = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=JL}]"), status);
fJV = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=JV}]"), status);
fJT = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=JT}]"), status);
fNS = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=NS}]"), status);
fOP = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=OP}]"), status);
fQU = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=QU}]"), status);
fIS = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=IS}]"), status);
fNU = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=NU}]"), status);
fPO = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=PO}]"), status);
fPR = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=PR}]"), status);
fSY = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=SY}]"), status);
fAI = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=AI}]"), status);
fAL = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=AL}]"), status);
fCJ = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=CJ}]"), status);
fHL = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=HL}]"), status);
fID = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=ID}]"), status);
fRI = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=RI}]"), status);
fSG = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\ud800-\\udfff]"), status);
fXX = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=XX}]"), status);
fEB = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=EB}]"), status);
fEM = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=EM}]"), status);
fZJ = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=ZWJ}]"), status);
if (U_FAILURE(status)) {
deferredStatus = status;
return;
}
fAL->addAll(*fXX); // Default behavior for XX is identical to AL
fAL->addAll(*fAI); // Default behavior for AI is identical to AL
fAL->addAll(*fSG); // Default behavior for SG is identical to AL.
fNS->addAll(*fCJ); // Default behavior for CJ is identical to NS.
fCM->addAll(*fZJ); // ZWJ behaves as a CM.
fSets->addElement(fBK, status);
fSets->addElement(fCR, status);
fSets->addElement(fLF, status);
fSets->addElement(fCM, status);
fSets->addElement(fNL, status);
fSets->addElement(fWJ, status);
fSets->addElement(fZW, status);
fSets->addElement(fGL, status);
fSets->addElement(fCB, status);
fSets->addElement(fSP, status);
fSets->addElement(fB2, status);
fSets->addElement(fBA, status);
fSets->addElement(fBB, status);
fSets->addElement(fHY, status);
fSets->addElement(fH2, status);
fSets->addElement(fH3, status);
fSets->addElement(fCL, status);
fSets->addElement(fCP, status);
fSets->addElement(fEX, status);
fSets->addElement(fIN, status);
fSets->addElement(fJL, status);
fSets->addElement(fJT, status);
fSets->addElement(fJV, status);
fSets->addElement(fNS, status);
fSets->addElement(fOP, status);
fSets->addElement(fQU, status);
fSets->addElement(fIS, status);
fSets->addElement(fNU, status);
fSets->addElement(fPO, status);
fSets->addElement(fPR, status);
fSets->addElement(fSY, status);
fSets->addElement(fAI, status);
fSets->addElement(fAL, status);
fSets->addElement(fHL, status);
fSets->addElement(fID, status);
fSets->addElement(fWJ, status);
fSets->addElement(fRI, status);
fSets->addElement(fSG, status);
fSets->addElement(fEB, status);
fSets->addElement(fEM, status);
fSets->addElement(fZJ, status);
const char *rules =
"((\\p{Line_Break=PR}|\\p{Line_Break=PO})(\\p{Line_Break=CM}|\\u200d)*)?"
"((\\p{Line_Break=OP}|\\p{Line_Break=HY})(\\p{Line_Break=CM}|\\u200d)*)?"
"\\p{Line_Break=NU}(\\p{Line_Break=CM}|\\u200d)*"
"((\\p{Line_Break=NU}|\\p{Line_Break=IS}|\\p{Line_Break=SY})(\\p{Line_Break=CM}|\\u200d)*)*"
"((\\p{Line_Break=CL}|\\p{Line_Break=CP})(\\p{Line_Break=CM}|\\u200d)*)?"
"((\\p{Line_Break=PR}|\\p{Line_Break=PO})(\\p{Line_Break=CM}|\\u200d)*)?";
fNumberMatcher = new RegexMatcher(
UnicodeString(rules, -1, US_INV), 0, status);
fCharBI = BreakIterator::createCharacterInstance(Locale::getEnglish(), status);
if (U_FAILURE(status)) {
deferredStatus = status;
}
}
void RBBILineMonkey::setText(const UnicodeString &s) {
fText = &s;
fCharBI->setText(s);
fNumberMatcher->reset(s);
}
//
// rule9Adjust
// Line Break TR rules 9 and 10 implementation.
// This deals with combining marks and other sequences that
// that must be treated as if they were something other than what they actually are.
//
// This is factored out into a separate function because it must be applied twice for
// each potential break, once to the chars before the position being checked, then
// again to the text following the possible break.
//
void RBBILineMonkey::rule9Adjust(int32_t pos, UChar32 *posChar, int32_t *nextPos, UChar32 *nextChar) {
if (pos == -1) {
// Invalid initial position. Happens during the warmup iteration of the
// main loop in next().
return;
}
int32_t nPos = *nextPos;
// LB 9 Keep combining sequences together.
// advance over any CM class chars. Note that Line Break CM is different
// from the normal Grapheme Extend property.
if (!(fSP->contains(*posChar) || fBK->contains(*posChar) || *posChar==0x0d ||
*posChar==0x0a ||fNL->contains(*posChar) || fZW->contains(*posChar))) {
for (;;) {
*nextChar = fText->char32At(nPos);
if (!fCM->contains(*nextChar)) {
break;
}
nPos = fText->moveIndex32(nPos, 1);
}
}
// LB 9 Treat X CM* as if it were x.
// No explicit action required.
// LB 10 Treat any remaining combining mark as AL
if (fCM->contains(*posChar)) {
*posChar = u'A';
}
// Push the updated nextPos and nextChar back to our caller.
// This only makes a difference if posChar got bigger by consuming a
// combining sequence.
*nextPos = nPos;
*nextChar = fText->char32At(nPos);
}
int32_t RBBILineMonkey::next(int32_t startPos) {
UErrorCode status = U_ZERO_ERROR;
int32_t pos; // Index of the char following a potential break position
UChar32 thisChar; // Character at above position "pos"
int32_t prevPos; // Index of the char preceding a potential break position
UChar32 prevChar; // Character at above position. Note that prevChar
// and thisChar may not be adjacent because combining
// characters between them will be ignored.
int32_t prevPosX2; // Second previous character. Wider context for LB21a.
UChar32 prevCharX2;
int32_t nextPos; // Index of the next character following pos.
// Usually skips over combining marks.
int32_t nextCPPos; // Index of the code point following "pos."
// May point to a combining mark.
int32_t tPos; // temp value.
UChar32 c;
if (U_FAILURE(deferredStatus)) {
return -1;
}
if (startPos >= fText->length()) {
return -1;
}
// Initial values for loop. Loop will run the first time without finding breaks,
// while the invalid values shift out and the "this" and
// "prev" positions are filled in with good values.
pos = prevPos = prevPosX2 = -1; // Invalid value, serves as flag for initial loop iteration.
thisChar = prevChar = prevCharX2 = 0;
nextPos = nextCPPos = startPos;
// Loop runs once per position in the test text, until a break position
// is found.
for (;;) {
prevPosX2 = prevPos;
prevCharX2 = prevChar;
prevPos = pos;
prevChar = thisChar;
pos = nextPos;
thisChar = fText->char32At(pos);
nextCPPos = fText->moveIndex32(pos, 1);
nextPos = nextCPPos;
// Rule LB2 - Break at end of text.
if (pos >= fText->length()) {
break;
}
// Rule LB 9 - adjust for combining sequences.
// We do this one out-of-order because the adjustment does not change anything
// that would match rules LB 3 - LB 6, but after the adjustment, LB 3-6 do need to
// be applied.
rule9Adjust(prevPos, &prevChar, &pos, &thisChar);
nextCPPos = nextPos = fText->moveIndex32(pos, 1);
c = fText->char32At(nextPos);
rule9Adjust(pos, &thisChar, &nextPos, &c);
// If the loop is still warming up - if we haven't shifted the initial
// -1 positions out of prevPos yet - loop back to advance the
// position in the input without any further looking for breaks.
if (prevPos == -1) {
continue;
}
// LB 4 Always break after hard line breaks,
if (fBK->contains(prevChar)) {
break;
}
// LB 5 Break after CR, LF, NL, but not inside CR LF
if (prevChar == 0x0d && thisChar == 0x0a) {
continue;
}
if (prevChar == 0x0d ||
prevChar == 0x0a ||
prevChar == 0x85) {
break;
}
// LB 6 Don't break before hard line breaks
if (thisChar == 0x0d || thisChar == 0x0a || thisChar == 0x85 ||
fBK->contains(thisChar)) {
continue;
}
// LB 7 Don't break before spaces or zero-width space.
if (fSP->contains(thisChar)) {
continue;
}
if (fZW->contains(thisChar)) {
continue;
}
// LB 8 Break after zero width space
if (fZW->contains(prevChar)) {
break;
}
// LB 25 Numbers
// Move this test up, before LB8a, because numbers can match a longer sequence that would
// also match 8a. e.g. NU ZWJ IS PO (ZWJ acts like CM)
if (fNumberMatcher->lookingAt(prevPos, status)) {
if (U_FAILURE(status)) {
break;
}
// Matched a number. But could have been just a single digit, which would
// not represent a "no break here" between prevChar and thisChar
int32_t numEndIdx = fNumberMatcher->end(status); // idx of first char following num
if (numEndIdx > pos) {
// Number match includes at least our two chars being checked
if (numEndIdx > nextPos) {
// Number match includes additional chars. Update pos and nextPos
// so that next loop iteration will continue at the end of the number,
// checking for breaks between last char in number & whatever follows.
pos = nextPos = numEndIdx;
do {
pos = fText->moveIndex32(pos, -1);
thisChar = fText->char32At(pos);
} while (fCM->contains(thisChar));
}
continue;
}
}
// LB 8a ZWJ x
// The monkey test's way of ignoring combining characters doesn't work
// for this rule. ZJ is also a CM. Need to get the actual character
// preceding "thisChar", not ignoring combining marks, possibly ZJ.
{
int32_t prevIdx = fText->moveIndex32(pos, -1);
UChar32 prevC = fText->char32At(prevIdx);
if (fZJ->contains(prevC)) {
continue;
}
}
// LB 9, 10 Already done, at top of loop.
//
// LB 11 Do not break before or after WORD JOINER and related characters.
// x WJ
// WJ x
//
if (fWJ->contains(thisChar) || fWJ->contains(prevChar)) {
continue;
}
// LB 12
// GL x
if (fGL->contains(prevChar)) {
continue;
}
// LB 12a
// [^SP BA HY] x GL
if (!(fSP->contains(prevChar) ||
fBA->contains(prevChar) ||
fHY->contains(prevChar) ) && fGL->contains(thisChar)) {
continue;
}
// LB 13 Don't break before closings.
// NU x CL, NU x CP and NU x IS are not matched here so that they will
// fall into LB 17 and the more general number regular expression.
//
if ((!fNU->contains(prevChar) && fCL->contains(thisChar)) ||
(!fNU->contains(prevChar) && fCP->contains(thisChar)) ||
fEX->contains(thisChar) ||
(!fNU->contains(prevChar) && fIS->contains(thisChar)) ||
(!fNU->contains(prevChar) && fSY->contains(thisChar))) {
continue;
}
// LB 14 Don't break after OP SP*
// Scan backwards, checking for this sequence.
// The OP char could include combining marks, so we actually check for
// OP CM* SP*
// Another Twist: The Rule 67 fixes may have changed a SP CM
// sequence into a ID char, so before scanning back through spaces,
// verify that prevChar is indeed a space. The prevChar variable
// may differ from fText[prevPos]
tPos = prevPos;
if (fSP->contains(prevChar)) {
while (tPos > 0 && fSP->contains(fText->char32At(tPos))) {
tPos=fText->moveIndex32(tPos, -1);
}
}
while (tPos > 0 && fCM->contains(fText->char32At(tPos))) {
tPos=fText->moveIndex32(tPos, -1);
}
if (fOP->contains(fText->char32At(tPos))) {
continue;
}
// LB 15 QU SP* x OP
if (fOP->contains(thisChar)) {
// Scan backwards from prevChar to see if it is preceded by QU CM* SP*
int tPos = prevPos;
while (tPos>0 && fSP->contains(fText->char32At(tPos))) {
tPos = fText->moveIndex32(tPos, -1);
}
while (tPos>0 && fCM->contains(fText->char32At(tPos))) {
tPos = fText->moveIndex32(tPos, -1);
}
if (fQU->contains(fText->char32At(tPos))) {
continue;
}
}
// LB 16 (CL | CP) SP* x NS
// Scan backwards for SP* CM* (CL | CP)
if (fNS->contains(thisChar)) {
int tPos = prevPos;
while (tPos>0 && fSP->contains(fText->char32At(tPos))) {
tPos = fText->moveIndex32(tPos, -1);
}
while (tPos>0 && fCM->contains(fText->char32At(tPos))) {
tPos = fText->moveIndex32(tPos, -1);
}
if (fCL->contains(fText->char32At(tPos)) || fCP->contains(fText->char32At(tPos))) {
continue;
}
}
// LB 17 B2 SP* x B2
if (fB2->contains(thisChar)) {
// Scan backwards, checking for the B2 CM* SP* sequence.
tPos = prevPos;
if (fSP->contains(prevChar)) {
while (tPos > 0 && fSP->contains(fText->char32At(tPos))) {
tPos=fText->moveIndex32(tPos, -1);
}
}
while (tPos > 0 && fCM->contains(fText->char32At(tPos))) {
tPos=fText->moveIndex32(tPos, -1);
}
if (fB2->contains(fText->char32At(tPos))) {
continue;
}
}
// LB 18 break after space
if (fSP->contains(prevChar)) {
break;
}
// LB 19
// x QU
// QU x
if (fQU->contains(thisChar) || fQU->contains(prevChar)) {
continue;
}
// LB 20 Break around a CB
if (fCB->contains(thisChar) || fCB->contains(prevChar)) {
break;
}
// LB 21
if (fBA->contains(thisChar) ||
fHY->contains(thisChar) ||
fNS->contains(thisChar) ||
fBB->contains(prevChar) ) {
continue;
}
// LB 21a
// HL (HY | BA) x
if (fHL->contains(prevCharX2) &&
(fHY->contains(prevChar) || fBA->contains(prevChar))) {
continue;
}
// LB 21b
// SY x HL
if (fSY->contains(prevChar) && fHL->contains(thisChar)) {
continue;
}
// LB 22
if ((fAL->contains(prevChar) && fIN->contains(thisChar)) ||
(fEX->contains(prevChar) && fIN->contains(thisChar)) ||
(fHL->contains(prevChar) && fIN->contains(thisChar)) ||
((fID->contains(prevChar) || fEB->contains(prevChar) || fEM->contains(prevChar)) && fIN->contains(thisChar)) ||
(fIN->contains(prevChar) && fIN->contains(thisChar)) ||
(fNU->contains(prevChar) && fIN->contains(thisChar)) ) {
continue;
}
// LB 23 (AL | HL) x NU
// NU x (AL | HL)
if ((fAL->contains(prevChar) || fHL->contains(prevChar)) && fNU->contains(thisChar)) {
continue;
}
if (fNU->contains(prevChar) && (fAL->contains(thisChar) || fHL->contains(thisChar))) {
continue;
}
// LB 23a Do not break between numeric prefixes and ideographs, or between ideographs and numeric postfixes.
// PR x (ID | EB | EM)
// (ID | EB | EM) x PO
if (fPR->contains(prevChar) &&
(fID->contains(thisChar) || fEB->contains(thisChar) || fEM->contains(thisChar))) {
continue;
}
if ((fID->contains(prevChar) || fEB->contains(prevChar) || fEM->contains(prevChar)) &&
fPO->contains(thisChar)) {
continue;
}
// LB 24 Do not break between prefix and letters or ideographs.
// (PR | PO) x (AL | HL)
// (AL | HL) x (PR | PO)
if ((fPR->contains(prevChar) || fPO->contains(prevChar)) &&
(fAL->contains(thisChar) || fHL->contains(thisChar))) {
continue;
}
if ((fAL->contains(prevChar) || fHL->contains(prevChar)) &&
(fPR->contains(thisChar) || fPO->contains(thisChar))) {
continue;
}
// LB 25 numbers match, moved up, before LB 8a,
// LB 26 Do not break a Korean syllable.
if (fJL->contains(prevChar) && (fJL->contains(thisChar) ||
fJV->contains(thisChar) ||
fH2->contains(thisChar) ||
fH3->contains(thisChar))) {
continue;
}
if ((fJV->contains(prevChar) || fH2->contains(prevChar)) &&
(fJV->contains(thisChar) || fJT->contains(thisChar))) {
continue;
}
if ((fJT->contains(prevChar) || fH3->contains(prevChar)) &&
fJT->contains(thisChar)) {
continue;
}
// LB 27 Treat a Korean Syllable Block the same as ID.
if ((fJL->contains(prevChar) || fJV->contains(prevChar) ||
fJT->contains(prevChar) || fH2->contains(prevChar) || fH3->contains(prevChar)) &&
fIN->contains(thisChar)) {
continue;
}
if ((fJL->contains(prevChar) || fJV->contains(prevChar) ||
fJT->contains(prevChar) || fH2->contains(prevChar) || fH3->contains(prevChar)) &&
fPO->contains(thisChar)) {
continue;
}
if (fPR->contains(prevChar) && (fJL->contains(thisChar) || fJV->contains(thisChar) ||
fJT->contains(thisChar) || fH2->contains(thisChar) || fH3->contains(thisChar))) {
continue;
}
// LB 28 Do not break between alphabetics ("at").
if ((fAL->contains(prevChar) || fHL->contains(prevChar)) && (fAL->contains(thisChar) || fHL->contains(thisChar))) {
continue;
}
// LB 29 Do not break between numeric punctuation and alphabetics ("e.g.").
if (fIS->contains(prevChar) && (fAL->contains(thisChar) || fHL->contains(thisChar))) {
continue;
}
// LB 30 Do not break between letters, numbers, or ordinary symbols and opening or closing punctuation.
// (AL | NU) x OP
// CP x (AL | NU)
if ((fAL->contains(prevChar) || fHL->contains(prevChar) || fNU->contains(prevChar)) && fOP->contains(thisChar)) {
continue;
}
if (fCP->contains(prevChar) && (fAL->contains(thisChar) || fHL->contains(thisChar) || fNU->contains(thisChar))) {
continue;
}
// LB30a RI RI <break> RI
// RI x RI
if (fRI->contains(prevCharX2) && fRI->contains(prevChar) && fRI->contains(thisChar)) {
break;
}
if (fRI->contains(prevChar) && fRI->contains(thisChar)) {
continue;
}
// LB30b Emoji Base x Emoji Modifier
if (fEB->contains(prevChar) && fEM->contains(thisChar)) {
continue;
}
// LB 31 Break everywhere else
break;
}
return pos;
}
UVector *RBBILineMonkey::charClasses() {
return fSets;
}
RBBILineMonkey::~RBBILineMonkey() {
delete fSets;
delete fBK;
delete fCR;
delete fLF;
delete fCM;
delete fNL;
delete fWJ;
delete fZW;
delete fGL;
delete fCB;
delete fSP;
delete fB2;
delete fBA;
delete fBB;
delete fHY;
delete fH2;
delete fH3;
delete fCL;
delete fCP;
delete fEX;
delete fIN;
delete fJL;
delete fJV;
delete fJT;
delete fNS;
delete fOP;
delete fQU;
delete fIS;
delete fNU;
delete fPO;
delete fPR;
delete fSY;
delete fAI;
delete fAL;
delete fCJ;
delete fHL;
delete fID;
delete fRI;
delete fSG;
delete fXX;
delete fEB;
delete fEM;
delete fZJ;
delete fCharBI;
delete fNumberMatcher;
}
//-------------------------------------------------------------------------------------------
//
// TestMonkey
//
// params
// seed=nnnnn Random number starting seed.
// Setting the seed allows errors to be reproduced.
// loop=nnn Looping count. Controls running time.
// -1: run forever.
// 0 or greater: run length.
//
// type = char | word | line | sent | title
//
// Example:
// intltest rbbi/RBBITest/TestMonkey@"type=line loop=-1"
//
//-------------------------------------------------------------------------------------------
static int32_t getIntParam(UnicodeString name, UnicodeString &params, int32_t defaultVal) {
int32_t val = defaultVal;
name.append(" *= *(-?\\d+)");
UErrorCode status = U_ZERO_ERROR;
RegexMatcher m(name, params, 0, status);
if (m.find()) {
// The param exists. Convert the string to an int.
char valString[100];
int32_t paramLength = m.end(1, status) - m.start(1, status);
if (paramLength >= (int32_t)(sizeof(valString)-1)) {
paramLength = (int32_t)(sizeof(valString)-2);
}
params.extract(m.start(1, status), paramLength, valString, sizeof(valString));
val = strtol(valString, NULL, 10);
// Delete this parameter from the params string.
m.reset();
params = m.replaceFirst("", status);
}
U_ASSERT(U_SUCCESS(status));
return val;
}
#endif
#if !UCONFIG_NO_REGULAR_EXPRESSIONS
static void testBreakBoundPreceding(RBBITest *test, UnicodeString ustr,
BreakIterator *bi,
int expected[],
int expectedcount)
{
int count = 0;
int i = 0;
int forward[50];
bi->setText(ustr);
for (i = bi->first(); i != BreakIterator::DONE; i = bi->next()) {
forward[count] = i;
if (count < expectedcount && expected[count] != i) {
test->errln("%s:%d break forward test failed: expected %d but got %d",
__FILE__, __LINE__, expected[count], i);
break;
}
count ++;
}
if (count != expectedcount) {
printStringBreaks(ustr, expected, expectedcount);
test->errln("%s:%d break forward test failed: missed %d match",
__FILE__, __LINE__, expectedcount - count);
return;
}
// testing boundaries
for (i = 1; i < expectedcount; i ++) {
int j = expected[i - 1];
if (!bi->isBoundary(j)) {
printStringBreaks(ustr, expected, expectedcount);
test->errln("%s:%d isBoundary() failed. Expected boundary at position %d",
__FILE__, __LINE__, j);
return;
}
for (j = expected[i - 1] + 1; j < expected[i]; j ++) {
if (bi->isBoundary(j)) {
printStringBreaks(ustr, expected, expectedcount);
test->errln("%s:%d isBoundary() failed. Not expecting boundary at position %d",
__FILE__, __LINE__, j);
return;
}
}
}
for (i = bi->last(); i != BreakIterator::DONE; i = bi->previous()) {
count --;
if (forward[count] != i) {
printStringBreaks(ustr, expected, expectedcount);
test->errln("%s:%d happy break test previous() failed: expected %d but got %d",
__FILE__, __LINE__, forward[count], i);
break;
}
}
if (count != 0) {
printStringBreaks(ustr, expected, expectedcount);
test->errln("break test previous() failed: missed a match");
return;
}
// testing preceding
for (i = 0; i < expectedcount - 1; i ++) {
// int j = expected[i] + 1;
int j = ustr.moveIndex32(expected[i], 1);
for (; j <= expected[i + 1]; j ++) {
int32_t expectedPreceding = expected[i];
int32_t actualPreceding = bi->preceding(j);
if (actualPreceding != expectedPreceding) {
printStringBreaks(ustr, expected, expectedcount);
test->errln("%s:%d preceding(%d): expected %d, got %d",
__FILE__, __LINE__, j, expectedPreceding, actualPreceding);
return;
}
}
}
}
#endif
void RBBITest::TestWordBreaks(void)
{
#if !UCONFIG_NO_REGULAR_EXPRESSIONS
Locale locale("en");
UErrorCode status = U_ZERO_ERROR;
// BreakIterator *bi = BreakIterator::createCharacterInstance(locale, status);
BreakIterator *bi = BreakIterator::createWordInstance(locale, status);
// Replaced any C+J characters in a row with a random sequence of characters
// of the same length to make our C+J segmentation not get in the way.
static const char *strlist[] =
{
"\\U000e0032\\u0097\\u0f94\\uc2d8\\u05f4\\U000e0031\\u060d",
"\\U000e0037\\u2666\\u1202\\u003a\\U000e0031\\u064d\\u0bea\\u091c\\U000e0040\\u003b",
"\\u0589\\u3e99\\U0001d7f3\\U000e0074\\u1810\\u200e\\U000e004b\\u0027\\U000e0061\\u003a",
"\\u398c\\U000104a5\\U0001d173\\u102d\\u002e\\uca3b\\u002e\\u002c\\u5622",
"\\uac00\\u3588\\u009c\\u0953\\u194b",
"\\u200e\\U000e0072\\u0a4b\\U000e003f\\ufd2b\\u2027\\u002e\\u002e",
"\\u0602\\u2019\\ua191\\U000e0063\\u0a4c\\u003a\\ub4b5\\u003a\\u827f\\u002e",
"\\u2f1f\\u1634\\u05f8\\u0944\\u04f2\\u0cdf\\u1f9c\\u05f4\\u002e",
"\\U000e0042\\u002e\\u0fb8\\u09ef\\u0ed1\\u2044",
"\\u003b\\u024a\\u102e\\U000e0071\\u0600",
"\\u2027\\U000e0067\\u0a47\\u00b7",
"\\u1fcd\\u002c\\u07aa\\u0027\\u11b0",
"\\u002c\\U000e003c\\U0001d7f4\\u003a\\u0c6f\\u0027",
"\\u0589\\U000e006e\\u0a42\\U000104a5",
"\\u0f66\\u2523\\u003a\\u0cae\\U000e0047\\u003a",
"\\u003a\\u0f21\\u0668\\u0dab\\u003a\\u0655\\u00b7",
"\\u0027\\u11af\\U000e0057\\u0602",
"\\U0001d7f2\\U000e007\\u0004\\u0589",
"\\U000e0022\\u003a\\u10b3\\u003a\\ua21b\\u002e\\U000e0058\\u1732\\U000e002b",
"\\U0001d7f2\\U000e007d\\u0004\\u0589",
"\\u82ab\\u17e8\\u0736\\u2019\\U0001d64d",
"\\ub55c\\u0a68\\U000e0037\\u0cd6\\u002c\\ub959",
"\\U000e0065\\u302c\\uc986\\u09ee\\U000e0068",
"\\u0be8\\u002e\\u0c68\\u066e\\u136d\\ufc99\\u59e7",
"\\u0233\\U000e0020\\u0a69\\u0d6a",
"\\u206f\\u0741\\ub3ab\\u2019\\ubcac\\u2019",
"\\u18f4\\U000e0049\\u20e7\\u2027",
"\\ub315\\U0001d7e5\\U000e0073\\u0c47\\u06f2\\u0c6a\\u0037\\u10fe",
"\\ua183\\u102d\\u0bec\\u003a",
"\\u17e8\\u06e7\\u002e\\u096d\\u003b",
"\\u003a\\u0e57\\u0fad\\u002e",
"\\u002e\\U000e004c\\U0001d7ea\\u05bb\\ud0fd\\u02de",
"\\u32e6\\U0001d7f6\\u0fa1\\u206a\\U000e003c\\u0cec\\u003a",
"\\U000e005d\\u2044\\u0731\\u0650\\u0061",
"\\u003a\\u0664\\u00b7\\u1fba",
"\\u003b\\u0027\\u00b7\\u47a3",
"\\u2027\\U000e0067\\u0a42\\u00b7\\u4edf\\uc26c\\u003a\\u4186\\u041b",
"\\u0027\\u003a\\U0001d70f\\U0001d7df\\ubf4a\\U0001d7f5\\U0001d177\\u003a\\u0e51\\u1058\\U000e0058\\u00b7\\u0673",
"\\uc30d\\u002e\\U000e002c\\u0c48\\u003a\\ub5a1\\u0661\\u002c",
};
int loop;
if (U_FAILURE(status)) {
errcheckln(status, "Creation of break iterator failed %s", u_errorName(status));
return;
}
for (loop = 0; loop < UPRV_LENGTHOF(strlist); loop ++) {
// printf("looping %d\n", loop);
UnicodeString ustr = CharsToUnicodeString(strlist[loop]);
// RBBICharMonkey monkey;
RBBIWordMonkey monkey;
int expected[50];
int expectedcount = 0;
monkey.setText(ustr);
int i;
for (i = 0; i != BreakIterator::DONE; i = monkey.next(i)) {
expected[expectedcount ++] = i;
}
testBreakBoundPreceding(this, ustr, bi, expected, expectedcount);
}
delete bi;
#endif
}
void RBBITest::TestWordBoundary(void)
{
// <data><>\u1d4a\u206e<?>\u0603\U0001d7ff<>\u2019<></data>
Locale locale("en");
UErrorCode status = U_ZERO_ERROR;
// BreakIterator *bi = BreakIterator::createCharacterInstance(locale, status);
LocalPointer<BreakIterator> bi(BreakIterator::createWordInstance(locale, status), status);
if (U_FAILURE(status)) {
errcheckln(status, "%s:%d Creation of break iterator failed %s",
__FILE__, __LINE__, u_errorName(status));
return;
}
UChar str[50];
static const char *strlist[] =
{
"\\u200e\\U000e0072\\u0a4b\\U000e003f\\ufd2b\\u2027\\u002e\\u002e",
"\\U000e0042\\u002e\\u0fb8\\u09ef\\u0ed1\\u2044",
"\\u003b\\u024a\\u102e\\U000e0071\\u0600",
"\\u2027\\U000e0067\\u0a47\\u00b7",
"\\u1fcd\\u002c\\u07aa\\u0027\\u11b0",
"\\u002c\\U000e003c\\U0001d7f4\\u003a\\u0c6f\\u0027",
"\\u0589\\U000e006e\\u0a42\\U000104a5",
"\\u4f66\\ub523\\u003a\\uacae\\U000e0047\\u003a",
"\\u003a\\u0f21\\u0668\\u0dab\\u003a\\u0655\\u00b7",
"\\u0027\\u11af\\U000e0057\\u0602",
"\\U0001d7f2\\U000e007\\u0004\\u0589",
"\\U000e0022\\u003a\\u10b3\\u003a\\ua21b\\u002e\\U000e0058\\u1732\\U000e002b",
"\\U0001d7f2\\U000e007d\\u0004\\u0589",
"\\u82ab\\u17e8\\u0736\\u2019\\U0001d64d",
"\\u0e01\\ub55c\\u0a68\\U000e0037\\u0cd6\\u002c\\ub959",
"\\U000e0065\\u302c\\u09ee\\U000e0068",
"\\u0be8\\u002e\\u0c68\\u066e\\u136d\\ufc99\\u59e7",
"\\u0233\\U000e0020\\u0a69\\u0d6a",
"\\u206f\\u0741\\ub3ab\\u2019\\ubcac\\u2019",
"\\u58f4\\U000e0049\\u20e7\\u2027",
"\\U0001d7e5\\U000e0073\\u0c47\\u06f2\\u0c6a\\u0037\\u10fe",
"\\ua183\\u102d\\u0bec\\u003a",
"\\u17e8\\u06e7\\u002e\\u096d\\u003b",
"\\u003a\\u0e57\\u0fad\\u002e",
"\\u002e\\U000e004c\\U0001d7ea\\u05bb\\ud0fd\\u02de",
"\\u32e6\\U0001d7f6\\u0fa1\\u206a\\U000e003c\\u0cec\\u003a",
"\\ua2a5\\u0038\\u2044\\u002e\\u0c67\\U000e003c\\u05f4\\u2027\\u05f4\\u2019",
"\\u003a\\u0664\\u00b7\\u1fba",
"\\u003b\\u0027\\u00b7\\u47a3",
};
int loop;
for (loop = 0; loop < UPRV_LENGTHOF(strlist); loop ++) {
u_unescape(strlist[loop], str, UPRV_LENGTHOF(str));
UnicodeString ustr(str);
int forward[50];
int count = 0;
bi->setText(ustr);
int prev = -1;
for (int32_t boundary = bi->first(); boundary != BreakIterator::DONE; boundary = bi->next()) {
++count;
if (count >= UPRV_LENGTHOF(forward)) {
errln("%s:%d too many breaks found. (loop, count, boundary) = (%d, %d, %d)",
__FILE__, __LINE__, loop, count, boundary);
return;
}
forward[count] = boundary;
if (boundary <= prev) {
errln("%s:%d bi::next() did not advance. (loop, prev, boundary) = (%d, %d, %d)\n",
__FILE__, __LINE__, loop, prev, boundary);
break;
}
for (int32_t nonBoundary = prev + 1; nonBoundary < boundary; nonBoundary ++) {
if (bi->isBoundary(nonBoundary)) {
printStringBreaks(ustr, forward, count);
errln("%s:%d isBoundary(nonBoundary) failed. (loop, prev, nonBoundary, boundary) = (%d, %d, %d, %d)",
__FILE__, __LINE__, loop, prev, nonBoundary, boundary);
return;
}
}
if (!bi->isBoundary(boundary)) {
printStringBreaks(ustr, forward, count);
errln("%s:%d happy boundary test failed: expected %d a boundary",
__FILE__, __LINE__, boundary);
return;
}
prev = boundary;
}
}
}
void RBBITest::TestLineBreaks(void)
{
#if !UCONFIG_NO_REGULAR_EXPRESSIONS
Locale locale("en");
UErrorCode status = U_ZERO_ERROR;
BreakIterator *bi = BreakIterator::createLineInstance(locale, status);
const int32_t STRSIZE = 50;
UChar str[STRSIZE];
static const char *strlist[] =
{
"\\u300f\\ufdfc\\ub798\\u2011\\u2011\\u0020\\u0b43\\u002d\\ubeec\\ufffc",
"\\u24ba\\u2060\\u3405\\ub290\\u000d\\U000e0032\\ufe35\\u00a0\\u0361\\"
"U000112ed\\u0f0c\\u000a\\u308e\\ua875\\u0085\\u114d",
"\\ufffc\\u3063\\u2e08\\u30e3\\u000d\\u002d\\u0ed8\\u002f\\U00011a57\\"
"u2014\\U000e0105\\u118c\\u000a\\u07f8",
"\\u0668\\u192b\\u002f\\u2034\\ufe39\\u00b4\\u0cc8\\u2571\\u200b\\u003f",
"\\ufeff\\ufffc\\u3289\\u0085\\u2772\\u0020\\U000e010a\\u0020\\u2025\\u000a\\U000e0123",
"\\ufe3c\\u201c\\u000d\\u2025\\u2007\\u201c\\u002d\\u20a0\\u002d\\u30a7\\u17a4",
"\\u2772\\u0020\\U000e010a\\u0020\\u2025\\u000a\\U000e0123",
"\\u002d\\uff1b\\u02c8\\u2029\\ufeff\\u0f22\\u2044\\ufe09\\u003a\\u096d\\u2009\\u000a\\u06f7\\u02cc\\u1019\\u2060",
"\\u2770\\u0020\\U000e010f\\u0020\\u2060\\u000a\\u02cc\\u0bcc\\u060d\\u30e7\\u0f3b\\u002f",
"\\ufeff\\u0028\\u003b\\U00012fec\\u2010\\u0020\\u0004\\u200b\\u0020\\u275c\\u002f\\u17b1",
"\\u20a9\\u2014\\u00a2\\u31f1\\u002f\\u0020\\u05b8\\u200b\\u0cc2\\u003b\\u060d\\u02c8\\ua4e8\\u002f\\u17d5",
"\\u002d\\u136f\\uff63\\u0084\\ua933\\u2028\\u002d\\u431b\\u200b\\u20b0",
"\\uade3\\u11d6\\u000a\\U0001107d\\u203a\\u201d\\ub070\\u000d\\u2024\\ufffc",
"\\uff5b\\u101c\\u1806\\u002f\\u2213\\uff5f",
"\\u2014\\u0a83\\ufdfc\\u003f\\u00a0\\u0020\\u000a\\u2991\\U0001d179\\u0020\\u201d\\U000125f6\\u0a67\\u20a7\\ufeff\\u043f",
"\\u169b\\U000e0130\\u002d\\u1041\\u0f3d\\u0abf\\u00b0\\u31fb\\u00a0\\u002d\\u02c8\\u003b",
"\\u2762\\u1680\\u002d\\u2028\\u0027\\u01dc\\ufe56\\u003a\\u000a\\uffe6\\u29fd\\u0020\\u30ee\\u007c\\U0001d178\\u0af1\\u0085",
"\\u3010\\u200b\\u2029\\ufeff\\ufe6a\\u275b\\U000e013b\\ufe37\\u24d4\\u002d\\u1806\\u256a\\u1806\\u247c\\u0085\\u17ac",
"\\u99ab\\u0027\\u003b\\u2026\\ueaf0\\u0020\\u0020\\u0313\\u0020\\u3099\\uff09\\u208e\\u2011\\u2007\\u2060\\u000a\\u0020\\u0020\\u300b\\u0bf9",
"\\u1806\\u060d\\u30f5\\u00b4\\u17e9\\u2544\\u2028\\u2024\\u2011\\u20a3\\u002d\\u09cc\\u1782\\u000d\\uff6f\\u0025",
"\\u002f\\uf22e\\u1944\\ufe3d\\u0020\\u206f\\u31b3\\u2014\\u002d\\u2025\\u0f0c\\u0085\\u2763",
"\\u002f\\u2563\\u202f\\u0085\\u17d5\\u200b\\u0020\\U000e0043\\u2014\\u058a\\u3d0a\\ufe57\\u2035\\u2028\\u2029",
"\\u20ae\\U0001d169\\u9293\\uff1f\\uff1f\\u0021\\u2012\\u2039\\u0085\\u02cc\\u00a2\\u0020\\U000e01ab\\u3085\\u0f3a\\u1806\\u0f0c\\u1945\\u000a\\U0001d7e7",
"\\u02cc\\ufe6a\\u00a0\\u0021\\u002d\\u7490\\uec2e\\u200b\\u000a",
"\\uec2e\\u200b\\u000a\\u0020\\u2028\\u2014\\u8945",
"\\u7490\\uec2e\\u200b\\u000a\\u0020\\u2028\\u2014",
"\\u0020\\u2028\\u2014\\u8945\\u002c\\u005b",
"\\u000a\\ufe3c\\u201c\\u000d\\u2025\\u2007\\u201c\\u002d\\u20a0",
"\\U0001d16e\\ufffc\\u2025\\u0021\\u002d",
"\\ufffc\\u301b\\u0fa5\\U000e0103\\u2060\\u208e\\u17d5\\u034f\\u1009\\u003a\\u180e\\u2009\\u3111",
"\\ufffc\\u0020\\u2116\\uff6c\\u200b\\u0ac3\\U0001028f",
"\\uaeb0\\u0344\\u0085\\ufffc\\u073b\\u2010",
"\\ufeff\\u0589\\u0085\\u0eb8\\u30fd\\u002f\\u003a\\u2014\\ufe43",
"\\u09cc\\u256a\\u276d\\u002d\\u3085\\u000d\\u0e05\\u2028\\u0fbb",
"\\u2034\\u00bb\\u0ae6\\u300c\\u0020\\u31f8\\ufffc",
"\\u2116\\u0ed2\\uff64\\u02cd\\u2001\\u2060",
"\\ufe10\\u2060\\u1a5a\\u2060\\u17e4\\ufffc\\ubbe1\\ufe15\\u0020\\u00a0",
"\\u2060\\u2213\\u200b\\u2019\\uc2dc\\uff6a\\u1736\\u0085\\udb07",
};
int loop;
TEST_ASSERT_SUCCESS(status);
if (U_FAILURE(status)) {
return;
}
for (loop = 0; loop < UPRV_LENGTHOF(strlist); loop ++) {
// printf("looping %d\n", loop);
int32_t t = u_unescape(strlist[loop], str, STRSIZE);
if (t >= STRSIZE) {
TEST_ASSERT(FALSE);
continue;
}
UnicodeString ustr(str);
RBBILineMonkey monkey;
if (U_FAILURE(monkey.deferredStatus)) {
continue;
}
const int EXPECTEDSIZE = 50;
int expected[EXPECTEDSIZE];
int expectedcount = 0;
monkey.setText(ustr);
int i;
for (i = 0; i != BreakIterator::DONE; i = monkey.next(i)) {
if (expectedcount >= EXPECTEDSIZE) {
TEST_ASSERT(expectedcount < EXPECTEDSIZE);
return;
}
expected[expectedcount ++] = i;
}
testBreakBoundPreceding(this, ustr, bi, expected, expectedcount);
}
delete bi;
#endif
}
void RBBITest::TestSentBreaks(void)
{
#if !UCONFIG_NO_REGULAR_EXPRESSIONS
Locale locale("en");
UErrorCode status = U_ZERO_ERROR;
BreakIterator *bi = BreakIterator::createSentenceInstance(locale, status);
UChar str[200];
static const char *strlist[] =
{
"Now\ris\nthe\r\ntime\n\rfor\r\r",
"This\n",
"Hello! how are you? I'am fine. Thankyou. How are you doing? This\n costs $20,00,000.",
"\"Sentence ending with a quote.\" Bye.",
" (This is it). Testing the sentence iterator. \"This isn't it.\"",
"Hi! This is a simple sample sentence. (This is it.) This is a simple sample sentence. \"This isn't it.\"",
"Hi! This is a simple sample sentence. It does not have to make any sense as you can see. ",
"Nel mezzo del cammin di nostra vita, mi ritrovai in una selva oscura. ",
"Che la dritta via aveo smarrita. He said, that I said, that you said!! ",
"Don't rock the boat.\\u2029Because I am the daddy, that is why. Not on my time (el timo.)!",
"\\U0001040a\\u203a\\u1217\\u2b23\\u000d\\uff3b\\u03dd\\uff57\\u0a69\\u104a\\ufe56\\ufe52"
"\\u3016\\U000e002f\\U000e0077\\u0662\\u1680\\u2984\\U000e006a\\u002e\\ua6ab\\u104a"
"\\u002e\\u019b\\u2005\\u002e\\u0477\\u0438\\u0085\\u0441\\u002e\\u5f61\\u202f"
"\\U0001019f\\uff08\\u27e8\\u055c\\u0352",
"\\u1f3e\\u004d\\u000a\\ua3e4\\U000e0023\\uff63\\u0c52\\u276d\\U0001d5de\\U0001d171"
"\\u0e38\\u17e5\\U00012fe6\\u0fa9\\u267f\\u1da3\\u0046\\u03ed\\udc72\\u0030"
"\\U0001d688\\u0b6d\\u0085\\u0c67\\u1f94\\u0c6c\\u9cb2\\u202a\\u180e\\u000b"
"\\u002e\\U000e005e\\u035b\\u061f\\u02c1\\U000e0025\\u0357\\u0969\\u202b"
"\\U000130c5\\u0486\\U000e0123\\u2019\\u01bc\\u2006\\u11ad\\u180e\\u2e05"
"\\u10b7\\u013e\\u000a\\u002e\\U00013ea4"
};
int loop;
if (U_FAILURE(status)) {
errcheckln(status, "Creation of break iterator failed %s", u_errorName(status));
return;
}
for (loop = 0; loop < UPRV_LENGTHOF(strlist); loop ++) {
u_unescape(strlist[loop], str, UPRV_LENGTHOF(str));
UnicodeString ustr(str);
RBBISentMonkey monkey;
if (U_FAILURE(monkey.deferredStatus)) {
continue;
}
const int EXPECTEDSIZE = 50;
int expected[EXPECTEDSIZE];
int expectedcount = 0;
monkey.setText(ustr);
int i;
for (i = 0; i != BreakIterator::DONE; i = monkey.next(i)) {
if (expectedcount >= EXPECTEDSIZE) {
TEST_ASSERT(expectedcount < EXPECTEDSIZE);
return;
}
expected[expectedcount ++] = i;
}
testBreakBoundPreceding(this, ustr, bi, expected, expectedcount);
}
delete bi;
#endif
}
void RBBITest::TestMonkey() {
#if !UCONFIG_NO_REGULAR_EXPRESSIONS
UErrorCode status = U_ZERO_ERROR;
int32_t loopCount = 500;
int32_t seed = 1;
UnicodeString breakType = "all";
Locale locale("en");
UBool useUText = FALSE;
if (quick == FALSE) {
loopCount = 10000;
}
if (fTestParams) {
UnicodeString p(fTestParams);
loopCount = getIntParam("loop", p, loopCount);
seed = getIntParam("seed", p, seed);
RegexMatcher m(" *type *= *(char|word|line|sent|title) *", p, 0, status);
if (m.find()) {
breakType = m.group(1, status);
m.reset();
p = m.replaceFirst("", status);
}
RegexMatcher u(" *utext", p, 0, status);
if (u.find()) {
useUText = TRUE;
u.reset();
p = u.replaceFirst("", status);
}
// m.reset(p);
if (RegexMatcher(UNICODE_STRING_SIMPLE("\\S"), p, 0, status).find()) {
// Each option is stripped out of the option string as it is processed.
// All options have been checked. The option string should have been completely emptied..
char buf[100];
p.extract(buf, sizeof(buf), NULL, status);
buf[sizeof(buf)-1] = 0;
errln("Unrecognized or extra parameter: %s\n", buf);
return;
}
}
if (breakType == "char" || breakType == "all") {
RBBICharMonkey m;
BreakIterator *bi = BreakIterator::createCharacterInstance(locale, status);
if (U_SUCCESS(status)) {
RunMonkey(bi, m, "char", seed, loopCount, useUText);
if (breakType == "all" && useUText==FALSE) {
// Also run a quick test with UText when "all" is specified
RunMonkey(bi, m, "char", seed, loopCount, TRUE);
}
}
else {
errcheckln(status, "Creation of character break iterator failed %s", u_errorName(status));
}
delete bi;
}
if (breakType == "word" || breakType == "all") {
logln("Word Break Monkey Test");
RBBIWordMonkey m;
BreakIterator *bi = BreakIterator::createWordInstance(locale, status);
if (U_SUCCESS(status)) {
RunMonkey(bi, m, "word", seed, loopCount, useUText);
}
else {
errcheckln(status, "Creation of word break iterator failed %s", u_errorName(status));
}
delete bi;
}
if (breakType == "line" || breakType == "all") {
logln("Line Break Monkey Test");
RBBILineMonkey m;
BreakIterator *bi = BreakIterator::createLineInstance(locale, status);
if (loopCount >= 10) {
loopCount = loopCount / 5; // Line break runs slower than the others.
}
if (U_SUCCESS(status)) {
RunMonkey(bi, m, "line", seed, loopCount, useUText);
}
else {
errcheckln(status, "Creation of line break iterator failed %s", u_errorName(status));
}
delete bi;
}
if (breakType == "sent" || breakType == "all" ) {
logln("Sentence Break Monkey Test");
RBBISentMonkey m;
BreakIterator *bi = BreakIterator::createSentenceInstance(locale, status);
if (loopCount >= 10) {
loopCount = loopCount / 10; // Sentence runs slower than the other break types
}
if (U_SUCCESS(status)) {
RunMonkey(bi, m, "sentence", seed, loopCount, useUText);
}
else {
errcheckln(status, "Creation of line break iterator failed %s", u_errorName(status));
}
delete bi;
}
#endif
}
//
// Run a RBBI monkey test. Common routine, for all break iterator types.
// Parameters:
// bi - the break iterator to use
// mk - MonkeyKind, abstraction for obtaining expected results
// name - Name of test (char, word, etc.) for use in error messages
// seed - Seed for starting random number generator (parameter from user)
// numIterations
//
void RBBITest::RunMonkey(BreakIterator *bi, RBBIMonkeyKind &mk, const char *name, uint32_t seed,
int32_t numIterations, UBool useUText) {
#if !UCONFIG_NO_REGULAR_EXPRESSIONS
const int32_t TESTSTRINGLEN = 500;
UnicodeString testText;
int32_t numCharClasses;
UVector *chClasses;
int expected[TESTSTRINGLEN*2 + 1];
int expectedCount = 0;
char expectedBreaks[TESTSTRINGLEN*2 + 1];
char forwardBreaks[TESTSTRINGLEN*2 + 1];
char reverseBreaks[TESTSTRINGLEN*2+1];
char isBoundaryBreaks[TESTSTRINGLEN*2+1];
char followingBreaks[TESTSTRINGLEN*2+1];
char precedingBreaks[TESTSTRINGLEN*2+1];
int i;
int loopCount = 0;
m_seed = seed;
numCharClasses = mk.charClasses()->size();
chClasses = mk.charClasses();
// Check for errors that occured during the construction of the MonkeyKind object.
// Can't report them where they occured because errln() is a method coming from intlTest,
// and is not visible outside of RBBITest :-(
if (U_FAILURE(mk.deferredStatus)) {
errln("status of \"%s\" in creation of RBBIMonkeyKind.", u_errorName(mk.deferredStatus));
return;
}
// Verify that the character classes all have at least one member.
for (i=0; i<numCharClasses; i++) {
UnicodeSet *s = (UnicodeSet *)chClasses->elementAt(i);
if (s == NULL || s->size() == 0) {
errln("Character Class #%d is null or of zero size.", i);
return;
}
}
while (loopCount < numIterations || numIterations == -1) {
if (numIterations == -1 && loopCount % 10 == 0) {
// If test is running in an infinite loop, display a periodic tic so
// we can tell that it is making progress.
fprintf(stderr, ".");
}
// Save current random number seed, so that we can recreate the random numbers
// for this loop iteration in event of an error.
seed = m_seed;
// Populate a test string with data.
testText.truncate(0);
for (i=0; i<TESTSTRINGLEN; i++) {
int32_t aClassNum = m_rand() % numCharClasses;
UnicodeSet *classSet = (UnicodeSet *)chClasses->elementAt(aClassNum);
int32_t charIdx = m_rand() % classSet->size();
UChar32 c = classSet->charAt(charIdx);
if (c < 0) { // TODO: deal with sets containing strings.
errln("%s:%d c < 0", __FILE__, __LINE__);
break;
}
// Do not assemble a supplementary character from randomly generated separate surrogates.
// (It could be a dictionary character)
if (U16_IS_TRAIL(c) && testText.length() > 0 && U16_IS_LEAD(testText.charAt(testText.length()-1))) {
continue;
}
testText.append(c);
}
// Calculate the expected results for this test string.
mk.setText(testText);
memset(expectedBreaks, 0, sizeof(expectedBreaks));
expectedBreaks[0] = 1;
int32_t breakPos = 0;
expectedCount = 0;
for (;;) {
breakPos = mk.next(breakPos);
if (breakPos == -1) {
break;
}
if (breakPos > testText.length()) {
errln("breakPos > testText.length()");
}
expectedBreaks[breakPos] = 1;
U_ASSERT(expectedCount<testText.length());
expected[expectedCount ++] = breakPos;
(void)expected; // Set but not used warning.
// TODO (andy): check it out.
}
// Find the break positions using forward iteration
memset(forwardBreaks, 0, sizeof(forwardBreaks));
if (useUText) {
UErrorCode status = U_ZERO_ERROR;
UText *testUText = utext_openReplaceable(NULL, &testText, &status);
// testUText = utext_openUnicodeString(testUText, &testText, &status);
bi->setText(testUText, status);
TEST_ASSERT_SUCCESS(status);
utext_close(testUText); // The break iterator does a shallow clone of the UText
// This UText can be closed immediately, so long as the
// testText string continues to exist.
} else {
bi->setText(testText);
}
for (i=bi->first(); i != BreakIterator::DONE; i=bi->next()) {
if (i < 0 || i > testText.length()) {
errln("%s break monkey test: Out of range value returned by breakIterator::next()", name);
break;
}
forwardBreaks[i] = 1;
}
// Find the break positions using reverse iteration
memset(reverseBreaks, 0, sizeof(reverseBreaks));
for (i=bi->last(); i != BreakIterator::DONE; i=bi->previous()) {
if (i < 0 || i > testText.length()) {
errln("%s break monkey test: Out of range value returned by breakIterator::next()", name);
break;
}
reverseBreaks[i] = 1;
}
// Find the break positions using isBoundary() tests.
memset(isBoundaryBreaks, 0, sizeof(isBoundaryBreaks));
U_ASSERT((int32_t)sizeof(isBoundaryBreaks) > testText.length());
for (i=0; i<=testText.length(); i++) {
isBoundaryBreaks[i] = bi->isBoundary(i);
}
// Find the break positions using the following() function.
// printf(".");
memset(followingBreaks, 0, sizeof(followingBreaks));
int32_t lastBreakPos = 0;
followingBreaks[0] = 1;
for (i=0; i<testText.length(); i++) {
breakPos = bi->following(i);
if (breakPos <= i ||
breakPos < lastBreakPos ||
breakPos > testText.length() ||
(breakPos > lastBreakPos && lastBreakPos > i)) {
errln("%s break monkey test: "
"Out of range value returned by BreakIterator::following().\n"
"Random seed=%d index=%d; following returned %d; lastbreak=%d",
name, seed, i, breakPos, lastBreakPos);
break;
}
followingBreaks[breakPos] = 1;
lastBreakPos = breakPos;
}
// Find the break positions using the preceding() function.
memset(precedingBreaks, 0, sizeof(precedingBreaks));
lastBreakPos = testText.length();
precedingBreaks[testText.length()] = 1;
for (i=testText.length(); i>0; i--) {
breakPos = bi->preceding(i);
if (breakPos >= i ||
breakPos > lastBreakPos ||
(breakPos < 0 && testText.getChar32Start(i)>0) ||
(breakPos < lastBreakPos && lastBreakPos < testText.getChar32Start(i)) ) {
errln("%s break monkey test: "
"Out of range value returned by BreakIterator::preceding().\n"
"index=%d; prev returned %d; lastBreak=%d" ,
name, i, breakPos, lastBreakPos);
if (breakPos >= 0 && breakPos < (int32_t)sizeof(precedingBreaks)) {
precedingBreaks[i] = 2; // Forces an error.
}
} else {
if (breakPos >= 0) {
precedingBreaks[breakPos] = 1;
}
lastBreakPos = breakPos;
}
}
// Compare the expected and actual results.
for (i=0; i<=testText.length(); i++) {
const char *errorType = NULL;
if (forwardBreaks[i] != expectedBreaks[i]) {
errorType = "next()";
} else if (reverseBreaks[i] != forwardBreaks[i]) {
errorType = "previous()";
} else if (isBoundaryBreaks[i] != expectedBreaks[i]) {
errorType = "isBoundary()";
} else if (followingBreaks[i] != expectedBreaks[i]) {
errorType = "following()";
} else if (precedingBreaks[i] != expectedBreaks[i]) {
errorType = "preceding()";
}
if (errorType != NULL) {
// Format a range of the test text that includes the failure as
// a data item that can be included in the rbbi test data file.
// Start of the range is the last point where expected and actual results
// both agreed that there was a break position.
int startContext = i;
int32_t count = 0;
for (;;) {
if (startContext==0) { break; }
startContext --;
if (expectedBreaks[startContext] != 0) {
if (count == 2) break;
count ++;
}
}
// End of range is two expected breaks past the start position.
int endContext = i + 1;
int ci;
for (ci=0; ci<2; ci++) { // Number of items to include in error text.
for (;;) {
if (endContext >= testText.length()) {break;}
if (expectedBreaks[endContext-1] != 0) {
if (count == 0) break;
count --;
}
endContext ++;
}
}
// Format looks like "<data>\\\uabcd\uabcd\\\U0001abcd...</data>"
UnicodeString errorText = "<data>";
/***if (strcmp(errorType, "next()") == 0) {
startContext = 0;
endContext = testText.length();
printStringBreaks(testText, expected, expectedCount);
}***/
for (ci=startContext; ci<endContext;) {
UnicodeString hexChars("0123456789abcdef");
UChar32 c;
int bn;
c = testText.char32At(ci);
if (ci == i) {
// This is the location of the error.
errorText.append("<?>");
} else if (expectedBreaks[ci] != 0) {
// This a non-error expected break position.
errorText.append("\\");
}
if (c < 0x10000) {
errorText.append("\\u");
for (bn=12; bn>=0; bn-=4) {
errorText.append(hexChars.charAt((c>>bn)&0xf));
}
} else {
errorText.append("\\U");
for (bn=28; bn>=0; bn-=4) {
errorText.append(hexChars.charAt((c>>bn)&0xf));
}
}
ci = testText.moveIndex32(ci, 1);
}
errorText.append("\\");
errorText.append("</data>\n");
// Output the error
char charErrorTxt[500];
UErrorCode status = U_ZERO_ERROR;
errorText.extract(charErrorTxt, sizeof(charErrorTxt), NULL, status);
charErrorTxt[sizeof(charErrorTxt)-1] = 0;
const char *badLocale = bi->getLocaleID(ULOC_ACTUAL_LOCALE, status);
errln("%s break monkey test error [%s]. %s. Operation = %s; Random seed = %d; buf Idx = %d\n%s",
name, badLocale, (expectedBreaks[i]? "break expected but not found" : "break found but not expected"),
errorType, seed, i, charErrorTxt);
break;
}
}
loopCount++;
}
#endif
}
// Bug 5532. UTF-8 based UText fails in dictionary code.
// This test checks the initial patch,
// which is to just keep it from crashing. Correct word boundaries
// await a proper fix to the dictionary code.
//
void RBBITest::TestBug5532(void) {
// Text includes a mixture of Thai and Latin.
const unsigned char utf8Data[] = {
0xE0u, 0xB8u, 0x82u, 0xE0u, 0xB8u, 0xB2u, 0xE0u, 0xB8u, 0xA2u, 0xE0u,
0xB9u, 0x80u, 0xE0u, 0xB8u, 0x84u, 0xE0u, 0xB8u, 0xA3u, 0xE0u, 0xB8u,
0xB7u, 0xE0u, 0xB9u, 0x88u, 0xE0u, 0xB8u, 0xADu, 0xE0u, 0xB8u, 0x87u,
0xE0u, 0xB9u, 0x80u, 0xE0u, 0xB8u, 0xA5u, 0xE0u, 0xB9u, 0x88u, 0xE0u,
0xB8u, 0x99u, 0xE0u, 0xB8u, 0x8Bu, 0xE0u, 0xB8u, 0xB5u, 0xE0u, 0xB8u,
0x94u, 0xE0u, 0xB8u, 0xB5u, 0x20u, 0x73u, 0x69u, 0x6Du, 0x20u, 0x61u,
0x75u, 0x64u, 0x69u, 0x6Fu, 0x2Fu, 0x20u, 0x4Du, 0x4Fu, 0x4Fu, 0x4Eu,
0x20u, 0x65u, 0x63u, 0x6Cu, 0x69u, 0x70u, 0x73u, 0x65u, 0x20u, 0xE0u,
0xB8u, 0xA3u, 0xE0u, 0xB8u, 0xB2u, 0xE0u, 0xB8u, 0x84u, 0xE0u, 0xB8u,
0xB2u, 0x20u, 0x34u, 0x37u, 0x30u, 0x30u, 0x20u, 0xE0u, 0xB8u, 0xA2u,
0xE0u, 0xB8u, 0xB9u, 0xE0u, 0xB9u, 0x82u, 0xE0u, 0xB8u, 0xA3u, 0x00};
UErrorCode status = U_ZERO_ERROR;
UText utext=UTEXT_INITIALIZER;
utext_openUTF8(&utext, (const char *)utf8Data, -1, &status);
TEST_ASSERT_SUCCESS(status);
BreakIterator *bi = BreakIterator::createWordInstance(Locale("th"), status);
TEST_ASSERT_SUCCESS(status);
if (U_SUCCESS(status)) {
bi->setText(&utext, status);
TEST_ASSERT_SUCCESS(status);
int32_t breakCount = 0;
int32_t previousBreak = -1;
for (bi->first(); bi->next() != BreakIterator::DONE; breakCount++) {
// For now, just make sure that the break iterator doesn't hang.
TEST_ASSERT(previousBreak < bi->current());
previousBreak = bi->current();
}
TEST_ASSERT(breakCount > 0);
}
delete bi;
utext_close(&utext);
}
void RBBITest::TestBug9983(void) {
UnicodeString text = UnicodeString("\\u002A" // * Other
"\\uFF65" // Other
"\\u309C" // Katakana
"\\uFF9F" // Extend
"\\uFF65" // Other
"\\u0020" // Other
"\\u0000").unescape();
UErrorCode status = U_ZERO_ERROR;
LocalPointer<RuleBasedBreakIterator> brkiter(static_cast<RuleBasedBreakIterator *>(
BreakIterator::createWordInstance(Locale::getRoot(), status)));
TEST_ASSERT_SUCCESS(status);
LocalPointer<RuleBasedBreakIterator> brkiterPOSIX(static_cast<RuleBasedBreakIterator *>(
BreakIterator::createWordInstance(Locale::createFromName("en_US_POSIX"), status)));
TEST_ASSERT_SUCCESS(status);
if (U_FAILURE(status)) {
return;
}
int32_t offset, rstatus, iterationCount;
brkiter->setText(text);
brkiter->last();
iterationCount = 0;
while ( (offset = brkiter->previous()) != UBRK_DONE ) {
iterationCount++;
rstatus = brkiter->getRuleStatus();
(void)rstatus; // Suppress set but not used warning.
if (iterationCount >= 10) {
break;
}
}
TEST_ASSERT(iterationCount == 6);
brkiterPOSIX->setText(text);
brkiterPOSIX->last();
iterationCount = 0;
while ( (offset = brkiterPOSIX->previous()) != UBRK_DONE ) {
iterationCount++;
rstatus = brkiterPOSIX->getRuleStatus();
(void)rstatus; // Suppress set but not used warning.
if (iterationCount >= 10) {
break;
}
}
TEST_ASSERT(iterationCount == 6);
}
// Bug 7547 - verify that building a break itereator from empty rules produces an error.
//
void RBBITest::TestBug7547() {
UnicodeString rules;
UErrorCode status = U_ZERO_ERROR;
UParseError parseError;
RuleBasedBreakIterator breakIterator(rules, parseError, status);
if (status != U_BRK_RULE_SYNTAX) {
errln("%s:%d Expected U_BRK_RULE_SYNTAX, got %s", __FILE__, __LINE__, u_errorName(status));
}
if (parseError.line != 1 || parseError.offset != 0) {
errln("parseError (line, offset) expected (1, 0), got (%d, %d)", parseError.line, parseError.offset);
}
}
void RBBITest::TestBug12797() {
UnicodeString rules = "!!chain; !!forward; $v=b c; a b; $v; !!reverse; .*;";
UErrorCode status = U_ZERO_ERROR;
UParseError parseError;
RuleBasedBreakIterator bi(rules, parseError, status);
if (U_FAILURE(status)) {
errln("%s:%s status = %s", __FILE__, __LINE__, u_errorName(status));
return;
}
UnicodeString text = "abc";
bi.setText(text);
bi.first();
int32_t boundary = bi.next();
if (boundary != 3) {
errln("%s:%d expected boundary==3, got %d", __FILE__, __LINE__, boundary);
}
}
void RBBITest::TestBug12918() {
// This test triggers an assertion failure in dictbe.cpp
const UChar *crasherString = u"\u3325\u4a16";
UErrorCode status = U_ZERO_ERROR;
UBreakIterator* iter = ubrk_open(UBRK_WORD, NULL, crasherString, -1, &status);
if (U_FAILURE(status)) {
dataerrln("%s:%d status = %s", __FILE__, __LINE__, u_errorName(status));
return;
}
ubrk_first(iter);
int32_t pos = 0;
int32_t lastPos = -1;
while((pos = ubrk_next(iter)) != UBRK_DONE) {
if (pos <= lastPos) {
errln("%s:%d (pos, lastPos) = (%d, %d)", __FILE__, __LINE__, pos, lastPos);
break;
}
}
ubrk_close(iter);
}
void RBBITest::TestBug12932() {
// Node Stack overflow in the RBBI rule parser caused a seg fault.
UnicodeString ruleStr(
"((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((("
"((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((("
"(((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((()))"
")))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))"
")))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))"
")))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))");
UErrorCode status = U_ZERO_ERROR;
UParseError parseError;
RuleBasedBreakIterator rbbi(ruleStr, parseError, status);
if (status != U_BRK_RULE_SYNTAX) {
errln("%s:%d expected U_BRK_RULE_SYNTAX, got %s",
__FILE__, __LINE__, u_errorName(status));
}
}
// Emoji Test. Verify that the sequences defined in the Unicode data file emoji-test.txt
// remain undevided by ICU char, word and line break.
void RBBITest::TestEmoji() {
#if !UCONFIG_NO_REGULAR_EXPRESSIONS
UErrorCode status = U_ZERO_ERROR;
CharString testFileName;
testFileName.append(IntlTest::getSourceTestData(status), status);
testFileName.appendPathPart("emoji-test.txt", status);
if (U_FAILURE(status)) {
errln("%s:%s %s while opening emoji-test.txt", __FILE__, __LINE__, u_errorName(status));
return;
}
logln("Opening data file %s\n", testFileName.data());
int len;
UChar *testFile = ReadAndConvertFile(testFileName.data(), len, "UTF-8", status);
if (U_FAILURE(status) || testFile == NULL) {
errln("%s:%s %s while opening emoji-test.txt", __FILE__, __LINE__, u_errorName(status));
return;
}
UnicodeString testFileAsString(testFile, len);
delete [] testFile;
RegexMatcher lineMatcher(u"^.*?$", testFileAsString, UREGEX_MULTILINE, status);
RegexMatcher hexMatcher(u"\\s*([a-f0-9]*)", UREGEX_CASE_INSENSITIVE, status);
// hexMatcher group(1) is a hex number, or empty string if no hex number present.
int32_t lineNumber = 0;
LocalPointer<BreakIterator> charBreaks(BreakIterator::createCharacterInstance(Locale::getEnglish(), status), status);
LocalPointer<BreakIterator> wordBreaks(BreakIterator::createWordInstance(Locale::getEnglish(), status), status);
LocalPointer<BreakIterator> lineBreaks(BreakIterator::createLineInstance(Locale::getEnglish(), status), status);
if (U_FAILURE(status)) {
dataerrln("%s:%d %s while opening break iterators", __FILE__, __LINE__, u_errorName(status));
return;
}
while (lineMatcher.find()) {
++lineNumber;
UnicodeString line = lineMatcher.group(status);
hexMatcher.reset(line);
UnicodeString testString; // accumulates the emoji sequence.
while (hexMatcher.find() && hexMatcher.group(1, status).length() > 0) {
UnicodeString hex = hexMatcher.group(1, status);
if (hex.length() > 8) {
errln("%s:%d emoji-test.txt:%d invalid code point %s", __FILE__, __LINE__, lineNumber, CStr(hex)());
break;
}
CharString hex8;
hex8.appendInvariantChars(hex, status);
UChar32 c = (UChar32)strtol(hex8.data(), NULL, 16);
if (c<=0x10ffff) {
testString.append(c);
} else {
errln("%s:%d emoji-test.txt:%d Error: Unicode Character %s value out of range.",
__FILE__, __LINE__, lineNumber, hex8.data());
break;
}
}
if (testString.length() > 1) {
charBreaks->setText(testString);
charBreaks->first();
int32_t firstBreak = charBreaks->next();
if (testString.length() != firstBreak) {
errln("%s:%d emoji-test.txt:%d Error, uexpected break at offset %d",
__FILE__, __LINE__, lineNumber, firstBreak);
}
wordBreaks->setText(testString);
wordBreaks->first();
firstBreak = wordBreaks->next();
if (testString.length() != firstBreak) {
errln("%s:%d emoji-test.txt:%d Error, uexpected break at offset %d",
__FILE__, __LINE__, lineNumber, firstBreak);
}
lineBreaks->setText(testString);
lineBreaks->first();
firstBreak = lineBreaks->next();
if (testString.length() != firstBreak) {
errln("%s:%d emoji-test.txt:%d Error, uexpected break at offset %d",
__FILE__, __LINE__, lineNumber, firstBreak);
}
}
}
#endif
}
// TestBug12519 - Correct handling of Locales by assignment / copy / clone
// WHERE Macro yields a literal string of the form "source_file_name:line number "
// TODO: propose something equivalent as a test framework addition.
#define WHERE __FILE__ ":" XLINE(__LINE__) " "
#define XLINE(s) LINE(s)
#define LINE(s) #s
void RBBITest::TestBug12519() {
UErrorCode status = U_ZERO_ERROR;
LocalPointer<RuleBasedBreakIterator> biEn((RuleBasedBreakIterator *)BreakIterator::createWordInstance(Locale::getEnglish(), status));
LocalPointer<RuleBasedBreakIterator> biFr((RuleBasedBreakIterator *)BreakIterator::createWordInstance(Locale::getFrance(), status));
if (!assertSuccess(WHERE, status)) {
dataerrln("%s %d status = %s", __FILE__, __LINE__, u_errorName(status));
return;
}
assertTrue(WHERE, Locale::getEnglish() == biEn->getLocale(ULOC_VALID_LOCALE, status));
assertTrue(WHERE, Locale::getFrench() == biFr->getLocale(ULOC_VALID_LOCALE, status));
assertTrue(WHERE "Locales do not participate in BreakIterator equality.", *biEn == *biFr);
LocalPointer<RuleBasedBreakIterator>cloneEn((RuleBasedBreakIterator *)biEn->clone());
assertTrue(WHERE, *biEn == *cloneEn);
assertTrue(WHERE, Locale::getEnglish() == cloneEn->getLocale(ULOC_VALID_LOCALE, status));
LocalPointer<RuleBasedBreakIterator>cloneFr((RuleBasedBreakIterator *)biFr->clone());
assertTrue(WHERE, *biFr == *cloneFr);
assertTrue(WHERE, Locale::getFrench() == cloneFr->getLocale(ULOC_VALID_LOCALE, status));
LocalPointer<RuleBasedBreakIterator>biDe((RuleBasedBreakIterator *)BreakIterator::createLineInstance(Locale::getGerman(), status));
UnicodeString text("Hallo Welt");
biDe->setText(text);
assertTrue(WHERE "before assignment of \"biDe = biFr\", they should be different, but are equal.", *biFr != *biDe);
*biDe = *biFr;
assertTrue(WHERE "after assignment of \"biDe = biFr\", they should be equal, but are not.", *biFr == *biDe);
}
void RBBITest::TestBug12677() {
// Check that stripping of comments from rules for getRules() is not confused by
// the presence of '#' characters in the rules that do not introduce comments.
UnicodeString rules(u"!!forward; \n"
"$x = [ab#]; # a set with a # literal. \n"
" # .; # a comment that looks sort of like a rule. \n"
" '#' '?'; # a rule with a quoted # \n"
);
UErrorCode status = U_ZERO_ERROR;
UParseError pe;
RuleBasedBreakIterator bi(rules, pe, status);
assertSuccess(WHERE, status);
UnicodeString rtRules = bi.getRules();
assertEquals(WHERE, UnicodeString(u"!!forward; $x = [ab#]; '#' '?'; "), rtRules);
}
void RBBITest::TestTableRedundancies() {
UErrorCode status = U_ZERO_ERROR;
LocalPointer<RuleBasedBreakIterator> bi (
(RuleBasedBreakIterator *)BreakIterator::createLineInstance(Locale::getEnglish(), status));
assertSuccess(WHERE, status);
if (U_FAILURE(status)) return;
RBBIDataWrapper *dw = bi->fData;
const RBBIStateTable *fwtbl = dw->fForwardTable;
int32_t numCharClasses = dw->fHeader->fCatCount;
// printf("Char Classes: %d states: %d\n", numCharClasses, fwtbl->fNumStates);
// Check for duplicate columns (character categories)
std::vector<UnicodeString> columns;
for (int32_t column = 0; column < numCharClasses; column++) {
UnicodeString s;
for (int32_t r = 1; r < (int32_t)fwtbl->fNumStates; r++) {
RBBIStateTableRow *row = (RBBIStateTableRow *) (fwtbl->fTableData + (fwtbl->fRowLen * r));
s.append(row->fNextState[column]);
}
columns.push_back(s);
}
// Ignore column (char class) 0 while checking; it's special, and may have duplicates.
for (int c1=1; c1<numCharClasses; c1++) {
for (int c2 = c1+1; c2 < numCharClasses; c2++) {
if (columns.at(c1) == columns.at(c2)) {
errln("%s:%d Duplicate columns (%d, %d)\n", __FILE__, __LINE__, c1, c2);
goto out;
}
}
}
out:
// Check for duplicate states
std::vector<UnicodeString> rows;
for (int32_t r=0; r < (int32_t)fwtbl->fNumStates; r++) {
UnicodeString s;
RBBIStateTableRow *row = (RBBIStateTableRow *) (fwtbl->fTableData + (fwtbl->fRowLen * r));
assertTrue(WHERE, row->fAccepting >= -1);
s.append(row->fAccepting + 1); // values of -1 are expected.
s.append(row->fLookAhead);
s.append(row->fTagIdx);
for (int32_t column = 0; column < numCharClasses; column++) {
s.append(row->fNextState[column]);
}
rows.push_back(s);
}
for (int r1=0; r1 < (int32_t)fwtbl->fNumStates; r1++) {
for (int r2 = r1+1; r2 < (int32_t)fwtbl->fNumStates; r2++) {
if (rows.at(r1) == rows.at(r2)) {
errln("%s:%d Duplicate rows (%d, %d)\n", __FILE__, __LINE__, r1, r2);
return;
}
}
}
}
// Bug 13447: verify that getRuleStatus() returns the value corresponding to current(),
// even after next() has returned DONE.
void RBBITest::TestBug13447() {
UErrorCode status = U_ZERO_ERROR;
LocalPointer<RuleBasedBreakIterator> bi(
(RuleBasedBreakIterator *)BreakIterator::createWordInstance(Locale::getEnglish(), status));
assertSuccess(WHERE, status);
if (U_FAILURE(status)) return;
UnicodeString data(u"1234");
bi->setText(data);
assertEquals(WHERE, UBRK_WORD_NONE, bi->getRuleStatus());
assertEquals(WHERE, 4, bi->next());
assertEquals(WHERE, UBRK_WORD_NUMBER, bi->getRuleStatus());
assertEquals(WHERE, UBRK_DONE, bi->next());
assertEquals(WHERE, 4, bi->current());
assertEquals(WHERE, UBRK_WORD_NUMBER, bi->getRuleStatus());
}
// TestReverse exercises both the synthesized safe reverse rules and the logic
// for filling the break iterator cache when starting from random positions
// in the text.
//
// It's a monkey test, working on random data, with the expected data obtained
// from forward iteration (no safe rules involved), comparing with results
// when indexing into the interior of the string (safe rules needed).
void RBBITest::TestReverse() {
UErrorCode status = U_ZERO_ERROR;
TestReverse(std::unique_ptr<RuleBasedBreakIterator>((RuleBasedBreakIterator *)
BreakIterator::createCharacterInstance(Locale::getEnglish(), status)));
assertSuccess(WHERE, status, true);
status = U_ZERO_ERROR;
TestReverse(std::unique_ptr<RuleBasedBreakIterator>((RuleBasedBreakIterator *)
BreakIterator::createWordInstance(Locale::getEnglish(), status)));
assertSuccess(WHERE, status, true);
status = U_ZERO_ERROR;
TestReverse(std::unique_ptr<RuleBasedBreakIterator>((RuleBasedBreakIterator *)
BreakIterator::createLineInstance(Locale::getEnglish(), status)));
assertSuccess(WHERE, status, true);
status = U_ZERO_ERROR;
TestReverse(std::unique_ptr<RuleBasedBreakIterator>((RuleBasedBreakIterator *)
BreakIterator::createSentenceInstance(Locale::getEnglish(), status)));
assertSuccess(WHERE, status, true);
}
void RBBITest::TestReverse(std::unique_ptr<RuleBasedBreakIterator>bi) {
if (!bi) {
return;
}
// From the mapping trie in the break iterator's internal data, create a
// vector of UnicodeStrings, one for each character category, containing
// all of the code points that map to that category. Unicode planes 0 and 1 only,
// to avoid an execess of unassigned code points.
RBBIDataWrapper *data = bi->fData;
int32_t categoryCount = data->fHeader->fCatCount;
UTrie2 *trie = data->fTrie;
std::vector<UnicodeString> strings(categoryCount, UnicodeString());
for (int cp=0; cp<0x1fff0; ++cp) {
int cat = utrie2_get32(trie, cp);
cat &= ~0x4000; // And off the dictionary bit from the category.
assertTrue(WHERE, cat < categoryCount && cat >= 0);
if (cat < 0 || cat >= categoryCount) return;
strings[cat].append(cp);
}
icu_rand randomGen;
const int testStringLength = 10000;
UnicodeString testString;
for (int i=0; i<testStringLength; ++i) {
int charClass = randomGen() % categoryCount;
if (strings[charClass].length() > 0) {
int cp = strings[charClass].char32At(randomGen() % strings[charClass].length());
testString.append(cp);
}
}
typedef std::pair<UBool, int32_t> Result;
std::vector<Result> expectedResults;
bi->setText(testString);
for (int i=0; i<testString.length(); ++i) {
bool isboundary = bi->isBoundary(i);
int ruleStatus = bi->getRuleStatus();
expectedResults.push_back(std::make_pair(isboundary, ruleStatus));
}
for (int i=testString.length()-1; i>=0; --i) {
bi->setText(testString); // clears the internal break cache
Result expected = expectedResults[i];
assertEquals(WHERE, expected.first, bi->isBoundary(i));
assertEquals(WHERE, expected.second, bi->getRuleStatus());
}
}
// Ticket 13692 - finding word boundaries in very large numbers or words could
// be very time consuming. When the problem was present, this void test
// would run more than fifteen minutes, which is to say, the failure was noticeale.
void RBBITest::TestBug13692() {
UErrorCode status = U_ZERO_ERROR;
LocalPointer<RuleBasedBreakIterator> bi ((RuleBasedBreakIterator *)
BreakIterator::createWordInstance(Locale::getEnglish(), status), status);
if (!assertSuccess(WHERE, status, true)) {
return;
}
constexpr int32_t LENGTH = 1000000;
UnicodeString longNumber(LENGTH, (UChar32)u'3', LENGTH);
for (int i=0; i<20; i+=2) {
longNumber.setCharAt(i, u' ');
}
bi->setText(longNumber);
assertFalse(WHERE, bi->isBoundary(LENGTH-5));
assertSuccess(WHERE, status);
}
//
// TestDebug - A place-holder test for debugging purposes.
// For putting in fragments of other tests that can be invoked
// for tracing without a lot of unwanted extra stuff happening.
//
void RBBITest::TestDebug(void) {
UErrorCode status = U_ZERO_ERROR;
LocalPointer<RuleBasedBreakIterator> bi ((RuleBasedBreakIterator *)
BreakIterator::createCharacterInstance(Locale::getEnglish(), status), status);
if (!assertSuccess(WHERE, status, true)) {
return;
}
const UnicodeString &rules = bi->getRules();
UParseError pe;
LocalPointer<RuleBasedBreakIterator> newbi(new RuleBasedBreakIterator(rules, pe, status));
assertSuccess(WHERE, status);
}
void RBBITest::TestProperties() {
UErrorCode errorCode = U_ZERO_ERROR;
UnicodeSet prependSet(UNICODE_STRING_SIMPLE("[:GCB=Prepend:]"), errorCode);
if (!prependSet.isEmpty()) {
errln(
"[:GCB=Prepend:] is not empty any more. "
"Uncomment relevant lines in source/data/brkitr/char.txt and "
"change this test to the opposite condition.");
}
}
#endif // #if !UCONFIG_NO_BREAK_ITERATION