Google Git
Sign in
skia / external / github.com / unicode-org / icu / e8dfea9bb6bb27596731173b352759e44ad06b21 / . / icu4c / source / test / intltest / rbbitst.cpp
blob: 8e3086b515199214c4607e181dcdeb986110c228 [file] [log] [blame]
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/********************************************************************
* COPYRIGHT:
* Copyright (c) 1999-2016, International Business Machines Corporation and
* others. All Rights Reserved.
********************************************************************/
/************************************************************************
* Date Name Description
* 12/15/99 Madhu Creation.
* 01/12/2000 Madhu Updated for changed API and added new tests
************************************************************************/
#include "unicode/utypes.h"
#if !UCONFIG_NO_BREAK_ITERATION
#include <sstream>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <utility>
#include <vector>
#include "unicode/brkiter.h"
#include "unicode/localpointer.h"
#include "unicode/numfmt.h"
#include "unicode/rbbi.h"
#if !UCONFIG_NO_REGULAR_EXPRESSIONS
#include "unicode/regex.h"
#endif
#include "unicode/schriter.h"
#include "unicode/uchar.h"
#include "unicode/utf16.h"
#include "unicode/ucnv.h"
#include "unicode/uniset.h"
#include "unicode/uscript.h"
#include "unicode/ustring.h"
#include "unicode/utext.h"
#include "unicode/utrace.h"
#include "charstr.h"
#include "cmemory.h"
#include "cstr.h"
#include "intltest.h"
#include "rbbitst.h"
#include "rbbidata.h"
#include "utypeinfo.h" // for 'typeid' to work
#include "uvector.h"
#include "uvectr32.h"
#if !UCONFIG_NO_FILTERED_BREAK_ITERATION
#include "unicode/filteredbrk.h"
#endif // !UCONFIG_NO_FILTERED_BREAK_ITERATION
#define TEST_ASSERT(x) UPRV_BLOCK_MACRO_BEGIN { \
if (!(x)) { \
errln("Failure in file %s, line %d", __FILE__, __LINE__); \
} \
} UPRV_BLOCK_MACRO_END
#define TEST_ASSERT_SUCCESS(errcode) UPRV_BLOCK_MACRO_BEGIN { \
if (U_FAILURE(errcode)) { \
errcheckln(errcode, "Failure in file %s, line %d, status = \"%s\"", __FILE__, __LINE__, u_errorName(errcode)); \
} \
} UPRV_BLOCK_MACRO_END
#define MONKEY_ERROR(msg, fRuleFileName, index, seed) { \
IntlTest::gTest->errln("\n%s:%d %s at index %d. Parameters to reproduce: @\"type=%s seed=%u loop=1\"", \
__FILE__, __LINE__, msg, index, fRuleFileName, seed); \
}
//---------------------------------------------
// runIndexedTest
//---------------------------------------------
// Note: Before adding new tests to this file, check whether the desired test data can
// simply be added to the file testdata/rbbitest.txt. In most cases it can,
// it's much less work than writing a new test, diagnostic output in the event of failures
// is good, and the test data file will is shared with ICU4J, so eventually the test
// will run there as well, without additional effort.
void RBBITest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* params )
{
if (exec) logln("TestSuite RuleBasedBreakIterator: ");
fTestParams = params;
TESTCASE_AUTO_BEGIN;
#if !UCONFIG_NO_FILE_IO
TESTCASE_AUTO(TestBug4153072);
#endif
#if !UCONFIG_NO_FILE_IO
TESTCASE_AUTO(TestUnicodeFiles);
#endif
TESTCASE_AUTO(TestGetAvailableLocales);
TESTCASE_AUTO(TestGetDisplayName);
#if !UCONFIG_NO_FILE_IO
TESTCASE_AUTO(TestEndBehaviour);
TESTCASE_AUTO(TestWordBreaks);
TESTCASE_AUTO(TestWordBoundary);
TESTCASE_AUTO(TestLineBreaks);
TESTCASE_AUTO(TestSentBreaks);
TESTCASE_AUTO(TestExtended);
#endif
#if !UCONFIG_NO_REGULAR_EXPRESSIONS && !UCONFIG_NO_FILE_IO
TESTCASE_AUTO(TestMonkey);
#endif
#if !UCONFIG_NO_FILE_IO
TESTCASE_AUTO(TestBug3818);
#endif
TESTCASE_AUTO(TestDebug);
#if !UCONFIG_NO_FILE_IO
TESTCASE_AUTO(TestBug5775);
#endif
TESTCASE_AUTO(TestBug9983);
TESTCASE_AUTO(TestDictRules);
TESTCASE_AUTO(TestBug5532);
TESTCASE_AUTO(TestBug7547);
TESTCASE_AUTO(TestBug12797);
TESTCASE_AUTO(TestBug12918);
TESTCASE_AUTO(TestBug12932);
TESTCASE_AUTO(TestEmoji);
TESTCASE_AUTO(TestBug12519);
TESTCASE_AUTO(TestBug12677);
TESTCASE_AUTO(TestTableRedundancies);
TESTCASE_AUTO(TestBug13447);
TESTCASE_AUTO(TestReverse);
TESTCASE_AUTO(TestBug13692);
TESTCASE_AUTO(TestDebugRules);
TESTCASE_AUTO(Test8BitsTrieWith8BitStateTable);
TESTCASE_AUTO(Test8BitsTrieWith16BitStateTable);
TESTCASE_AUTO(Test16BitsTrieWith8BitStateTable);
TESTCASE_AUTO(Test16BitsTrieWith16BitStateTable);
TESTCASE_AUTO(TestTable_8_16_Bits);
TESTCASE_AUTO(TestBug13590);
#if U_ENABLE_TRACING
TESTCASE_AUTO(TestTraceCreateCharacter);
TESTCASE_AUTO(TestTraceCreateWord);
TESTCASE_AUTO(TestTraceCreateSentence);
TESTCASE_AUTO(TestTraceCreateTitle);
TESTCASE_AUTO(TestTraceCreateLine);
TESTCASE_AUTO(TestTraceCreateLineNormal);
TESTCASE_AUTO(TestTraceCreateLineLoose);
TESTCASE_AUTO(TestTraceCreateLineStrict);
TESTCASE_AUTO(TestTraceCreateBreakEngine);
#endif
TESTCASE_AUTO_END;
}
//--------------------------------------------------------------------------------------
//
// RBBITest constructor and destructor
//
//--------------------------------------------------------------------------------------
RBBITest::RBBITest() {
fTestParams = NULL;
}
RBBITest::~RBBITest() {
}
static void printStringBreaks(UText *tstr, int expected[], int expectedCount) {
UErrorCode status = U_ZERO_ERROR;
char name[100];
printf("code alpha extend alphanum type word sent line name\n");
int nextExpectedIndex = 0;
utext_setNativeIndex(tstr, 0);
for (int j = 0; j < static_cast<int>(utext_nativeLength(tstr)); j=static_cast<int>(utext_getNativeIndex(tstr))) {
if (nextExpectedIndex < expectedCount && j >= expected[nextExpectedIndex] ) {
printf("------------------------------------------------ %d\n", j);
++nextExpectedIndex;
}
UChar32 c = utext_next32(tstr);
u_charName(c, U_UNICODE_CHAR_NAME, name, 100, &status);
printf("%7x %5d %6d %8d %4s %4s %4s %4s %s\n", (int)c,
u_isUAlphabetic(c),
u_hasBinaryProperty(c, UCHAR_GRAPHEME_EXTEND),
u_isalnum(c),
u_getPropertyValueName(UCHAR_GENERAL_CATEGORY,
u_charType(c),
U_SHORT_PROPERTY_NAME),
u_getPropertyValueName(UCHAR_WORD_BREAK,
u_getIntPropertyValue(c,
UCHAR_WORD_BREAK),
U_SHORT_PROPERTY_NAME),
u_getPropertyValueName(UCHAR_SENTENCE_BREAK,
u_getIntPropertyValue(c,
UCHAR_SENTENCE_BREAK),
U_SHORT_PROPERTY_NAME),
u_getPropertyValueName(UCHAR_LINE_BREAK,
u_getIntPropertyValue(c,
UCHAR_LINE_BREAK),
U_SHORT_PROPERTY_NAME),
name);
}
}
static void printStringBreaks(const UnicodeString &ustr, int expected[], int expectedCount) {
UErrorCode status = U_ZERO_ERROR;
UText *tstr = NULL;
tstr = utext_openConstUnicodeString(NULL, &ustr, &status);
if (U_FAILURE(status)) {
printf("printStringBreaks, utext_openConstUnicodeString() returns %s\n", u_errorName(status));
return;
}
printStringBreaks(tstr, expected, expectedCount);
utext_close(tstr);
}
void RBBITest::TestBug3818() {
UErrorCode status = U_ZERO_ERROR;
// Four Thai words...
static const UChar thaiWordData[] = { 0x0E43,0x0E2B,0x0E0D,0x0E48, 0x0E43,0x0E2B,0x0E0D,0x0E48,
0x0E43,0x0E2B,0x0E0D,0x0E48, 0x0E43,0x0E2B,0x0E0D,0x0E48, 0 };
UnicodeString thaiStr(thaiWordData);
BreakIterator* bi = BreakIterator::createWordInstance(Locale("th"), status);
if (U_FAILURE(status) || bi == NULL) {
errcheckln(status, "Fail at file %s, line %d, status = %s", __FILE__, __LINE__, u_errorName(status));
return;
}
bi->setText(thaiStr);
int32_t startOfSecondWord = bi->following(1);
if (startOfSecondWord != 4) {
errln("Fail at file %s, line %d expected start of word at 4, got %d",
__FILE__, __LINE__, startOfSecondWord);
}
startOfSecondWord = bi->following(0);
if (startOfSecondWord != 4) {
errln("Fail at file %s, line %d expected start of word at 4, got %d",
__FILE__, __LINE__, startOfSecondWord);
}
delete bi;
}
//---------------------------------------------
//
// other tests
//
//---------------------------------------------
void RBBITest::TestGetAvailableLocales()
{
int32_t locCount = 0;
const Locale* locList = BreakIterator::getAvailableLocales(locCount);
if (locCount == 0)
dataerrln("getAvailableLocales() returned an empty list!");
// Just make sure that it's returning good memory.
int32_t i;
for (i = 0; i < locCount; ++i) {
logln(locList[i].getName());
}
}
//Testing the BreakIterator::getDisplayName() function
void RBBITest::TestGetDisplayName()
{
UnicodeString result;
BreakIterator::getDisplayName(Locale::getUS(), result);
if (Locale::getDefault() == Locale::getUS() && result != "English (United States)")
dataerrln("BreakIterator::getDisplayName() failed: expected \"English (United States)\", got \""
+ result);
BreakIterator::getDisplayName(Locale::getFrance(), Locale::getUS(), result);
if (result != "French (France)")
dataerrln("BreakIterator::getDisplayName() failed: expected \"French (France)\", got \""
+ result);
}
/**
* Test End Behaviour
* @bug 4068137
*/
void RBBITest::TestEndBehaviour()
{
UErrorCode status = U_ZERO_ERROR;
UnicodeString testString("boo.");
BreakIterator *wb = BreakIterator::createWordInstance(Locale::getDefault(), status);
if (U_FAILURE(status))
{
errcheckln(status, "Failed to create the BreakIterator for default locale in TestEndBehaviour. - %s", u_errorName(status));
return;
}
wb->setText(testString);
if (wb->first() != 0)
errln("Didn't get break at beginning of string.");
if (wb->next() != 3)
errln("Didn't get break before period in \"boo.\"");
if (wb->current() != 4 && wb->next() != 4)
errln("Didn't get break at end of string.");
delete wb;
}
/*
* @bug 4153072
*/
void RBBITest::TestBug4153072() {
UErrorCode status = U_ZERO_ERROR;
BreakIterator *iter = BreakIterator::createWordInstance(Locale::getDefault(), status);
if (U_FAILURE(status))
{
errcheckln(status, "Failed to create the BreakIterator for default locale in TestBug4153072 - %s", u_errorName(status));
return;
}
UnicodeString str("...Hello, World!...");
int32_t begin = 3;
int32_t end = str.length() - 3;
UBool onBoundary;
StringCharacterIterator* textIterator = new StringCharacterIterator(str, begin, end, begin);
iter->adoptText(textIterator);
int index;
// Note: with the switch to UText, there is no way to restrict the
// iteration range to begin at an index other than zero.
// String character iterators created with a non-zero bound are
// treated by RBBI as being empty.
for (index = -1; index < begin + 1; ++index) {
onBoundary = iter->isBoundary(index);
if (index == 0? !onBoundary : onBoundary) {
errln((UnicodeString)"Didn't handle isBoundary correctly with offset = " + index +
" and begin index = " + begin);
}
}
delete iter;
}
//
// Test for problem reported by Ashok Matoria on 9 July 2007
// One.<kSoftHyphen><kSpace>Two.
//
// Sentence break at start (0) and then on calling next() it breaks at
// 'T' of "Two". Now, at this point if I do next() and
// then previous(), it breaks at <kSOftHyphen> instead of 'T' of "Two".
//
void RBBITest::TestBug5775() {
UErrorCode status = U_ZERO_ERROR;
BreakIterator *bi = BreakIterator::createSentenceInstance(Locale::getEnglish(), status);
TEST_ASSERT_SUCCESS(status);
if (U_FAILURE(status)) {
return;
}
// Check for status first for better handling of no data errors.
TEST_ASSERT(bi != NULL);
if (bi == NULL) {
return;
}
UnicodeString s("One.\\u00ad Two.", -1, US_INV);
// 01234 56789
s = s.unescape();
bi->setText(s);
int pos = bi->next();
TEST_ASSERT(pos == 6);
pos = bi->next();
TEST_ASSERT(pos == 10);
pos = bi->previous();
TEST_ASSERT(pos == 6);
delete bi;
}
//------------------------------------------------------------------------------
//
// RBBITest::Extended Run RBBI Tests from an external test data file
//
//------------------------------------------------------------------------------
struct TestParams {
BreakIterator *bi; // Break iterator is set while parsing test source.
// Changed out whenever test data changes break type.
UnicodeString dataToBreak; // Data that is built up while parsing the test.
UVector32 *expectedBreaks; // Expected break positions, matches dataToBreak UnicodeString.
UVector32 *srcLine; // Positions in source file, indexed same as dataToBreak.
UVector32 *srcCol;
UText *textToBreak; // UText, could be UTF8 or UTF16.
UVector32 *textMap; // Map from UTF-16 dataToBreak offsets to UText offsets.
CharString utf8String; // UTF-8 form of text to break.
TestParams(UErrorCode &status) : dataToBreak() {
bi = NULL;
expectedBreaks = new UVector32(status);
srcLine = new UVector32(status);
srcCol = new UVector32(status);
textToBreak = NULL;
textMap = new UVector32(status);
}
~TestParams() {
delete bi;
delete expectedBreaks;
delete srcLine;
delete srcCol;
utext_close(textToBreak);
delete textMap;
}
int32_t getSrcLine(int32_t bp);
int32_t getExpectedBreak(int32_t bp);
int32_t getSrcCol(int32_t bp);
void setUTF16(UErrorCode &status);
void setUTF8(UErrorCode &status);
};
// Append a UnicodeString to a CharString with UTF-8 encoding.
// Substitute any invalid chars.
// Note: this is used with test data that includes a few unpaired surrogates in the UTF-16 that will be substituted.
static void CharStringAppend(CharString &dest, const UnicodeString &src, UErrorCode &status) {
if (U_FAILURE(status)) {
return;
}
int32_t utf8Length;
u_strToUTF8WithSub(NULL, 0, &utf8Length, // Output Buffer, NULL for preflight.
src.getBuffer(), src.length(), // UTF-16 data
0xfffd, NULL, // Substitution char, number of subs.
&status);
if (U_FAILURE(status) && status != U_BUFFER_OVERFLOW_ERROR) {
return;
}
status = U_ZERO_ERROR;
int32_t capacity;
char *buffer = dest.getAppendBuffer(utf8Length, utf8Length, capacity, status);
u_strToUTF8WithSub(buffer, utf8Length, NULL,
src.getBuffer(), src.length(),
0xfffd, NULL, &status);
dest.append(buffer, utf8Length, status);
}
void TestParams::setUTF16(UErrorCode &status) {
textToBreak = utext_openUnicodeString(textToBreak, &dataToBreak, &status);
textMap->removeAllElements();
for (int32_t i=0; i<dataToBreak.length(); i++) {
if (i == dataToBreak.getChar32Start(i)) {
textMap->addElement(i, status);
} else {
textMap->addElement(-1, status);
}
}
textMap->addElement(dataToBreak.length(), status);
U_ASSERT(dataToBreak.length() + 1 == textMap->size());
}
void TestParams::setUTF8(UErrorCode &status) {
if (U_FAILURE(status)) {
return;
}
utf8String.clear();
CharStringAppend(utf8String, dataToBreak, status);
textToBreak = utext_openUTF8(textToBreak, utf8String.data(), utf8String.length(), &status);
if (U_FAILURE(status)) {
return;
}
textMap->removeAllElements();
int32_t utf16Index = 0;
for (;;) {
textMap->addElement(utf16Index, status);
UChar32 c32 = utext_current32(textToBreak);
if (c32 < 0) {
break;
}
utf16Index += U16_LENGTH(c32);
utext_next32(textToBreak);
while (textMap->size() < utext_getNativeIndex(textToBreak)) {
textMap->addElement(-1, status);
}
}
U_ASSERT(utext_nativeLength(textToBreak) + 1 == textMap->size());
}
int32_t TestParams::getSrcLine(int32_t bp) {
if (bp >= textMap->size()) {
bp = textMap->size() - 1;
}
int32_t i = 0;
for(; bp >= 0 ; --bp) {
// Move to a character boundary if we are not on one already.
i = textMap->elementAti(bp);
if (i >= 0) {
break;
}
}
return srcLine->elementAti(i);
}
int32_t TestParams::getExpectedBreak(int32_t bp) {
if (bp >= textMap->size()) {
return 0;
}
int32_t i = textMap->elementAti(bp);
int32_t retVal = 0;
if (i >= 0) {
retVal = expectedBreaks->elementAti(i);
}
return retVal;
}
int32_t TestParams::getSrcCol(int32_t bp) {
if (bp >= textMap->size()) {
bp = textMap->size() - 1;
}
int32_t i = 0;
for(; bp >= 0; --bp) {
// Move bp to a character boundary if we are not on one already.
i = textMap->elementAti(bp);
if (i >= 0) {
break;
}
}
return srcCol->elementAti(i);
}
void RBBITest::executeTest(TestParams *t, UErrorCode &status) {
int32_t bp;
int32_t prevBP;
int32_t i;
TEST_ASSERT_SUCCESS(status);
if (U_FAILURE(status)) {
return;
}
if (t->bi == NULL) {
return;
}
t->bi->setText(t->textToBreak, status);
//
// Run the iterator forward
//
prevBP = -1;
for (bp = t->bi->first(); bp != BreakIterator::DONE; bp = t->bi->next()) {
if (prevBP == bp) {
// Fail for lack of forward progress.
errln("Forward Iteration, no forward progress. Break Pos=%4d File line,col=%4d,%4d",
bp, t->getSrcLine(bp), t->getSrcCol(bp));
break;
}
// Check that there we didn't miss an expected break between the last one
// and this one.
for (i=prevBP+1; i<bp; i++) {
if (t->getExpectedBreak(i) != 0) {
int expected[] = {0, i};
printStringBreaks(t->dataToBreak, expected, 2);
errln("Forward Iteration, break expected, but not found. Pos=%4d File line,col= %4d,%4d",
i, t->getSrcLine(i), t->getSrcCol(i));
}
}
// Check that the break we did find was expected
if (t->getExpectedBreak(bp) == 0) {
int expected[] = {0, bp};
printStringBreaks(t->textToBreak, expected, 2);
errln("Forward Iteration, break found, but not expected. Pos=%4d File line,col= %4d,%4d",
bp, t->getSrcLine(bp), t->getSrcCol(bp));
} else {
// The break was expected.
// Check that the {nnn} tag value is correct.
int32_t expectedTagVal = t->getExpectedBreak(bp);
if (expectedTagVal == -1) {
expectedTagVal = 0;
}
int32_t line = t->getSrcLine(bp);
int32_t rs = t->bi->getRuleStatus();
if (rs != expectedTagVal) {
errln("Incorrect status for forward break. Pos=%4d File line,col= %4d,%4d.\n"
" Actual, Expected status = %4d, %4d",
bp, line, t->getSrcCol(bp), rs, expectedTagVal);
}
}
prevBP = bp;
}
// Verify that there were no missed expected breaks after the last one found
for (i=prevBP+1; i<utext_nativeLength(t->textToBreak); i++) {
if (t->getExpectedBreak(i) != 0) {
errln("Forward Iteration, break expected, but not found. Pos=%4d File line,col= %4d,%4d",
i, t->getSrcLine(i), t->getSrcCol(i));
}
}
//
// Run the iterator backwards, verify that the same breaks are found.
//
prevBP = static_cast<int32_t>(utext_nativeLength(t->textToBreak) + 2); // start with a phony value for the last break pos seen.
bp = t->bi->last();
while (bp != BreakIterator::DONE) {
if (prevBP == bp) {
// Fail for lack of progress.
errln("Reverse Iteration, no progress. Break Pos=%4d File line,col=%4d,%4d",
bp, t->getSrcLine(bp), t->getSrcCol(bp));
break;
}
// Check that we didn't miss an expected break between the last one
// and this one. (UVector returns zeros for index out of bounds.)
for (i=prevBP-1; i>bp; i--) {
if (t->getExpectedBreak(i) != 0) {
errln("Reverse Iteration, break expected, but not found. Pos=%4d File line,col= %4d,%4d",
i, t->getSrcLine(i), t->getSrcCol(i));
}
}
// Check that the break we did find was expected
if (t->getExpectedBreak(bp) == 0) {
errln("Reverse Itertion, break found, but not expected. Pos=%4d File line,col= %4d,%4d",
bp, t->getSrcLine(bp), t->getSrcCol(bp));
} else {
// The break was expected.
// Check that the {nnn} tag value is correct.
int32_t expectedTagVal = t->getExpectedBreak(bp);
if (expectedTagVal == -1) {
expectedTagVal = 0;
}
int line = t->getSrcLine(bp);
int32_t rs = t->bi->getRuleStatus();
if (rs != expectedTagVal) {
errln("Incorrect status for reverse break. Pos=%4d File line,col= %4d,%4d.\n"
" Actual, Expected status = %4d, %4d",
bp, line, t->getSrcCol(bp), rs, expectedTagVal);
}
}
prevBP = bp;
bp = t->bi->previous();
}
// Verify that there were no missed breaks prior to the last one found
for (i=prevBP-1; i>=0; i--) {
if (t->getExpectedBreak(i) != 0) {
errln("Forward Itertion, break expected, but not found. Pos=%4d File line,col= %4d,%4d",
i, t->getSrcLine(i), t->getSrcCol(i));
}
}
// Check isBoundary()
for (i=0; i < utext_nativeLength(t->textToBreak); i++) {
UBool boundaryExpected = (t->getExpectedBreak(i) != 0);
UBool boundaryFound = t->bi->isBoundary(i);
if (boundaryExpected != boundaryFound) {
errln("isBoundary(%d) incorrect. File line,col= %4d,%4d\n"
" Expected, Actual= %s, %s",
i, t->getSrcLine(i), t->getSrcCol(i),
boundaryExpected ? "true":"false", boundaryFound? "true" : "false");
}
}
// Check following()
for (i=0; i < static_cast<int32_t>(utext_nativeLength(t->textToBreak)); i++) {
int32_t actualBreak = t->bi->following(i);
int32_t expectedBreak = BreakIterator::DONE;
for (int32_t j=i+1; j <= static_cast<int32_t>(utext_nativeLength(t->textToBreak)); j++) {
if (t->getExpectedBreak(j) != 0) {
expectedBreak = j;
break;
}
}
if (expectedBreak != actualBreak) {
errln("following(%d) incorrect. File line,col= %4d,%4d\n"
" Expected, Actual= %d, %d",
i, t->getSrcLine(i), t->getSrcCol(i), expectedBreak, actualBreak);
}
}
// Check preceding()
for (i=static_cast<int32_t>(utext_nativeLength(t->textToBreak)); i>=0; i--) {
int32_t actualBreak = t->bi->preceding(i);
int32_t expectedBreak = BreakIterator::DONE;
// For UTF-8 & UTF-16 supplementals, all code units of a character are equivalent.
// preceding(trailing byte) will return the index of some preceding code point,
// not the lead byte of the current code point, even though that has a smaller index.
// Therefore, start looking at the expected break data not at i-1, but at
// the start of code point index - 1.
utext_setNativeIndex(t->textToBreak, i);
int32_t j = static_cast<int32_t>(utext_getNativeIndex(t->textToBreak) - 1);
for (; j >= 0; j--) {
if (t->getExpectedBreak(j) != 0) {
expectedBreak = j;
break;
}
}
if (expectedBreak != actualBreak) {
errln("preceding(%d) incorrect. File line,col= %4d,%4d\n"
" Expected, Actual= %d, %d",
i, t->getSrcLine(i), t->getSrcCol(i), expectedBreak, actualBreak);
}
}
}
void RBBITest::TestExtended() {
// Skip test for now when UCONFIG_NO_FILTERED_BREAK_ITERATION is set. This
// data driven test closely entangles filtered and regular data.
#if !UCONFIG_NO_REGULAR_EXPRESSIONS && !UCONFIG_NO_FILTERED_BREAK_ITERATION
UErrorCode status = U_ZERO_ERROR;
Locale locale("");
TestParams tp(status);
RegexMatcher localeMatcher(UnicodeString(u"<locale *([\\p{L}\\p{Nd}_@&=-]*) *>"), 0, status);
if (U_FAILURE(status)) {
dataerrln("Failure in file %s, line %d, status = \"%s\"", __FILE__, __LINE__, u_errorName(status));
}
//
// Open and read the test data file.
//
const char *testDataDirectory = IntlTest::getSourceTestData(status);
CharString testFileName(testDataDirectory, -1, status);
testFileName.append("rbbitst.txt", -1, status);
int len;
UChar *testFile = ReadAndConvertFile(testFileName.data(), len, "UTF-8", status);
if (U_FAILURE(status)) {
errln("%s:%d Error %s opening file rbbitst.txt", __FILE__, __LINE__, u_errorName(status));
return;
}
bool skipTest = false; // Skip this test?
//
// Put the test data into a UnicodeString
//
UnicodeString testString(FALSE, testFile, len);
enum EParseState{
PARSE_COMMENT,
PARSE_TAG,
PARSE_DATA,
PARSE_NUM,
PARSE_RULES
}
parseState = PARSE_TAG;
EParseState savedState = PARSE_TAG;
int32_t lineNum = 1;
int32_t colStart = 0;
int32_t column = 0;
int32_t charIdx = 0;
int32_t tagValue = 0; // The numeric value of a <nnn> tag.
UnicodeString rules; // Holds rules from a <rules> ... </rules> block
int32_t rulesFirstLine = 0; // Line number of the start of current <rules> block
for (charIdx = 0; charIdx < len; ) {
status = U_ZERO_ERROR;
UChar c = testString.charAt(charIdx);
charIdx++;
if (c == u'\r' && charIdx<len && testString.charAt(charIdx) == u'\n') {
// treat CRLF as a unit
c = u'\n';
charIdx++;
}
if (c == u'\n' || c == u'\r') {
lineNum++;
colStart = charIdx;
}
column = charIdx - colStart + 1;
switch (parseState) {
case PARSE_COMMENT:
if (c == u'\n' || c == u'\r') {
parseState = savedState;
}
break;
case PARSE_TAG:
{
if (c == u'#') {
parseState = PARSE_COMMENT;
savedState = PARSE_TAG;
break;
}
if (u_isUWhiteSpace(c)) {
break;
}
if (testString.compare(charIdx-1, 6, u"<word>") == 0) {
delete tp.bi;
tp.bi = BreakIterator::createWordInstance(locale, status);
skipTest = false;
charIdx += 5;
break;
}
if (testString.compare(charIdx-1, 6, u"<char>") == 0) {
delete tp.bi;
tp.bi = BreakIterator::createCharacterInstance(locale, status);
skipTest = false;
charIdx += 5;
break;
}
if (testString.compare(charIdx-1, 6, u"<line>") == 0) {
delete tp.bi;
tp.bi = BreakIterator::createLineInstance(locale, status);
skipTest = false;
charIdx += 5;
break;
}
if (testString.compare(charIdx-1, 6, u"<sent>") == 0) {
delete tp.bi;
tp.bi = BreakIterator::createSentenceInstance(locale, status);
skipTest = false;
charIdx += 5;
break;
}
if (testString.compare(charIdx-1, 7, u"<title>") == 0) {
delete tp.bi;
tp.bi = BreakIterator::createTitleInstance(locale, status);
charIdx += 6;
break;
}
if (testString.compare(charIdx-1, 7, u"<rules>") == 0 ||
testString.compare(charIdx-1, 10, u"<badrules>") == 0) {
charIdx = testString.indexOf(u'>', charIdx) + 1;
parseState = PARSE_RULES;
rules.remove();
rulesFirstLine = lineNum;
break;
}
// <locale loc_name>
localeMatcher.reset(testString);
if (localeMatcher.lookingAt(charIdx-1, status)) {
UnicodeString localeName = localeMatcher.group(1, status);
char localeName8[100];
localeName.extract(0, localeName.length(), localeName8, sizeof(localeName8), 0);
locale = Locale::createFromName(localeName8);
charIdx += localeMatcher.group(0, status).length() - 1;
TEST_ASSERT_SUCCESS(status);
break;
}
if (testString.compare(charIdx-1, 6, u"<data>") == 0) {
parseState = PARSE_DATA;
charIdx += 5;
tp.dataToBreak = "";
tp.expectedBreaks->removeAllElements();
tp.srcCol ->removeAllElements();
tp.srcLine->removeAllElements();
break;
}
errln("line %d: Tag expected in test file.", lineNum);
parseState = PARSE_COMMENT;
savedState = PARSE_DATA;
goto end_test; // Stop the test.
}
break;
case PARSE_RULES:
if (testString.compare(charIdx-1, 8, u"</rules>") == 0) {
charIdx += 7;
parseState = PARSE_TAG;
delete tp.bi;
UParseError pe;
tp.bi = new RuleBasedBreakIterator(rules, pe, status);
skipTest = U_FAILURE(status);
if (U_FAILURE(status)) {
errln("file rbbitst.txt: %d - Error %s creating break iterator from rules.",
rulesFirstLine + pe.line - 1, u_errorName(status));
}
} else if (testString.compare(charIdx-1, 11, u"</badrules>") == 0) {
charIdx += 10;
parseState = PARSE_TAG;
UErrorCode ec = U_ZERO_ERROR;
UParseError pe;
RuleBasedBreakIterator bi(rules, pe, ec);
if (U_SUCCESS(ec)) {
errln("file rbbitst.txt: %d - Expected, but did not get, a failure creating break iterator from rules.",
rulesFirstLine + pe.line - 1);
}
} else {
rules.append(c);
}
break;
case PARSE_DATA:
if (c == u'•') {
int32_t breakIdx = tp.dataToBreak.length();
if (tp.expectedBreaks->size() > breakIdx) {
errln("rbbitst.txt:%d:%d adjacent expected breaks with no intervening test text",
lineNum, column);
}
tp.expectedBreaks->setSize(breakIdx+1);
tp.expectedBreaks->setElementAt(-1, breakIdx);
tp.srcLine->setSize(breakIdx+1);
tp.srcLine->setElementAt(lineNum, breakIdx);
tp.srcCol ->setSize(breakIdx+1);
tp.srcCol ->setElementAt(column, breakIdx);
break;
}
if (testString.compare(charIdx-1, 7, u"</data>") == 0) {
// Add final entry to mappings from break location to source file position.
// Need one extra because last break position returned is after the
// last char in the data, not at the last char.
tp.srcLine->addElement(lineNum, status);
tp.srcCol ->addElement(column, status);
parseState = PARSE_TAG;
charIdx += 6;
if (!skipTest) {
// RUN THE TEST!
status = U_ZERO_ERROR;
tp.setUTF16(status);
executeTest(&tp, status);
TEST_ASSERT_SUCCESS(status);
// Run again, this time with UTF-8 text wrapped in a UText.
status = U_ZERO_ERROR;
tp.setUTF8(status);
TEST_ASSERT_SUCCESS(status);
executeTest(&tp, status);
}
break;
}
if (testString.compare(charIdx-1, 3, u"\\N{") == 0) {
// Named character, e.g. \N{COMBINING GRAVE ACCENT}
// Get the code point from the name and insert it into the test data.
// (Damn, no API takes names in Unicode !!!
// we've got to take it back to char *)
int32_t nameEndIdx = testString.indexOf(u'}', charIdx);
int32_t nameLength = nameEndIdx - (charIdx+2);
char charNameBuf[200];
UChar32 theChar = -1;
if (nameEndIdx != -1) {
UErrorCode status = U_ZERO_ERROR;
testString.extract(charIdx+2, nameLength, charNameBuf, sizeof(charNameBuf));
charNameBuf[sizeof(charNameBuf)-1] = 0;
theChar = u_charFromName(U_UNICODE_CHAR_NAME, charNameBuf, &status);
if (U_FAILURE(status)) {
theChar = -1;
}
}
if (theChar == -1) {
errln("Error in named character in test file at line %d, col %d",
lineNum, column);
} else {
// Named code point was recognized. Insert it
// into the test data.
tp.dataToBreak.append(theChar);
while (tp.dataToBreak.length() > tp.srcLine->size()) {
tp.srcLine->addElement(lineNum, status);
tp.srcCol ->addElement(column, status);
}
}
if (nameEndIdx > charIdx) {
charIdx = nameEndIdx+1;
}
break;
}
if (testString.compare(charIdx-1, 2, u"<>") == 0) {
charIdx++;
int32_t breakIdx = tp.dataToBreak.length();
tp.expectedBreaks->setSize(breakIdx+1);
tp.expectedBreaks->setElementAt(-1, breakIdx);
tp.srcLine->setSize(breakIdx+1);
tp.srcLine->setElementAt(lineNum, breakIdx);
tp.srcCol ->setSize(breakIdx+1);
tp.srcCol ->setElementAt(column, breakIdx);
break;
}
if (c == u'<') {
tagValue = 0;
parseState = PARSE_NUM;
break;
}
if (c == u'#' && column==3) { // TODO: why is column off so far?
parseState = PARSE_COMMENT;
savedState = PARSE_DATA;
break;
}
if (c == u'\\') {
// Check for \ at end of line, a line continuation.
// Advance over (discard) the newline
UChar32 cp = testString.char32At(charIdx);
if (cp == u'\r' && charIdx<len && testString.charAt(charIdx+1) == u'\n') {
// We have a CR LF
// Need an extra increment of the input ptr to move over both of them
charIdx++;
}
if (cp == u'\n' || cp == u'\r') {
lineNum++;
colStart = charIdx;
charIdx++;
break;
}
// Let unescape handle the back slash.
cp = testString.unescapeAt(charIdx);
if (cp != -1) {
// Escape sequence was recognized. Insert the char
// into the test data.
tp.dataToBreak.append(cp);
while (tp.dataToBreak.length() > tp.srcLine->size()) {
tp.srcLine->addElement(lineNum, status);
tp.srcCol ->addElement(column, status);
}
break;
}
// Not a recognized backslash escape sequence.
// Take the next char as a literal.
// TODO: Should this be an error?
c = testString.charAt(charIdx);
charIdx = testString.moveIndex32(charIdx, 1);
}
// Normal, non-escaped data char.
tp.dataToBreak.append(c);
// Save the mapping from offset in the data to line/column numbers in
// the original input file. Will be used for better error messages only.
// If there's an expected break before this char, the slot in the mapping
// vector will already be set for this char; don't overwrite it.
if (tp.dataToBreak.length() > tp.srcLine->size()) {
tp.srcLine->addElement(lineNum, status);
tp.srcCol ->addElement(column, status);
}
break;
case PARSE_NUM:
// We are parsing an expected numeric tag value, like <1234>,
// within a chunk of data.
if (u_isUWhiteSpace(c)) {
break;
}
if (c == u'>') {
// Finished the number. Add the info to the expected break data,
// and switch parse state back to doing plain data.
parseState = PARSE_DATA;
if (tagValue == 0) {
tagValue = -1;
}
int32_t breakIdx = tp.dataToBreak.length();
if (tp.expectedBreaks->size() > breakIdx) {
errln("rbbitst.txt:%d:%d adjacent expected breaks with no intervening test text",
lineNum, column);
}
tp.expectedBreaks->setSize(breakIdx+1);
tp.expectedBreaks->setElementAt(tagValue, breakIdx);
tp.srcLine->setSize(breakIdx+1);
tp.srcLine->setElementAt(lineNum, breakIdx);
tp.srcCol ->setSize(breakIdx+1);
tp.srcCol ->setElementAt(column, breakIdx);
break;
}
if (u_isdigit(c)) {
tagValue = tagValue*10 + u_charDigitValue(c);
break;
}
errln("Syntax Error in test file at line %d, col %d",
lineNum, column);
parseState = PARSE_COMMENT;
goto end_test; // Stop the test
break;
}
if (U_FAILURE(status)) {
dataerrln("ICU Error %s while parsing test file at line %d.",
u_errorName(status), lineNum);
status = U_ZERO_ERROR;
goto end_test; // Stop the test
}
}
// Reached end of test file. Raise an error if parseState indicates that we are
// within a block that should have been terminated.
if (parseState == PARSE_RULES) {
errln("rbbitst.txt:%d <rules> block beginning at line %d is not closed.",
lineNum, rulesFirstLine);
}
if (parseState == PARSE_DATA) {
errln("rbbitst.txt:%d <data> block not closed.", lineNum);
}
end_test:
delete [] testFile;
#endif
}
//-------------------------------------------------------------------------------
//
// TestDictRules create a break iterator from source rules that includes a
// dictionary range. Regression for bug #7130. Source rules
// do not declare a break iterator type (word, line, sentence, etc.
// but the dictionary code, without a type, would loop.
//
//-------------------------------------------------------------------------------
void RBBITest::TestDictRules() {
const char *rules = "$dictionary = [a-z]; \n"
"!!forward; \n"
"$dictionary $dictionary; \n"
"!!reverse; \n"
"$dictionary $dictionary; \n";
const char *text = "aa";
UErrorCode status = U_ZERO_ERROR;
UParseError parseError;
RuleBasedBreakIterator bi(rules, parseError, status);
if (U_SUCCESS(status)) {
UnicodeString utext = text;
bi.setText(utext);
int32_t position;
int32_t loops;
for (loops = 0; loops<10; loops++) {
position = bi.next();
if (position == RuleBasedBreakIterator::DONE) {
break;
}
}
TEST_ASSERT(loops == 1);
} else {
dataerrln("Error creating RuleBasedBreakIterator: %s", u_errorName(status));
}
}
//-------------------------------------------------------------------------------
//
// ReadAndConvertFile Read a text data file, convert it to UChars, and
// return the data in one big UChar * buffer, which the caller must delete.
//
// parameters:
// fileName: the name of the file, with no directory part. The test data directory
// is assumed.
// ulen an out parameter, receives the actual length (in UChars) of the file data.
// encoding The file encoding. If the file contains a BOM, that will override the encoding
// specified here. The BOM, if it exists, will be stripped from the returned data.
// Pass NULL for the system default encoding.
// status
// returns:
// The file data, converted to UChar.
// The caller must delete this when done with
// delete [] theBuffer;
//
// TODO: This is a clone of RegexTest::ReadAndConvertFile.
// Move this function to some common place.
//
//--------------------------------------------------------------------------------
UChar *RBBITest::ReadAndConvertFile(const char *fileName, int &ulen, const char *encoding, UErrorCode &status) {
UChar *retPtr = NULL;
char *fileBuf = NULL;
UConverter* conv = NULL;
FILE *f = NULL;
ulen = 0;
if (U_FAILURE(status)) {
return retPtr;
}
//
// Open the file.
//
f = fopen(fileName, "rb");
if (f == 0) {
dataerrln("Error opening test data file %s\n", fileName);
status = U_FILE_ACCESS_ERROR;
return NULL;
}
//
// Read it in
//
int fileSize;
int amt_read;
fseek( f, 0, SEEK_END);
fileSize = ftell(f);
fileBuf = new char[fileSize];
fseek(f, 0, SEEK_SET);
amt_read = static_cast<int>(fread(fileBuf, 1, fileSize, f));
if (amt_read != fileSize || fileSize <= 0) {
errln("Error reading test data file.");
goto cleanUpAndReturn;
}
//
// Look for a Unicode Signature (BOM) on the data just read
//
int32_t signatureLength;
const char * fileBufC;
const char* bomEncoding;
fileBufC = fileBuf;
bomEncoding = ucnv_detectUnicodeSignature(
fileBuf, fileSize, &signatureLength, &status);
if(bomEncoding!=NULL ){
fileBufC += signatureLength;
fileSize -= signatureLength;
encoding = bomEncoding;
}
//
// Open a converter to take the rule file to UTF-16
//
conv = ucnv_open(encoding, &status);
if (U_FAILURE(status)) {
goto cleanUpAndReturn;
}
//
// Convert the rules to UChar.
// Preflight first to determine required buffer size.
//
ulen = ucnv_toUChars(conv,
NULL, // dest,
0, // destCapacity,
fileBufC,
fileSize,
&status);
if (status == U_BUFFER_OVERFLOW_ERROR) {
// Buffer Overflow is expected from the preflight operation.
status = U_ZERO_ERROR;
retPtr = new UChar[ulen+1];
ucnv_toUChars(conv,
retPtr, // dest,
ulen+1,
fileBufC,
fileSize,
&status);
}
cleanUpAndReturn:
fclose(f);
delete []fileBuf;
ucnv_close(conv);
if (U_FAILURE(status)) {
errln("ucnv_toUChars: ICU Error \"%s\"\n", u_errorName(status));
delete []retPtr;
retPtr = 0;
ulen = 0;
}
return retPtr;
}
//--------------------------------------------------------------------------------------------
//
// Run tests from each of the boundary test data files distributed by the Unicode Consortium
//
//-------------------------------------------------------------------------------------------
void RBBITest::TestUnicodeFiles() {
RuleBasedBreakIterator *bi;
UErrorCode status = U_ZERO_ERROR;
bi = (RuleBasedBreakIterator *)BreakIterator::createCharacterInstance(Locale::getEnglish(), status);
TEST_ASSERT_SUCCESS(status);
if (U_SUCCESS(status)) {
runUnicodeTestData("GraphemeBreakTest.txt", bi);
}
delete bi;
bi = (RuleBasedBreakIterator *)BreakIterator::createWordInstance(Locale::getEnglish(), status);
TEST_ASSERT_SUCCESS(status);
if (U_SUCCESS(status)) {
runUnicodeTestData("WordBreakTest.txt", bi);
}
delete bi;
bi = (RuleBasedBreakIterator *)BreakIterator::createSentenceInstance(Locale::getEnglish(), status);
TEST_ASSERT_SUCCESS(status);
if (U_SUCCESS(status)) {
runUnicodeTestData("SentenceBreakTest.txt", bi);
}
delete bi;
bi = (RuleBasedBreakIterator *)BreakIterator::createLineInstance(Locale::getEnglish(), status);
TEST_ASSERT_SUCCESS(status);
if (U_SUCCESS(status)) {
runUnicodeTestData("LineBreakTest.txt", bi);
}
delete bi;
}
// Check for test cases from the Unicode test data files that are known to fail
// and should be skipped as known issues because ICU does not fully implement
// the Unicode specifications, or because ICU includes tailorings that differ from
// the Unicode standard.
//
// Test cases are identified by the test data sequence, which tends to be more stable
// across Unicode versions than the test file line numbers.
//
// The test case with ticket "10666" is a dummy, included as an example.
UBool RBBITest::testCaseIsKnownIssue(const UnicodeString &testCase, const char *fileName) {
static struct TestCase {
const char *fTicketNum;
const char *fFileName;
const UChar *fString;
} badTestCases[] = {
{"10666", "GraphemeBreakTest.txt", u"\u0020\u0020\u0033"}, // Fake example, for illustration.
// The following tests were originally for
// Issue 8151, move the Finnish tailoring of the line break of hyphens to root.
// However, that ticket has been closed as fixed but these tests still fail, so
// ICU-21097 has been created to investigate and address these remaining issues.
{"21097", "LineBreakTest.txt", u"-#"},
{"21097", "LineBreakTest.txt", u"\u002d\u0308\u0023"},
{"21097", "LineBreakTest.txt", u"\u002d\u00a7"},
{"21097", "LineBreakTest.txt", u"\u002d\u0308\u00a7"},
{"21097", "LineBreakTest.txt", u"\u002d\U00050005"},
{"21097", "LineBreakTest.txt", u"\u002d\u0308\U00050005"},
{"21097", "LineBreakTest.txt", u"\u002d\u0e01"},
{"21097", "LineBreakTest.txt", u"\u002d\u0308\u0e01"},
// The following tests were originally for
// Issue ICU-12017 Improve line break around numbers.
// However, that ticket has been closed as fixed but these tests still fail, so
// ICU-21097 has been created to investigate and address these remaining issues.
{"21097", "LineBreakTest.txt", u"\u002C\u0030"}, // ",0"
{"21097", "LineBreakTest.txt", u"\u002C\u0308\u0030"},
{"21097", "LineBreakTest.txt", u"equals .35 cents"},
{"21097", "LineBreakTest.txt", u"a.2 "},
{"21097", "LineBreakTest.txt", u"a.2 \u0915"},
{"21097", "LineBreakTest.txt", u"a.2 \u672C"},
{"21097", "LineBreakTest.txt", u"a.2\u3000\u672C"},
{"21097", "LineBreakTest.txt", u"a.2\u3000\u307E"},
{"21097", "LineBreakTest.txt", u"a.2\u3000\u0033"},
{"21097", "LineBreakTest.txt", u"A.1 \uBABB"},
{"21097", "LineBreakTest.txt", u"\uBD24\uC5B4\u002E\u0020\u0041\u002E\u0032\u0020\uBCFC"},
{"21097", "LineBreakTest.txt", u"\uBD10\uC694\u002E\u0020\u0041\u002E\u0033\u0020\uBABB"},
{"21097", "LineBreakTest.txt", u"\uC694\u002E\u0020\u0041\u002E\u0034\u0020\uBABB"},
{"21097", "LineBreakTest.txt", u"a.2\u3000\u300C"},
};
for (int n=0; n<UPRV_LENGTHOF(badTestCases); n++) {
const TestCase &badCase = badTestCases[n];
if (!strcmp(fileName, badCase.fFileName) &&
testCase == UnicodeString(badCase.fString)) {
return logKnownIssue(badCase.fTicketNum);
}
}
return FALSE;
}
//--------------------------------------------------------------------------------------------
//
// Run tests from one of the boundary test data files distributed by the Unicode Consortium
//
//-------------------------------------------------------------------------------------------
void RBBITest::runUnicodeTestData(const char *fileName, RuleBasedBreakIterator *bi) {
#if !UCONFIG_NO_REGULAR_EXPRESSIONS
UErrorCode status = U_ZERO_ERROR;
//
// Open and read the test data file, put it into a UnicodeString.
//
const char *testDataDirectory = IntlTest::getSourceTestData(status);
char testFileName[1000];
if (testDataDirectory == NULL || strlen(testDataDirectory) >= sizeof(testFileName)) {
dataerrln("Can't open test data. Path too long.");
return;
}
strcpy(testFileName, testDataDirectory);
strcat(testFileName, fileName);
logln("Opening data file %s\n", fileName);
int len;
UChar *testFile = ReadAndConvertFile(testFileName, len, "UTF-8", status);
if (status != U_FILE_ACCESS_ERROR) {
TEST_ASSERT_SUCCESS(status);
TEST_ASSERT(testFile != NULL);
}
if (U_FAILURE(status) || testFile == NULL) {
return; /* something went wrong, error already output */
}
UnicodeString testFileAsString(TRUE, testFile, len);
//
// Parse the test data file using a regular expression.
// Each kind of token is recognized in its own capture group; what type of item was scanned
// is identified by which group had a match.
//
// Caputure Group # 1 2 3 4 5
// Parses this item: divide x hex digits comment \n unrecognized \n
//
UnicodeString tokenExpr("[ \t]*(?:(\\u00F7)|(\\u00D7)|([0-9a-fA-F]+)|((?:#.*?)?$.)|(.*?$.))", -1, US_INV);
RegexMatcher tokenMatcher(tokenExpr, testFileAsString, UREGEX_MULTILINE | UREGEX_DOTALL, status);
UnicodeString testString;
UVector32 breakPositions(status);
int lineNumber = 1;
TEST_ASSERT_SUCCESS(status);
if (U_FAILURE(status)) {
return;
}
//
// Scan through each test case, building up the string to be broken in testString,
// and the positions that should be boundaries in the breakPositions vector.
//
int spin = 0;
while (tokenMatcher.find()) {
if(tokenMatcher.hitEnd()) {
/* Shouldnt Happen(TM). This means we didn't find the symbols we were looking for.
This occurred when the text file was corrupt (wasn't marked as UTF-8)
and caused an infinite loop here on EBCDIC systems!
*/
fprintf(stderr,"FAIL: hit end of file %s for the %8dth time- corrupt data file?\r", fileName, ++spin);
// return;
}
if (tokenMatcher.start(1, status) >= 0) {
// Scanned a divide sign, indicating a break position in the test data.
if (testString.length()>0) {
breakPositions.addElement(testString.length(), status);
}
}
else if (tokenMatcher.start(2, status) >= 0) {
// Scanned an 'x', meaning no break at this position in the test data
// Nothing to be done here.
}
else if (tokenMatcher.start(3, status) >= 0) {
// Scanned Hex digits. Convert them to binary, append to the character data string.
const UnicodeString &hexNumber = tokenMatcher.group(3, status);
int length = hexNumber.length();
if (length<=8) {
char buf[10];
hexNumber.extract (0, length, buf, sizeof(buf), US_INV);
UChar32 c = (UChar32)strtol(buf, NULL, 16);
if (c<=0x10ffff) {
testString.append(c);
} else {
errln("Error: Unicode Character value out of range. \'%s\', line %d.\n",
fileName, lineNumber);
}
} else {
errln("Syntax Error: Hex Unicode Character value must have no more than 8 digits at \'%s\', line %d.\n",
fileName, lineNumber);
}
}
else if (tokenMatcher.start(4, status) >= 0) {
// Scanned to end of a line, possibly skipping over a comment in the process.
// If the line from the file contained test data, run the test now.
if (testString.length() > 0 && !testCaseIsKnownIssue(testString, fileName)) {
checkUnicodeTestCase(fileName, lineNumber, testString, &breakPositions, bi);
}
// Clear out this test case.
// The string and breakPositions vector will be refilled as the next
// test case is parsed.
testString.remove();
breakPositions.removeAllElements();
lineNumber++;
} else {
// Scanner catchall. Something unrecognized appeared on the line.
char token[16];
UnicodeString uToken = tokenMatcher.group(0, status);
uToken.extract(0, uToken.length(), token, (uint32_t)sizeof(token));
token[sizeof(token)-1] = 0;
errln("Syntax error in test data file \'%s\', line %d. Scanning \"%s\"\n", fileName, lineNumber, token);
// Clean up, in preparation for continuing with the next line.
testString.remove();
breakPositions.removeAllElements();
lineNumber++;
}
TEST_ASSERT_SUCCESS(status);
if (U_FAILURE(status)) {
break;
}
}
delete [] testFile;
#endif // !UCONFIG_NO_REGULAR_EXPRESSIONS
}
//--------------------------------------------------------------------------------------------
//
// checkUnicodeTestCase() Run one test case from one of the Unicode Consortium
// test data files. Do only a simple, forward-only check -
// this test is mostly to check that ICU and the Unicode
// data agree with each other.
//
//--------------------------------------------------------------------------------------------
void RBBITest::checkUnicodeTestCase(const char *testFileName, int lineNumber,
const UnicodeString &testString, // Text data to be broken
UVector32 *breakPositions, // Positions where breaks should be found.
RuleBasedBreakIterator *bi) {
int32_t pos; // Break Position in the test string
int32_t expectedI = 0; // Index of expected break position in the vector of expected results.
int32_t expectedPos; // Expected break position (index into test string)
bi->setText(testString);
pos = bi->first();
pos = bi->next();
while (pos != BreakIterator::DONE) {
if (expectedI >= breakPositions->size()) {
errln("Test file \"%s\", line %d, unexpected break found at position %d",
testFileName, lineNumber, pos);
break;
}
expectedPos = breakPositions->elementAti(expectedI);
if (pos < expectedPos) {
errln("Test file \"%s\", line %d, unexpected break found at position %d",
testFileName, lineNumber, pos);
break;
}
if (pos > expectedPos) {
errln("Test file \"%s\", line %d, failed to find expected break at position %d",
testFileName, lineNumber, expectedPos);
break;
}
pos = bi->next();
expectedI++;
}
if (pos==BreakIterator::DONE && expectedI<breakPositions->size()) {
errln("Test file \"%s\", line %d, failed to find expected break at position %d",
testFileName, lineNumber, breakPositions->elementAti(expectedI));
}
}
#if !UCONFIG_NO_REGULAR_EXPRESSIONS
//---------------------------------------------------------------------------------------
//
// classs RBBIMonkeyKind
//
// Monkey Test for Break Iteration
// Abstract interface class. Concrete derived classes independently
// implement the break rules for different iterator types.
//
// The Monkey Test itself uses doesn't know which type of break iterator it is
// testing, but works purely in terms of the interface defined here.
//
//---------------------------------------------------------------------------------------
class RBBIMonkeyKind {
public:
// Return a UVector of UnicodeSets, representing the character classes used
// for this type of iterator.
virtual UVector *charClasses() = 0;
// Set the test text on which subsequent calls to next() will operate
virtual void setText(const UnicodeString &s) = 0;
// Find the next break postion, starting from the prev break position, or from zero.
// Return -1 after reaching end of string.
virtual int32_t next(int32_t i) = 0;
// Name of each character class, parallel with charClasses. Used for debugging output
// of characters.
virtual std::vector<std::string>& characterClassNames();
void setAppliedRule(int32_t position, const char* value);
std::string getAppliedRule(int32_t position);
virtual ~RBBIMonkeyKind();
UErrorCode deferredStatus;
std::string classNameFromCodepoint(const UChar32 c);
unsigned int maxClassNameSize();
protected:
RBBIMonkeyKind();
std::vector<std::string> classNames;
std::vector<std::string> appliedRules;
// Clear `appliedRules` and fill it with empty strings in the size of test text.
void prepareAppliedRules(int32_t size );
private:
};
RBBIMonkeyKind::RBBIMonkeyKind() {
deferredStatus = U_ZERO_ERROR;
}
RBBIMonkeyKind::~RBBIMonkeyKind() {
}
std::vector<std::string>& RBBIMonkeyKind::characterClassNames() {
return classNames;
}
void RBBIMonkeyKind::prepareAppliedRules(int32_t size) {
// Remove all the information in the `appliedRules`.
appliedRules.clear();
appliedRules.resize(size + 1);
}
void RBBIMonkeyKind::setAppliedRule(int32_t position, const char* value) {
appliedRules[position] = value;
}
std::string RBBIMonkeyKind::getAppliedRule(int32_t position){
return appliedRules[position];
}
std::string RBBIMonkeyKind::classNameFromCodepoint(const UChar32 c) {
// Simply iterate through charClasses to find character's class
for (int aClassNum = 0; aClassNum < charClasses()->size(); aClassNum++) {
UnicodeSet *classSet = (UnicodeSet *)charClasses()->elementAt(aClassNum);
if (classSet->contains(c)) {
return classNames[aClassNum];
}
}
U_ASSERT(FALSE); // This should not happen.
return "bad class name";
}
unsigned int RBBIMonkeyKind::maxClassNameSize() {
unsigned int maxSize = 0;
for (int aClassNum = 0; aClassNum < charClasses()->size(); aClassNum++) {
auto aClassNumSize = static_cast<unsigned int>(classNames[aClassNum].size());
if (aClassNumSize > maxSize) {
maxSize = aClassNumSize;
}
}
return maxSize;
}
//----------------------------------------------------------------------------------------
//
// Random Numbers. Similar to standard lib rand() and srand()
// Not using library to
// 1. Get same results on all platforms.
// 2. Get access to current seed, to more easily reproduce failures.
//
//---------------------------------------------------------------------------------------
static uint32_t m_seed = 1;
static uint32_t m_rand()
{
m_seed = m_seed * 1103515245 + 12345;
return (uint32_t)(m_seed/65536) % 32768;
}
//------------------------------------------------------------------------------------------
//
// class RBBICharMonkey Character (Grapheme Cluster) specific implementation
// of RBBIMonkeyKind.
//
//------------------------------------------------------------------------------------------
class RBBICharMonkey: public RBBIMonkeyKind {
public:
RBBICharMonkey();
virtual ~RBBICharMonkey();
virtual UVector *charClasses();
virtual void setText(const UnicodeString &s);
virtual int32_t next(int32_t i);
private:
UVector *fSets;
UnicodeSet *fCRLFSet;
UnicodeSet *fControlSet;
UnicodeSet *fExtendSet;
UnicodeSet *fZWJSet;
UnicodeSet *fRegionalIndicatorSet;
UnicodeSet *fPrependSet;
UnicodeSet *fSpacingSet;
UnicodeSet *fLSet;
UnicodeSet *fVSet;
UnicodeSet *fTSet;
UnicodeSet *fLVSet;
UnicodeSet *fLVTSet;
UnicodeSet *fHangulSet;
UnicodeSet *fExtendedPictSet;
UnicodeSet *fViramaSet;
UnicodeSet *fLinkingConsonantSet;
UnicodeSet *fExtCccZwjSet;
UnicodeSet *fAnySet;
const UnicodeString *fText;
};
RBBICharMonkey::RBBICharMonkey() {
UErrorCode status = U_ZERO_ERROR;
fText = NULL;
fCRLFSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\r\\n]"), status);
fControlSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[[\\p{Grapheme_Cluster_Break = Control}]]"), status);
fExtendSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[[\\p{Grapheme_Cluster_Break = Extend}]]"), status);
fZWJSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = ZWJ}]"), status);
fRegionalIndicatorSet =
new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = Regional_Indicator}]"), status);
fPrependSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = Prepend}]"), status);
fSpacingSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = SpacingMark}]"), status);
fLSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = L}]"), status);
fVSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = V}]"), status);
fTSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = T}]"), status);
fLVSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = LV}]"), status);
fLVTSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = LVT}]"), status);
fHangulSet = new UnicodeSet();
fHangulSet->addAll(*fLSet);
fHangulSet->addAll(*fVSet);
fHangulSet->addAll(*fTSet);
fHangulSet->addAll(*fLVSet);
fHangulSet->addAll(*fLVTSet);
fExtendedPictSet = new UnicodeSet(u"[:Extended_Pictographic:]", status);
fViramaSet = new UnicodeSet(u"[\\p{Gujr}\\p{sc=Telu}\\p{sc=Mlym}\\p{sc=Orya}\\p{sc=Beng}\\p{sc=Deva}&"
"\\p{Indic_Syllabic_Category=Virama}]", status);
fLinkingConsonantSet = new UnicodeSet(u"[\\p{Gujr}\\p{sc=Telu}\\p{sc=Mlym}\\p{sc=Orya}\\p{sc=Beng}\\p{sc=Deva}&"
"\\p{Indic_Syllabic_Category=Consonant}]", status);
fExtCccZwjSet = new UnicodeSet(u"[[\\p{gcb=Extend}-\\p{ccc=0}] \\p{gcb=ZWJ}]", status);
fAnySet = new UnicodeSet(0, 0x10ffff);
// Create sets of characters, and add the names of the above character sets.
// In each new ICU release, add new names corresponding to the sets above.
fSets = new UVector(status);
// Important: Keep class names the same as the class contents.
fSets->addElement(fCRLFSet, status); classNames.push_back("CRLF");
fSets->addElement(fControlSet, status); classNames.push_back("Control");
fSets->addElement(fExtendSet, status); classNames.push_back("Extended");
fSets->addElement(fRegionalIndicatorSet, status); classNames.push_back("RegionalIndicator");
if (!fPrependSet->isEmpty()) {
fSets->addElement(fPrependSet, status); classNames.push_back("Prepend");
}
fSets->addElement(fSpacingSet, status); classNames.push_back("Spacing");
fSets->addElement(fHangulSet, status); classNames.push_back("Hangul");
fSets->addElement(fZWJSet, status); classNames.push_back("ZWJ");
fSets->addElement(fExtendedPictSet, status); classNames.push_back("ExtendedPict");
fSets->addElement(fViramaSet, status); classNames.push_back("Virama");
fSets->addElement(fLinkingConsonantSet, status); classNames.push_back("LinkingConsonant");
fSets->addElement(fExtCccZwjSet, status); classNames.push_back("ExtCcccZwj");
fSets->addElement(fAnySet, status); classNames.push_back("Any");
if (U_FAILURE(status)) {
deferredStatus = status;
}
}
void RBBICharMonkey::setText(const UnicodeString &s) {
fText = &s;
prepareAppliedRules(s.length());
}
int32_t RBBICharMonkey::next(int32_t prevPos) {
int p0, p1, p2, p3; // Indices of the significant code points around the
// break position being tested. The candidate break
// location is before p2.
int breakPos = -1;
UChar32 c0, c1, c2, c3; // The code points at p0, p1, p2 & p3.
UChar32 cBase; // for (X Extend*) patterns, the X character.
if (U_FAILURE(deferredStatus)) {
return -1;
}
// Previous break at end of string. return DONE.
if (prevPos >= fText->length()) {
return -1;
}
p0 = p1 = p2 = p3 = prevPos;
c3 = fText->char32At(prevPos);
c0 = c1 = c2 = cBase = 0;
(void)p0; // suppress set but not used warning.
(void)c0;
// Loop runs once per "significant" character position in the input text.
for (;;) {
// Move all of the positions forward in the input string.
p0 = p1; c0 = c1;
p1 = p2; c1 = c2;
p2 = p3; c2 = c3;
// Advance p3 by one codepoint
p3 = fText->moveIndex32(p3, 1);
c3 = fText->char32At(p3);
if (p1 == p2) {
// Still warming up the loop. (won't work with zero length strings, but we don't care)
continue;
}
if (p2 == fText->length()) {
setAppliedRule(p2, "End of String");
break;
}
// No Extend or Format characters may appear between the CR and LF,
// which requires the additional check for p2 immediately following p1.
//
if (c1==0x0D && c2==0x0A && p1==(p2-1)) {
setAppliedRule(p2, "GB3 CR x LF");
continue;
}
if (fControlSet->contains(c1) ||
c1 == 0x0D ||
c1 == 0x0A) {
setAppliedRule(p2, "GB4 ( Control | CR | LF ) <break>");
break;
}
if (fControlSet->contains(c2) ||
c2 == 0x0D ||
c2 == 0x0A) {
setAppliedRule(p2, "GB5 <break> ( Control | CR | LF )");
break;
}
if (fLSet->contains(c1) &&
(fLSet->contains(c2) ||
fVSet->contains(c2) ||
fLVSet->contains(c2) ||
fLVTSet->contains(c2))) {
setAppliedRule(p2, "GB6 L x ( L | V | LV | LVT )");
continue;
}
if ((fLVSet->contains(c1) || fVSet->contains(c1)) &&
(fVSet->contains(c2) || fTSet->contains(c2))) {
setAppliedRule(p2, "GB7 ( LV | V ) x ( V | T )");
continue;
}
if ((fLVTSet->contains(c1) || fTSet->contains(c1)) &&
fTSet->contains(c2)) {
setAppliedRule(p2, "GB8 ( LVT | T) x T");
continue;
}
if (fExtendSet->contains(c2) || fZWJSet->contains(c2)) {
if (!fExtendSet->contains(c1)) {
cBase = c1;
}
setAppliedRule(p2, "GB9 x (Extend | ZWJ)");
continue;
}
if (fSpacingSet->contains(c2)) {
setAppliedRule(p2, "GB9a x SpacingMark");
continue;
}
if (fPrependSet->contains(c1)) {
setAppliedRule(p2, "GB9b Prepend x");
continue;
}
// Note: Viramas are also included in the ExtCccZwj class.
if (fLinkingConsonantSet->contains(c2)) {
int pi = p1;
bool sawVirama = false;
while (pi > 0 && fExtCccZwjSet->contains(fText->char32At(pi))) {
if (fViramaSet->contains(fText->char32At(pi))) {
sawVirama = true;
}
pi = fText->moveIndex32(pi, -1);
}
if (sawVirama && fLinkingConsonantSet->contains(fText->char32At(pi))) {
setAppliedRule(p2, "GB9.3 LinkingConsonant ExtCccZwj* Virama ExtCccZwj* × LinkingConsonant");
continue;
}
}
if (fExtendedPictSet->contains(cBase) && fZWJSet->contains(c1) && fExtendedPictSet->contains(c2)) {
setAppliedRule(p2, "GB11 Extended_Pictographic Extend * ZWJ x Extended_Pictographic");
continue;
}
// Note: The first if condition is a little tricky. We only need to force
// a break if there are three or more contiguous RIs. If there are
// only two, a break following will occur via other rules, and will include
// any trailing extend characters, which is needed behavior.
if (fRegionalIndicatorSet->contains(c0) && fRegionalIndicatorSet->contains(c1)
&& fRegionalIndicatorSet->contains(c2)) {
setAppliedRule(p2, "GB12-13 Regional_Indicator x Regional_Indicator");
break;
}
if (fRegionalIndicatorSet->contains(c1) && fRegionalIndicatorSet->contains(c2)) {
setAppliedRule(p2, "GB12-13 Regional_Indicator x Regional_Indicator");
continue;
}
setAppliedRule(p2, "GB999 Any <break> Any");
break;
}
breakPos = p2;
return breakPos;
}
UVector *RBBICharMonkey::charClasses() {
return fSets;
}
RBBICharMonkey::~RBBICharMonkey() {
delete fSets;
delete fCRLFSet;
delete fControlSet;
delete fExtendSet;
delete fRegionalIndicatorSet;
delete fPrependSet;
delete fSpacingSet;
delete fLSet;
delete fVSet;
delete fTSet;
delete fLVSet;
delete fLVTSet;
delete fHangulSet;
delete fAnySet;
delete fZWJSet;
delete fExtendedPictSet;
delete fViramaSet;
delete fLinkingConsonantSet;
delete fExtCccZwjSet;
}
//------------------------------------------------------------------------------------------
//
// class RBBIWordMonkey Word Break specific implementation
// of RBBIMonkeyKind.
//
//------------------------------------------------------------------------------------------
class RBBIWordMonkey: public RBBIMonkeyKind {
public:
RBBIWordMonkey();
virtual ~RBBIWordMonkey();
virtual UVector *charClasses();
virtual void setText(const UnicodeString &s);
virtual int32_t next(int32_t i);
private:
UVector *fSets;
UnicodeSet *fCRSet;
UnicodeSet *fLFSet;
UnicodeSet *fNewlineSet;
UnicodeSet *fRegionalIndicatorSet;
UnicodeSet *fKatakanaSet;
UnicodeSet *fHebrew_LetterSet;
UnicodeSet *fALetterSet;
UnicodeSet *fSingle_QuoteSet;
UnicodeSet *fDouble_QuoteSet;
UnicodeSet *fMidNumLetSet;
UnicodeSet *fMidLetterSet;
UnicodeSet *fMidNumSet;
UnicodeSet *fNumericSet;
UnicodeSet *fFormatSet;
UnicodeSet *fOtherSet;
UnicodeSet *fExtendSet;
UnicodeSet *fExtendNumLetSet;
UnicodeSet *fWSegSpaceSet;
UnicodeSet *fDictionarySet;
UnicodeSet *fZWJSet;
UnicodeSet *fExtendedPictSet;
const UnicodeString *fText;
};
RBBIWordMonkey::RBBIWordMonkey()
{
UErrorCode status = U_ZERO_ERROR;
fSets = new UVector(status);
fCRSet = new UnicodeSet(u"[\\p{Word_Break = CR}]", status);
fLFSet = new UnicodeSet(u"[\\p{Word_Break = LF}]", status);
fNewlineSet = new UnicodeSet(u"[\\p{Word_Break = Newline}]", status);
fKatakanaSet = new UnicodeSet(u"[\\p{Word_Break = Katakana}]", status);
fRegionalIndicatorSet = new UnicodeSet(u"[\\p{Word_Break = Regional_Indicator}]", status);
fHebrew_LetterSet = new UnicodeSet(u"[\\p{Word_Break = Hebrew_Letter}]", status);
fALetterSet = new UnicodeSet(u"[\\p{Word_Break = ALetter}]", status);
fSingle_QuoteSet = new UnicodeSet(u"[\\p{Word_Break = Single_Quote}]", status);
fDouble_QuoteSet = new UnicodeSet(u"[\\p{Word_Break = Double_Quote}]", status);
fMidNumLetSet = new UnicodeSet(u"[\\p{Word_Break = MidNumLet}]", status);
fMidLetterSet = new UnicodeSet(u"[\\p{Word_Break = MidLetter}]", status);
fMidNumSet = new UnicodeSet(u"[\\p{Word_Break = MidNum}]", status);
fNumericSet = new UnicodeSet(u"[\\p{Word_Break = Numeric}]", status);
fFormatSet = new UnicodeSet(u"[\\p{Word_Break = Format}]", status);
fExtendNumLetSet = new UnicodeSet(u"[\\p{Word_Break = ExtendNumLet}]", status);
// There are some sc=Hani characters with WB=Extend.
// The break rules need to pick one or the other because
// Extend overlapping with something else is messy.
// For Unicode 13, we chose to keep U+16FF0 & U+16FF1
// in $Han (for $dictionary) and out of $Extend.
fExtendSet = new UnicodeSet(u"[\\p{Word_Break = Extend}-[:Hani:]]", status);
fWSegSpaceSet = new UnicodeSet(u"[\\p{Word_Break = WSegSpace}]", status);
fZWJSet = new UnicodeSet(u"[\\p{Word_Break = ZWJ}]", status);
fExtendedPictSet = new UnicodeSet(u"[:Extended_Pictographic:]", status);
fDictionarySet = new UnicodeSet(u"[[\\uac00-\\ud7a3][:Han:][:Hiragana:]]", status);
fDictionarySet->addAll(*fKatakanaSet);
fDictionarySet->addAll(UnicodeSet(u"[\\p{LineBreak = Complex_Context}]", status));
fALetterSet->removeAll(*fDictionarySet);
fOtherSet = new UnicodeSet();
if(U_FAILURE(status)) {
IntlTest::gTest->errln("%s:%d %s", __FILE__, __LINE__, u_errorName(status));
deferredStatus = status;
return;
}
fOtherSet->complement();
fOtherSet->removeAll(*fCRSet);
fOtherSet->removeAll(*fLFSet);
fOtherSet->removeAll(*fNewlineSet);
fOtherSet->removeAll(*fKatakanaSet);
fOtherSet->removeAll(*fHebrew_LetterSet);
fOtherSet->removeAll(*fALetterSet);
fOtherSet->removeAll(*fSingle_QuoteSet);
fOtherSet->removeAll(*fDouble_QuoteSet);
fOtherSet->removeAll(*fMidLetterSet);
fOtherSet->removeAll(*fMidNumSet);
fOtherSet->removeAll(*fNumericSet);
fOtherSet->removeAll(*fExtendNumLetSet);
fOtherSet->removeAll(*fWSegSpaceSet);
fOtherSet->removeAll(*fFormatSet);
fOtherSet->removeAll(*fExtendSet);
fOtherSet->removeAll(*fRegionalIndicatorSet);
fOtherSet->removeAll(*fZWJSet);
fOtherSet->removeAll(*fExtendedPictSet);
// Inhibit dictionary characters from being tested at all.
fOtherSet->removeAll(*fDictionarySet);
// Add classes and their names
fSets->addElement(fCRSet, status); classNames.push_back("CR");
fSets->addElement(fLFSet, status); classNames.push_back("LF");
fSets->addElement(fNewlineSet, status); classNames.push_back("Newline");
fSets->addElement(fRegionalIndicatorSet, status); classNames.push_back("RegionalIndicator");
fSets->addElement(fHebrew_LetterSet, status); classNames.push_back("Hebrew");
fSets->addElement(fALetterSet, status); classNames.push_back("ALetter");
fSets->addElement(fSingle_QuoteSet, status); classNames.push_back("Single Quote");
fSets->addElement(fDouble_QuoteSet, status); classNames.push_back("Double Quote");
// Omit Katakana from fSets, which omits Katakana characters
// from the test data. They are all in the dictionary set,
// which this (old, to be retired) monkey test cannot handle.
//fSets->addElement(fKatakanaSet, status);
fSets->addElement(fMidLetterSet, status); classNames.push_back("MidLetter");
fSets->addElement(fMidNumLetSet, status); classNames.push_back("MidNumLet");
fSets->addElement(fMidNumSet, status); classNames.push_back("MidNum");
fSets->addElement(fNumericSet, status); classNames.push_back("Numeric");
fSets->addElement(fFormatSet, status); classNames.push_back("Format");
fSets->addElement(fExtendSet, status); classNames.push_back("Extend");
fSets->addElement(fOtherSet, status); classNames.push_back("Other");
fSets->addElement(fExtendNumLetSet, status); classNames.push_back("ExtendNumLet");
fSets->addElement(fWSegSpaceSet, status); classNames.push_back("WSegSpace");
fSets->addElement(fZWJSet, status); classNames.push_back("ZWJ");
fSets->addElement(fExtendedPictSet, status); classNames.push_back("ExtendedPict");
if (U_FAILURE(status)) {
deferredStatus = status;
}
}
void RBBIWordMonkey::setText(const UnicodeString &s) {
fText = &s;
prepareAppliedRules(s.length());
}
int32_t RBBIWordMonkey::next(int32_t prevPos) {
int p0, p1, p2, p3; // Indices of the significant code points around the
// break position being tested. The candidate break
// location is before p2.
int breakPos = -1;
UChar32 c0, c1, c2, c3; // The code points at p0, p1, p2 & p3.
if (U_FAILURE(deferredStatus)) {
return -1;
}
// Prev break at end of string. return DONE.
if (prevPos >= fText->length()) {
return -1;
}
p0 = p1 = p2 = p3 = prevPos;
c3 = fText->char32At(prevPos);
c0 = c1 = c2 = 0;
(void)p0; // Suppress set but not used warning.
// Loop runs once per "significant" character position in the input text.
for (;;) {
// Move all of the positions forward in the input string.
p0 = p1; c0 = c1;
p1 = p2; c1 = c2;
p2 = p3; c2 = c3;
// Advance p3 by X(Extend | Format)* Rule 4
// But do not advance over Extend & Format following a new line. (Unicode 5.1 change)
do {
p3 = fText->moveIndex32(p3, 1);
c3 = fText->char32At(p3);
if (fCRSet->contains(c2) || fLFSet->contains(c2) || fNewlineSet->contains(c2)) {
break;
}
}
while (fFormatSet->contains(c3) || fExtendSet->contains(c3) || fZWJSet->contains(c3));
if (p1 == p2) {
// Still warming up the loop. (won't work with zero length strings, but we don't care)
continue;
}
if (p2 == fText->length()) {
// Reached end of string. Always a break position.
break;
}
// No Extend or Format characters may appear between the CR and LF,
// which requires the additional check for p2 immediately following p1.
//
if (c1==0x0D && c2==0x0A) {
setAppliedRule(p2, "WB3 CR x LF");
continue;
}
if (fCRSet->contains(c1) || fLFSet->contains(c1) || fNewlineSet->contains(c1)) {
setAppliedRule(p2, "WB3a Break before and after newlines (including CR and LF)");
break;
}
if (fCRSet->contains(c2) || fLFSet->contains(c2) || fNewlineSet->contains(c2)) {
setAppliedRule(p2, "WB3a Break before and after newlines (including CR and LF)");
break;
}
// Not ignoring extend chars, so peek into input text to
// get the potential ZWJ, the character immediately preceding c2.
// Sloppy UChar32 indexing: p2-1 may reference trail half
// but char32At will get the full code point.
if (fZWJSet->contains(fText->char32At(p2 - 1)) && fExtendedPictSet->contains(c2)){
setAppliedRule(p2, "WB3c ZWJ x Extended_Pictographic");
continue;
}
if (fWSegSpaceSet->contains(fText->char32At(p2-1)) && fWSegSpaceSet->contains(c2)) {
setAppliedRule(p2, "WB3d Keep horizontal whitespace together.");
continue;
}
if ((fALetterSet->contains(c1) || fHebrew_LetterSet->contains(c1)) &&
(fALetterSet->contains(c2) || fHebrew_LetterSet->contains(c2))) {
setAppliedRule(p2, "WB4 (ALetter | Hebrew_Letter) x (ALetter | Hebrew_Letter)");
continue;
}
if ( (fALetterSet->contains(c1) || fHebrew_LetterSet->contains(c1)) &&
(fMidLetterSet->contains(c2) || fMidNumLetSet->contains(c2) || fSingle_QuoteSet->contains(c2)) &&
(fALetterSet->contains(c3) || fHebrew_LetterSet->contains(c3))) {
setAppliedRule(p2,
"WB6 (ALetter | Hebrew_Letter) x (MidLetter | MidNumLet | Single_Quote) (ALetter _Letter)");
continue;
}
if ((fALetterSet->contains(c0) || fHebrew_LetterSet->contains(c0)) &&
(fMidLetterSet->contains(c1) || fMidNumLetSet->contains(c1) || fSingle_QuoteSet->contains(c1)) &&
(fALetterSet->contains(c2) || fHebrew_LetterSet->contains(c2))) {
setAppliedRule(p2,
"WB7 (ALetter | Hebrew_Letter) (MidLetter | MidNumLet | Single_Quote) x (ALetter | Hebrew_Letter)");
continue;
}
if (fHebrew_LetterSet->contains(c1) && fSingle_QuoteSet->contains(c2)) {
setAppliedRule(p2, "WB7a Hebrew_Letter x Single_Quote");
continue;
}
if (fHebrew_LetterSet->contains(c1) && fDouble_QuoteSet->contains(c2) && fHebrew_LetterSet->contains(c3)) {
setAppliedRule(p2, "WB7b Hebrew_Letter x Double_Quote Hebrew_Letter");
continue;
}
if (fHebrew_LetterSet->contains(c0) && fDouble_QuoteSet->contains(c1) && fHebrew_LetterSet->contains(c2)) {
setAppliedRule(p2, "WB7c Hebrew_Letter Double_Quote x Hebrew_Letter");
continue;
}
if (fNumericSet->contains(c1) &&
fNumericSet->contains(c2)) {
setAppliedRule(p2, "WB8 Numeric x Numeric");
continue;
}
if ((fALetterSet->contains(c1) || fHebrew_LetterSet->contains(c1)) &&
fNumericSet->contains(c2)) {
setAppliedRule(p2, "WB9 (ALetter | Hebrew_Letter) x Numeric");
continue;
}
if (fNumericSet->contains(c1) &&
(fALetterSet->contains(c2) || fHebrew_LetterSet->contains(c2))) {
setAppliedRule(p2, "WB10 Numeric x (ALetter | Hebrew_Letter)");
continue;
}
if (fNumericSet->contains(c0) &&
(fMidNumSet->contains(c1) || fMidNumLetSet->contains(c1) || fSingle_QuoteSet->contains(c1)) &&
fNumericSet->contains(c2)) {
setAppliedRule(p2, "WB11 Numeric (MidNum | MidNumLet | Single_Quote) x Numeric");
continue;
}
if (fNumericSet->contains(c1) &&
(fMidNumSet->contains(c2) || fMidNumLetSet->contains(c2) || fSingle_QuoteSet->contains(c2)) &&
fNumericSet->contains(c3)) {
setAppliedRule(p2, "WB12 Numeric x (MidNum | MidNumLet | SingleQuote) Numeric");
continue;
}
// Note: matches UAX 29 rules, but doesn't come into play for ICU because
// all Katakana are handled by the dictionary breaker.
if (fKatakanaSet->contains(c1) &&
fKatakanaSet->contains(c2)) {
setAppliedRule(p2, "WB13 Katakana x Katakana");
continue;
}
if ((fALetterSet->contains(c1) || fHebrew_LetterSet->contains(c1) ||fNumericSet->contains(c1) ||
fKatakanaSet->contains(c1) || fExtendNumLetSet->contains(c1)) &&
fExtendNumLetSet->contains(c2)) {
setAppliedRule(p2,
"WB13a (ALetter | Hebrew_Letter | Numeric | KataKana | ExtendNumLet) x ExtendNumLet");
continue;
}
if (fExtendNumLetSet->contains(c1) &&
(fALetterSet->contains(c2) || fHebrew_LetterSet->contains(c2) ||
fNumericSet->contains(c2) || fKatakanaSet->contains(c2))) {
setAppliedRule(p2, "WB13b ExtendNumLet x (ALetter | Hebrew_Letter | Numeric | Katakana)");
continue;
}
if (fRegionalIndicatorSet->contains(c0) && fRegionalIndicatorSet->contains(c1)) {
setAppliedRule(p2, "WB15 - WB17 Group pairs of Regional Indicators.");
break;
}
if (fRegionalIndicatorSet->contains(c1) && fRegionalIndicatorSet->contains(c2)) {
setAppliedRule(p2, "WB15 - WB17 Group pairs of Regional Indicators.");
continue;
}
setAppliedRule(p2, "WB999");
break;
}
breakPos = p2;
return breakPos;
}
UVector *RBBIWordMonkey::charClasses() {
return fSets;
}
RBBIWordMonkey::~RBBIWordMonkey() {
delete fSets;
delete fCRSet;
delete fLFSet;
delete fNewlineSet;
delete fKatakanaSet;
delete fHebrew_LetterSet;
delete fALetterSet;
delete fSingle_QuoteSet;
delete fDouble_QuoteSet;
delete fMidNumLetSet;
delete fMidLetterSet;
delete fMidNumSet;
delete fNumericSet;
delete fFormatSet;
delete fExtendSet;
delete fExtendNumLetSet;
delete fWSegSpaceSet;
delete fRegionalIndicatorSet;
delete fDictionarySet;
delete fOtherSet;
delete fZWJSet;
delete fExtendedPictSet;
}
//------------------------------------------------------------------------------------------
//
// class RBBISentMonkey Sentence Break specific implementation
// of RBBIMonkeyKind.
//
//------------------------------------------------------------------------------------------
class RBBISentMonkey: public RBBIMonkeyKind {
public:
RBBISentMonkey();
virtual ~RBBISentMonkey();
virtual UVector *charClasses();
virtual void setText(const UnicodeString &s);
virtual int32_t next(int32_t i);
private:
int moveBack(int posFrom);
int moveForward(int posFrom);
UChar32 cAt(int pos);
UVector *fSets;
UnicodeSet *fSepSet;
UnicodeSet *fFormatSet;
UnicodeSet *fSpSet;
UnicodeSet *fLowerSet;
UnicodeSet *fUpperSet;
UnicodeSet *fOLetterSet;
UnicodeSet *fNumericSet;
UnicodeSet *fATermSet;
UnicodeSet *fSContinueSet;
UnicodeSet *fSTermSet;
UnicodeSet *fCloseSet;
UnicodeSet *fOtherSet;
UnicodeSet *fExtendSet;
const UnicodeString *fText;
};
RBBISentMonkey::RBBISentMonkey()
{
UErrorCode status = U_ZERO_ERROR;
fSets = new UVector(status);
// Separator Set Note: Beginning with Unicode 5.1, CR and LF were removed from the separator
// set and made into character classes of their own. For the monkey impl,
// they remain in SEP, since Sep always appears with CR and LF in the rules.
fSepSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Sep} \\u000a \\u000d]"), status);
fFormatSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Format}]"), status);
fSpSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Sp}]"), status);
fLowerSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Lower}]"), status);
fUpperSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Upper}]"), status);
fOLetterSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = OLetter}]"), status);
fNumericSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Numeric}]"), status);
fATermSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = ATerm}]"), status);
fSContinueSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = SContinue}]"), status);
fSTermSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = STerm}]"), status);
fCloseSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Close}]"), status);
fExtendSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Extend}]"), status);
fOtherSet = new UnicodeSet();
if(U_FAILURE(status)) {
deferredStatus = status;
return;
}
fOtherSet->complement();
fOtherSet->removeAll(*fSepSet);
fOtherSet->removeAll(*fFormatSet);
fOtherSet->removeAll(*fSpSet);
fOtherSet->removeAll(*fLowerSet);
fOtherSet->removeAll(*fUpperSet);
fOtherSet->removeAll(*fOLetterSet);
fOtherSet->removeAll(*fNumericSet);
fOtherSet->removeAll(*fATermSet);
fOtherSet->removeAll(*fSContinueSet);
fOtherSet->removeAll(*fSTermSet);
fOtherSet->removeAll(*fCloseSet);
fOtherSet->removeAll(*fExtendSet);
fSets->addElement(fSepSet, status); classNames.push_back("Sep");
fSets->addElement(fFormatSet, status); classNames.push_back("Format");
fSets->addElement(fSpSet, status); classNames.push_back("Sp");
fSets->addElement(fLowerSet, status); classNames.push_back("Lower");
fSets->addElement(fUpperSet, status); classNames.push_back("Upper");
fSets->addElement(fOLetterSet, status); classNames.push_back("OLetter");
fSets->addElement(fNumericSet, status); classNames.push_back("Numeric");
fSets->addElement(fATermSet, status); classNames.push_back("ATerm");
fSets->addElement(fSContinueSet, status); classNames.push_back("SContinue");
fSets->addElement(fSTermSet, status); classNames.push_back("STerm");
fSets->addElement(fCloseSet, status); classNames.push_back("Close");
fSets->addElement(fOtherSet, status); classNames.push_back("Other");
fSets->addElement(fExtendSet, status); classNames.push_back("Extend");
if (U_FAILURE(status)) {
deferredStatus = status;
}
}
void RBBISentMonkey::setText(const UnicodeString &s) {
fText = &s;
prepareAppliedRules(s.length());
}
UVector *RBBISentMonkey::charClasses() {
return fSets;
}
// moveBack() Find the "significant" code point preceding the index i.
// Skips over ($Extend | $Format)* .
//
int RBBISentMonkey::moveBack(int i) {
if (i <= 0) {
return -1;
}
UChar32 c;
int32_t j = i;
do {
j = fText->moveIndex32(j, -1);
c = fText->char32At(j);
}
while (j>0 &&(fFormatSet->contains(c) || fExtendSet->contains(c)));
return j;
}
int RBBISentMonkey::moveForward(int i) {
if (i>=fText->length()) {
return fText->length();
}
UChar32 c;
int32_t j = i;
do {
j = fText->moveIndex32(j, 1);
c = cAt(j);
}
while (fFormatSet->contains(c) || fExtendSet->contains(c));
return j;
}
UChar32 RBBISentMonkey::cAt(int pos) {
if (pos<0 || pos>=fText->length()) {
return -1;
} else {
return fText->char32At(pos);
}
}
int32_t RBBISentMonkey::next(int32_t prevPos) {
int p0, p1, p2, p3; // Indices of the significant code points around the
// break position being tested. The candidate break
// location is before p2.
int breakPos = -1;
UChar32 c0, c1, c2, c3; // The code points at p0, p1, p2 & p3.
UChar32 c;
if (U_FAILURE(deferredStatus)) {
return -1;
}
// Prev break at end of string. return DONE.
if (prevPos >= fText->length()) {
return -1;
}
p0 = p1 = p2 = p3 = prevPos;
c3 = fText->char32At(prevPos);
c0 = c1 = c2 = 0;
(void)p0; // Suppress set but not used warning.
// Loop runs once per "significant" character position in the input text.
for (;;) {
// Move all of the positions forward in the input string.
p0 = p1; c0 = c1;
p1 = p2; c1 = c2;
p2 = p3; c2 = c3;
// Advance p3 by X(Extend | Format)* Rule 4
p3 = moveForward(p3);
c3 = cAt(p3);
if (c1==0x0d && c2==0x0a && p2==(p1+1)) {
setAppliedRule(p2, "SB3 CR x LF");
continue;
}
if (fSepSet->contains(c1)) {
p2 = p1+1; // Separators don't combine with Extend or Format.
setAppliedRule(p2, "SB4 Sep <break>");
break;
}
if (p2 >= fText->length()) {
// Reached end of string. Always a break position.
setAppliedRule(p2, "SB4 Sep <break>");
break;
}
if (p2 == prevPos) {
// Still warming up the loop. (won't work with zero length strings, but we don't care)
setAppliedRule(p2, "SB4 Sep <break>");
continue;
}
if (fATermSet->contains(c1) && fNumericSet->contains(c2)) {
setAppliedRule(p2, "SB6 ATerm x Numeric");
continue;
}
if ((fUpperSet->contains(c0) || fLowerSet->contains(c0)) &&
fATermSet->contains(c1) && fUpperSet->contains(c2)) {
setAppliedRule(p2, "SB7 (Upper | Lower) ATerm x Uppper");
continue;
}
// Note: STerm | ATerm are added to the negated part of the expression by a
// note to the Unicode 5.0 documents.
int p8 = p1;
while (fSpSet->contains(cAt(p8))) {
p8 = moveBack(p8);
}
while (fCloseSet->contains(cAt(p8))) {
p8 = moveBack(p8);
}
if (fATermSet->contains(cAt(p8))) {
p8=p2;
for (;;) {
c = cAt(p8);
if (c==-1 || fOLetterSet->contains(c) || fUpperSet->contains(c) ||
fLowerSet->contains(c) || fSepSet->contains(c) ||
fATermSet->contains(c) || fSTermSet->contains(c)) {
setAppliedRule(p2,
"SB8 ATerm Close* Sp* x (not (OLettter | Upper | Lower | Sep | STerm | ATerm))* ");
break;
}
p8 = moveForward(p8);
}
if (fLowerSet->contains(cAt(p8))) {
setAppliedRule(p2,
"SB8 ATerm Close* Sp* x (not (OLettter | Upper | Lower | Sep | STerm | ATerm))* ");
continue;
}
}
if (fSContinueSet->contains(c2) || fSTermSet->contains(c2) || fATermSet->contains(c2)) {
p8 = p1;
while (fSpSet->contains(cAt(p8))) {
p8 = moveBack(p8);
}
while (fCloseSet->contains(cAt(p8))) {
p8 = moveBack(p8);
}
c = cAt(p8);
if (fSTermSet->contains(c) || fATermSet->contains(c)) {
setAppliedRule(p2, "SB8a (STerm | ATerm) Close* Sp* x (SContinue | STerm | ATerm)");
continue;
}
}
int p9 = p1;
while (fCloseSet->contains(cAt(p9))) {
p9 = moveBack(p9);
}
c = cAt(p9);
if ((fSTermSet->contains(c) || fATermSet->contains(c))) {
if (fCloseSet->contains(c2) || fSpSet->contains(c2) || fSepSet->contains(c2)) {
setAppliedRule(p2, "SB9 (STerm | ATerm) Close* x (Close | Sp | Sep | CR | LF)");
continue;
}
}
int p10 = p1;
while (fSpSet->contains(cAt(p10))) {
p10 = moveBack(p10);
}
while (fCloseSet->contains(cAt(p10))) {
p10 = moveBack(p10);
}
if (fSTermSet->contains(cAt(p10)) || fATermSet->contains(cAt(p10))) {
if (fSpSet->contains(c2) || fSepSet->contains(c2)) {
setAppliedRule(p2, "SB10 (Sterm | ATerm) Close* Sp* x (Sp | Sep | CR | LF)");
continue;
}
}
int p11 = p1;
if (fSepSet->contains(cAt(p11))) {
p11 = moveBack(p11);
}
while (fSpSet->contains(cAt(p11))) {
p11 = moveBack(p11);
}
while (fCloseSet->contains(cAt(p11))) {
p11 = moveBack(p11);
}
if (fSTermSet->contains(cAt(p11)) || fATermSet->contains(cAt(p11))) {
setAppliedRule(p2, "SB11 (STerm | ATerm) Close* Sp* (Sep | CR | LF)? <break>");
break;
}
setAppliedRule(p2, "SB12 Any x Any");
continue;
}
breakPos = p2;
return breakPos;
}
RBBISentMonkey::~RBBISentMonkey() {
delete fSets;
delete fSepSet;
delete fFormatSet;
delete fSpSet;
delete fLowerSet;
delete fUpperSet;
delete fOLetterSet;
delete fNumericSet;
delete fATermSet;
delete fSContinueSet;
delete fSTermSet;
delete fCloseSet;
delete fOtherSet;
delete fExtendSet;
}
//-------------------------------------------------------------------------------------------
//
// RBBILineMonkey
//
//-------------------------------------------------------------------------------------------
class RBBILineMonkey: public RBBIMonkeyKind {
public:
RBBILineMonkey();
virtual ~RBBILineMonkey();
virtual UVector *charClasses();
virtual void setText(const UnicodeString &s);
virtual int32_t next(int32_t i);
virtual void rule9Adjust(int32_t pos, UChar32 *posChar, int32_t *nextPos, UChar32 *nextChar);
private:
UVector *fSets;
UnicodeSet *fBK;
UnicodeSet *fCR;
UnicodeSet *fLF;
UnicodeSet *fCM;
UnicodeSet *fNL;
UnicodeSet *fSG;
UnicodeSet *fWJ;
UnicodeSet *fZW;
UnicodeSet *fGL;
UnicodeSet *fCB;
UnicodeSet *fSP;
UnicodeSet *fB2;
UnicodeSet *fBA;
UnicodeSet *fBB;
UnicodeSet *fHH;
UnicodeSet *fHY;
UnicodeSet *fH2;
UnicodeSet *fH3;
UnicodeSet *fCL;
UnicodeSet *fCP;
UnicodeSet *fEX;
UnicodeSet *fIN;
UnicodeSet *fJL;
UnicodeSet *fJV;
UnicodeSet *fJT;
UnicodeSet *fNS;
UnicodeSet *fOP;
UnicodeSet *fQU;
UnicodeSet *fIS;
UnicodeSet *fNU;
UnicodeSet *fPO;
UnicodeSet *fPR;
UnicodeSet *fSY;
UnicodeSet *fAI;
UnicodeSet *fAL;
UnicodeSet *fCJ;
UnicodeSet *fHL;
UnicodeSet *fID;
UnicodeSet *fRI;
UnicodeSet *fXX;
UnicodeSet *fEB;
UnicodeSet *fEM;
UnicodeSet *fZWJ;
UnicodeSet *fOP30;
UnicodeSet *fCP30;
BreakIterator *fCharBI;
const UnicodeString *fText;
RegexMatcher *fNumberMatcher;
};
RBBILineMonkey::RBBILineMonkey() :
RBBIMonkeyKind(),
fSets(NULL),
fCharBI(NULL),
fText(NULL),
fNumberMatcher(NULL)
{
if (U_FAILURE(deferredStatus)) {
return;
}
UErrorCode status = U_ZERO_ERROR;
fSets = new UVector(status);
fBK = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_Break=BK}]"), status);
fCR = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=CR}]"), status);
fLF = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=LF}]"), status);
fCM = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=CM}]"), status);
fNL = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=NL}]"), status);
fWJ = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=WJ}]"), status);
fZW = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=ZW}]"), status);
fGL = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=GL}]"), status);
fCB = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=CB}]"), status);
fSP = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=SP}]"), status);
fB2 = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=B2}]"), status);
fBA = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=BA}]"), status);
fBB = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=BB}]"), status);
fHH = new UnicodeSet();
fHY = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=HY}]"), status);
fH2 = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=H2}]"), status);
fH3 = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=H3}]"), status);
fCL = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=CL}]"), status);
fCP = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=CP}]"), status);
fEX = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=EX}]"), status);
fIN = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=IN}]"), status);
fJL = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=JL}]"), status);
fJV = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=JV}]"), status);
fJT = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=JT}]"), status);
fNS = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=NS}]"), status);
fOP = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=OP}]"), status);
fQU = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=QU}]"), status);
fIS = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=IS}]"), status);
fNU = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=NU}]"), status);
fPO = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=PO}]"), status);
fPR = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=PR}]"), status);
fSY = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=SY}]"), status);
fAI = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=AI}]"), status);
fAL = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=AL}]"), status);
fCJ = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=CJ}]"), status);
fHL = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=HL}]"), status);
fID = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=ID}]"), status);
fRI = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=RI}]"), status);
fSG = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\ud800-\\udfff]"), status);
fXX = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=XX}]"), status);
fEB = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=EB}]"), status);
fEM = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=EM}]"), status);
fZWJ = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=ZWJ}]"), status);
fOP30 = new UnicodeSet(u"[\\p{Line_break=OP}-[\\p{ea=F}\\p{ea=W}\\p{ea=H}]]", status);
fCP30 = new UnicodeSet(u"[\\p{Line_break=CP}-[\\p{ea=F}\\p{ea=W}\\p{ea=H}]]", status);
if (U_FAILURE(status)) {
deferredStatus = status;
return;
}
fAL->addAll(*fXX); // Default behavior for XX is identical to AL
fAL->addAll(*fAI); // Default behavior for AI is identical to AL
fAL->addAll(*fSG); // Default behavior for SG is identical to AL.
fNS->addAll(*fCJ); // Default behavior for CJ is identical to NS.
fCM->addAll(*fZWJ); // ZWJ behaves as a CM.
fHH->add(u'\u2010'); // Hyphen, '‐'
// Sets and names.
fSets->addElement(fBK, status); classNames.push_back("fBK");
fSets->addElement(fCR, status); classNames.push_back("fCR");
fSets->addElement(fLF, status); classNames.push_back("fLF");
fSets->addElement(fCM, status); classNames.push_back("fCM");
fSets->addElement(fNL, status); classNames.push_back("fNL");
fSets->addElement(fWJ, status); classNames.push_back("fWJ");
fSets->addElement(fZW, status); classNames.push_back("fZW");
fSets->addElement(fGL, status); classNames.push_back("fGL");
fSets