| // © 2016 and later: Unicode, Inc. and others. |
| // License & terms of use: http://www.unicode.org/copyright.html |
| /* |
| *************************************************************************** |
| * Copyright (C) 1999-2016 International Business Machines Corporation |
| * and others. All rights reserved. |
| *************************************************************************** |
| */ |
| // |
| // file: rbbi.cpp Contains the implementation of the rule based break iterator |
| // runtime engine and the API implementation for |
| // class RuleBasedBreakIterator |
| // |
| |
| #include "utypeinfo.h" // for 'typeid' to work |
| |
| #include "unicode/utypes.h" |
| |
| #if !UCONFIG_NO_BREAK_ITERATION |
| |
| #include <cinttypes> |
| |
| #include "unicode/rbbi.h" |
| #include "unicode/schriter.h" |
| #include "unicode/uchriter.h" |
| #include "unicode/uclean.h" |
| #include "unicode/udata.h" |
| |
| #include "brkeng.h" |
| #include "ucln_cmn.h" |
| #include "cmemory.h" |
| #include "cstring.h" |
| #include "localsvc.h" |
| #include "rbbidata.h" |
| #include "rbbi_cache.h" |
| #include "rbbirb.h" |
| #include "uassert.h" |
| #include "umutex.h" |
| #include "uvectr32.h" |
| |
| #ifdef RBBI_DEBUG |
| static UBool gTrace = FALSE; |
| #endif |
| |
| U_NAMESPACE_BEGIN |
| |
| // The state number of the starting state |
| constexpr int32_t START_STATE = 1; |
| |
| // The state-transition value indicating "stop" |
| constexpr int32_t STOP_STATE = 0; |
| |
| |
| UOBJECT_DEFINE_RTTI_IMPLEMENTATION(RuleBasedBreakIterator) |
| |
| |
| //======================================================================= |
| // constructors |
| //======================================================================= |
| |
| /** |
| * Constructs a RuleBasedBreakIterator that uses the already-created |
| * tables object that is passed in as a parameter. |
| */ |
| RuleBasedBreakIterator::RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode &status) |
| : fSCharIter(UnicodeString()) |
| { |
| init(status); |
| fData = new RBBIDataWrapper(data, status); // status checked in constructor |
| if (U_FAILURE(status)) {return;} |
| if(fData == nullptr) { |
| status = U_MEMORY_ALLOCATION_ERROR; |
| return; |
| } |
| if (fData->fForwardTable->fLookAheadResultsSize > 0) { |
| fLookAheadMatches = static_cast<int32_t *>( |
| uprv_malloc(fData->fForwardTable->fLookAheadResultsSize * sizeof(int32_t))); |
| if (fLookAheadMatches == nullptr) { |
| status = U_MEMORY_ALLOCATION_ERROR; |
| return; |
| } |
| } |
| } |
| |
| // |
| // Construct from precompiled binary rules (tables). This constructor is public API, |
| // taking the rules as a (const uint8_t *) to match the type produced by getBinaryRules(). |
| // |
| RuleBasedBreakIterator::RuleBasedBreakIterator(const uint8_t *compiledRules, |
| uint32_t ruleLength, |
| UErrorCode &status) |
| : fSCharIter(UnicodeString()) |
| { |
| init(status); |
| if (U_FAILURE(status)) { |
| return; |
| } |
| if (compiledRules == NULL || ruleLength < sizeof(RBBIDataHeader)) { |
| status = U_ILLEGAL_ARGUMENT_ERROR; |
| return; |
| } |
| const RBBIDataHeader *data = (const RBBIDataHeader *)compiledRules; |
| if (data->fLength > ruleLength) { |
| status = U_ILLEGAL_ARGUMENT_ERROR; |
| return; |
| } |
| fData = new RBBIDataWrapper(data, RBBIDataWrapper::kDontAdopt, status); |
| if (U_FAILURE(status)) {return;} |
| if(fData == nullptr) { |
| status = U_MEMORY_ALLOCATION_ERROR; |
| return; |
| } |
| if (fData->fForwardTable->fLookAheadResultsSize > 0) { |
| fLookAheadMatches = static_cast<int32_t *>( |
| uprv_malloc(fData->fForwardTable->fLookAheadResultsSize * sizeof(int32_t))); |
| if (fLookAheadMatches == nullptr) { |
| status = U_MEMORY_ALLOCATION_ERROR; |
| return; |
| } |
| } |
| } |
| |
| |
| //------------------------------------------------------------------------------- |
| // |
| // Constructor from a UDataMemory handle to precompiled break rules |
| // stored in an ICU data file. |
| // |
| //------------------------------------------------------------------------------- |
| RuleBasedBreakIterator::RuleBasedBreakIterator(UDataMemory* udm, UErrorCode &status) |
| : fSCharIter(UnicodeString()) |
| { |
| init(status); |
| fData = new RBBIDataWrapper(udm, status); // status checked in constructor |
| if (U_FAILURE(status)) {return;} |
| if(fData == nullptr) { |
| status = U_MEMORY_ALLOCATION_ERROR; |
| return; |
| } |
| if (fData->fForwardTable->fLookAheadResultsSize > 0) { |
| fLookAheadMatches = static_cast<int32_t *>( |
| uprv_malloc(fData->fForwardTable->fLookAheadResultsSize * sizeof(int32_t))); |
| if (fLookAheadMatches == nullptr) { |
| status = U_MEMORY_ALLOCATION_ERROR; |
| return; |
| } |
| } |
| } |
| |
| |
| |
| //------------------------------------------------------------------------------- |
| // |
| // Constructor from a set of rules supplied as a string. |
| // |
| //------------------------------------------------------------------------------- |
| RuleBasedBreakIterator::RuleBasedBreakIterator( const UnicodeString &rules, |
| UParseError &parseError, |
| UErrorCode &status) |
| : fSCharIter(UnicodeString()) |
| { |
| init(status); |
| if (U_FAILURE(status)) {return;} |
| RuleBasedBreakIterator *bi = (RuleBasedBreakIterator *) |
| RBBIRuleBuilder::createRuleBasedBreakIterator(rules, &parseError, status); |
| // Note: This is a bit awkward. The RBBI ruleBuilder has a factory method that |
| // creates and returns a complete RBBI. From here, in a constructor, we |
| // can't just return the object created by the builder factory, hence |
| // the assignment of the factory created object to "this". |
| if (U_SUCCESS(status)) { |
| *this = *bi; |
| delete bi; |
| } |
| } |
| |
| |
| //------------------------------------------------------------------------------- |
| // |
| // Default Constructor. Create an empty shell that can be set up later. |
| // Used when creating a RuleBasedBreakIterator from a set |
| // of rules. |
| //------------------------------------------------------------------------------- |
| RuleBasedBreakIterator::RuleBasedBreakIterator() |
| : fSCharIter(UnicodeString()) |
| { |
| UErrorCode status = U_ZERO_ERROR; |
| init(status); |
| } |
| |
| |
| //------------------------------------------------------------------------------- |
| // |
| // Copy constructor. Will produce a break iterator with the same behavior, |
| // and which iterates over the same text, as the one passed in. |
| // |
| //------------------------------------------------------------------------------- |
| RuleBasedBreakIterator::RuleBasedBreakIterator(const RuleBasedBreakIterator& other) |
| : BreakIterator(other), |
| fSCharIter(UnicodeString()) |
| { |
| UErrorCode status = U_ZERO_ERROR; |
| this->init(status); |
| *this = other; |
| } |
| |
| |
| /** |
| * Destructor |
| */ |
| RuleBasedBreakIterator::~RuleBasedBreakIterator() { |
| if (fCharIter != &fSCharIter) { |
| // fCharIter was adopted from the outside. |
| delete fCharIter; |
| } |
| fCharIter = nullptr; |
| |
| utext_close(&fText); |
| |
| if (fData != nullptr) { |
| fData->removeReference(); |
| fData = nullptr; |
| } |
| delete fBreakCache; |
| fBreakCache = nullptr; |
| |
| delete fDictionaryCache; |
| fDictionaryCache = nullptr; |
| |
| delete fLanguageBreakEngines; |
| fLanguageBreakEngines = nullptr; |
| |
| delete fUnhandledBreakEngine; |
| fUnhandledBreakEngine = nullptr; |
| |
| uprv_free(fLookAheadMatches); |
| fLookAheadMatches = nullptr; |
| } |
| |
| /** |
| * Assignment operator. Sets this iterator to have the same behavior, |
| * and iterate over the same text, as the one passed in. |
| * TODO: needs better handling of memory allocation errors. |
| */ |
| RuleBasedBreakIterator& |
| RuleBasedBreakIterator::operator=(const RuleBasedBreakIterator& that) { |
| if (this == &that) { |
| return *this; |
| } |
| BreakIterator::operator=(that); |
| |
| if (fLanguageBreakEngines != NULL) { |
| delete fLanguageBreakEngines; |
| fLanguageBreakEngines = NULL; // Just rebuild for now |
| } |
| // TODO: clone fLanguageBreakEngines from "that" |
| UErrorCode status = U_ZERO_ERROR; |
| utext_clone(&fText, &that.fText, FALSE, TRUE, &status); |
| |
| if (fCharIter != &fSCharIter) { |
| delete fCharIter; |
| } |
| fCharIter = &fSCharIter; |
| |
| if (that.fCharIter != NULL && that.fCharIter != &that.fSCharIter) { |
| // This is a little bit tricky - it will intially appear that |
| // this->fCharIter is adopted, even if that->fCharIter was |
| // not adopted. That's ok. |
| fCharIter = that.fCharIter->clone(); |
| } |
| fSCharIter = that.fSCharIter; |
| if (fCharIter == NULL) { |
| fCharIter = &fSCharIter; |
| } |
| |
| if (fData != NULL) { |
| fData->removeReference(); |
| fData = NULL; |
| } |
| if (that.fData != NULL) { |
| fData = that.fData->addReference(); |
| } |
| |
| uprv_free(fLookAheadMatches); |
| fLookAheadMatches = nullptr; |
| if (fData && fData->fForwardTable->fLookAheadResultsSize > 0) { |
| fLookAheadMatches = static_cast<int32_t *>( |
| uprv_malloc(fData->fForwardTable->fLookAheadResultsSize * sizeof(int32_t))); |
| } |
| |
| |
| fPosition = that.fPosition; |
| fRuleStatusIndex = that.fRuleStatusIndex; |
| fDone = that.fDone; |
| |
| // TODO: both the dictionary and the main cache need to be copied. |
| // Current position could be within a dictionary range. Trying to continue |
| // the iteration without the caches present would go to the rules, with |
| // the assumption that the current position is on a rule boundary. |
| fBreakCache->reset(fPosition, fRuleStatusIndex); |
| fDictionaryCache->reset(); |
| |
| return *this; |
| } |
| |
| |
| |
| //----------------------------------------------------------------------------- |
| // |
| // init() Shared initialization routine. Used by all the constructors. |
| // Initializes all fields, leaving the object in a consistent state. |
| // |
| //----------------------------------------------------------------------------- |
| void RuleBasedBreakIterator::init(UErrorCode &status) { |
| fCharIter = nullptr; |
| fData = nullptr; |
| fPosition = 0; |
| fRuleStatusIndex = 0; |
| fDone = false; |
| fDictionaryCharCount = 0; |
| fLanguageBreakEngines = nullptr; |
| fUnhandledBreakEngine = nullptr; |
| fBreakCache = nullptr; |
| fDictionaryCache = nullptr; |
| fLookAheadMatches = nullptr; |
| |
| // Note: IBM xlC is unable to assign or initialize member fText from UTEXT_INITIALIZER. |
| // fText = UTEXT_INITIALIZER; |
| static const UText initializedUText = UTEXT_INITIALIZER; |
| uprv_memcpy(&fText, &initializedUText, sizeof(UText)); |
| |
| if (U_FAILURE(status)) { |
| return; |
| } |
| |
| utext_openUChars(&fText, NULL, 0, &status); |
| fDictionaryCache = new DictionaryCache(this, status); |
| fBreakCache = new BreakCache(this, status); |
| if (U_SUCCESS(status) && (fDictionaryCache == NULL || fBreakCache == NULL)) { |
| status = U_MEMORY_ALLOCATION_ERROR; |
| } |
| |
| #ifdef RBBI_DEBUG |
| static UBool debugInitDone = FALSE; |
| if (debugInitDone == FALSE) { |
| char *debugEnv = getenv("U_RBBIDEBUG"); |
| if (debugEnv && uprv_strstr(debugEnv, "trace")) { |
| gTrace = TRUE; |
| } |
| debugInitDone = TRUE; |
| } |
| #endif |
| } |
| |
| |
| |
| //----------------------------------------------------------------------------- |
| // |
| // clone - Returns a newly-constructed RuleBasedBreakIterator with the same |
| // behavior, and iterating over the same text, as this one. |
| // Virtual function: does the right thing with subclasses. |
| // |
| //----------------------------------------------------------------------------- |
| RuleBasedBreakIterator* |
| RuleBasedBreakIterator::clone() const { |
| return new RuleBasedBreakIterator(*this); |
| } |
| |
| /** |
| * Equality operator. Returns TRUE if both BreakIterators are of the |
| * same class, have the same behavior, and iterate over the same text. |
| */ |
| UBool |
| RuleBasedBreakIterator::operator==(const BreakIterator& that) const { |
| if (typeid(*this) != typeid(that)) { |
| return FALSE; |
| } |
| if (this == &that) { |
| return TRUE; |
| } |
| |
| // The base class BreakIterator carries no state that participates in equality, |
| // and does not implement an equality function that would otherwise be |
| // checked at this point. |
| |
| const RuleBasedBreakIterator& that2 = (const RuleBasedBreakIterator&) that; |
| |
| if (!utext_equals(&fText, &that2.fText)) { |
| // The two break iterators are operating on different text, |
| // or have a different iteration position. |
| // Note that fText's position is always the same as the break iterator's position. |
| return FALSE; |
| } |
| |
| if (!(fPosition == that2.fPosition && |
| fRuleStatusIndex == that2.fRuleStatusIndex && |
| fDone == that2.fDone)) { |
| return FALSE; |
| } |
| |
| if (that2.fData == fData || |
| (fData != NULL && that2.fData != NULL && *that2.fData == *fData)) { |
| // The two break iterators are using the same rules. |
| return TRUE; |
| } |
| return FALSE; |
| } |
| |
| /** |
| * Compute a hash code for this BreakIterator |
| * @return A hash code |
| */ |
| int32_t |
| RuleBasedBreakIterator::hashCode(void) const { |
| int32_t hash = 0; |
| if (fData != NULL) { |
| hash = fData->hashCode(); |
| } |
| return hash; |
| } |
| |
| |
| void RuleBasedBreakIterator::setText(UText *ut, UErrorCode &status) { |
| if (U_FAILURE(status)) { |
| return; |
| } |
| fBreakCache->reset(); |
| fDictionaryCache->reset(); |
| utext_clone(&fText, ut, FALSE, TRUE, &status); |
| |
| // Set up a dummy CharacterIterator to be returned if anyone |
| // calls getText(). With input from UText, there is no reasonable |
| // way to return a characterIterator over the actual input text. |
| // Return one over an empty string instead - this is the closest |
| // we can come to signaling a failure. |
| // (GetText() is obsolete, this failure is sort of OK) |
| fSCharIter.setText(UnicodeString()); |
| |
| if (fCharIter != &fSCharIter) { |
| // existing fCharIter was adopted from the outside. Delete it now. |
| delete fCharIter; |
| } |
| fCharIter = &fSCharIter; |
| |
| this->first(); |
| } |
| |
| |
| UText *RuleBasedBreakIterator::getUText(UText *fillIn, UErrorCode &status) const { |
| UText *result = utext_clone(fillIn, &fText, FALSE, TRUE, &status); |
| return result; |
| } |
| |
| |
| //======================================================================= |
| // BreakIterator overrides |
| //======================================================================= |
| |
| /** |
| * Return a CharacterIterator over the text being analyzed. |
| */ |
| CharacterIterator& |
| RuleBasedBreakIterator::getText() const { |
| return *fCharIter; |
| } |
| |
| /** |
| * Set the iterator to analyze a new piece of text. This function resets |
| * the current iteration position to the beginning of the text. |
| * @param newText An iterator over the text to analyze. |
| */ |
| void |
| RuleBasedBreakIterator::adoptText(CharacterIterator* newText) { |
| // If we are holding a CharacterIterator adopted from a |
| // previous call to this function, delete it now. |
| if (fCharIter != &fSCharIter) { |
| delete fCharIter; |
| } |
| |
| fCharIter = newText; |
| UErrorCode status = U_ZERO_ERROR; |
| fBreakCache->reset(); |
| fDictionaryCache->reset(); |
| if (newText==NULL || newText->startIndex() != 0) { |
| // startIndex !=0 wants to be an error, but there's no way to report it. |
| // Make the iterator text be an empty string. |
| utext_openUChars(&fText, NULL, 0, &status); |
| } else { |
| utext_openCharacterIterator(&fText, newText, &status); |
| } |
| this->first(); |
| } |
| |
| /** |
| * Set the iterator to analyze a new piece of text. This function resets |
| * the current iteration position to the beginning of the text. |
| * @param newText An iterator over the text to analyze. |
| */ |
| void |
| RuleBasedBreakIterator::setText(const UnicodeString& newText) { |
| UErrorCode status = U_ZERO_ERROR; |
| fBreakCache->reset(); |
| fDictionaryCache->reset(); |
| utext_openConstUnicodeString(&fText, &newText, &status); |
| |
| // Set up a character iterator on the string. |
| // Needed in case someone calls getText(). |
| // Can not, unfortunately, do this lazily on the (probably never) |
| // call to getText(), because getText is const. |
| fSCharIter.setText(newText); |
| |
| if (fCharIter != &fSCharIter) { |
| // old fCharIter was adopted from the outside. Delete it. |
| delete fCharIter; |
| } |
| fCharIter = &fSCharIter; |
| |
| this->first(); |
| } |
| |
| |
| /** |
| * Provide a new UText for the input text. Must reference text with contents identical |
| * to the original. |
| * Intended for use with text data originating in Java (garbage collected) environments |
| * where the data may be moved in memory at arbitrary times. |
| */ |
| RuleBasedBreakIterator &RuleBasedBreakIterator::refreshInputText(UText *input, UErrorCode &status) { |
| if (U_FAILURE(status)) { |
| return *this; |
| } |
| if (input == NULL) { |
| status = U_ILLEGAL_ARGUMENT_ERROR; |
| return *this; |
| } |
| int64_t pos = utext_getNativeIndex(&fText); |
| // Shallow read-only clone of the new UText into the existing input UText |
| utext_clone(&fText, input, FALSE, TRUE, &status); |
| if (U_FAILURE(status)) { |
| return *this; |
| } |
| utext_setNativeIndex(&fText, pos); |
| if (utext_getNativeIndex(&fText) != pos) { |
| // Sanity check. The new input utext is supposed to have the exact same |
| // contents as the old. If we can't set to the same position, it doesn't. |
| // The contents underlying the old utext might be invalid at this point, |
| // so it's not safe to check directly. |
| status = U_ILLEGAL_ARGUMENT_ERROR; |
| } |
| return *this; |
| } |
| |
| |
| /** |
| * Sets the current iteration position to the beginning of the text, position zero. |
| * @return The new iterator position, which is zero. |
| */ |
| int32_t RuleBasedBreakIterator::first(void) { |
| UErrorCode status = U_ZERO_ERROR; |
| if (!fBreakCache->seek(0)) { |
| fBreakCache->populateNear(0, status); |
| } |
| fBreakCache->current(); |
| U_ASSERT(fPosition == 0); |
| return 0; |
| } |
| |
| /** |
| * Sets the current iteration position to the end of the text. |
| * @return The text's past-the-end offset. |
| */ |
| int32_t RuleBasedBreakIterator::last(void) { |
| int32_t endPos = (int32_t)utext_nativeLength(&fText); |
| UBool endShouldBeBoundary = isBoundary(endPos); // Has side effect of setting iterator position. |
| (void)endShouldBeBoundary; |
| U_ASSERT(endShouldBeBoundary); |
| U_ASSERT(fPosition == endPos); |
| return endPos; |
| } |
| |
| /** |
| * Advances the iterator either forward or backward the specified number of steps. |
| * Negative values move backward, and positive values move forward. This is |
| * equivalent to repeatedly calling next() or previous(). |
| * @param n The number of steps to move. The sign indicates the direction |
| * (negative is backwards, and positive is forwards). |
| * @return The character offset of the boundary position n boundaries away from |
| * the current one. |
| */ |
| int32_t RuleBasedBreakIterator::next(int32_t n) { |
| int32_t result = 0; |
| if (n > 0) { |
| for (; n > 0 && result != UBRK_DONE; --n) { |
| result = next(); |
| } |
| } else if (n < 0) { |
| for (; n < 0 && result != UBRK_DONE; ++n) { |
| result = previous(); |
| } |
| } else { |
| result = current(); |
| } |
| return result; |
| } |
| |
| /** |
| * Advances the iterator to the next boundary position. |
| * @return The position of the first boundary after this one. |
| */ |
| int32_t RuleBasedBreakIterator::next(void) { |
| fBreakCache->next(); |
| return fDone ? UBRK_DONE : fPosition; |
| } |
| |
| /** |
| * Move the iterator backwards, to the boundary preceding the current one. |
| * |
| * Starts from the current position within fText. |
| * Starting position need not be on a boundary. |
| * |
| * @return The position of the boundary position immediately preceding the starting position. |
| */ |
| int32_t RuleBasedBreakIterator::previous(void) { |
| UErrorCode status = U_ZERO_ERROR; |
| fBreakCache->previous(status); |
| return fDone ? UBRK_DONE : fPosition; |
| } |
| |
| /** |
| * Sets the iterator to refer to the first boundary position following |
| * the specified position. |
| * @param startPos The position from which to begin searching for a break position. |
| * @return The position of the first break after the current position. |
| */ |
| int32_t RuleBasedBreakIterator::following(int32_t startPos) { |
| // if the supplied position is before the beginning, return the |
| // text's starting offset |
| if (startPos < 0) { |
| return first(); |
| } |
| |
| // Move requested offset to a code point start. It might be on a trail surrogate, |
| // or on a trail byte if the input is UTF-8. Or it may be beyond the end of the text. |
| utext_setNativeIndex(&fText, startPos); |
| startPos = (int32_t)utext_getNativeIndex(&fText); |
| |
| UErrorCode status = U_ZERO_ERROR; |
| fBreakCache->following(startPos, status); |
| return fDone ? UBRK_DONE : fPosition; |
| } |
| |
| /** |
| * Sets the iterator to refer to the last boundary position before the |
| * specified position. |
| * @param offset The position to begin searching for a break from. |
| * @return The position of the last boundary before the starting position. |
| */ |
| int32_t RuleBasedBreakIterator::preceding(int32_t offset) { |
| if (offset > utext_nativeLength(&fText)) { |
| return last(); |
| } |
| |
| // Move requested offset to a code point start. It might be on a trail surrogate, |
| // or on a trail byte if the input is UTF-8. |
| |
| utext_setNativeIndex(&fText, offset); |
| int32_t adjustedOffset = static_cast<int32_t>(utext_getNativeIndex(&fText)); |
| |
| UErrorCode status = U_ZERO_ERROR; |
| fBreakCache->preceding(adjustedOffset, status); |
| return fDone ? UBRK_DONE : fPosition; |
| } |
| |
| /** |
| * Returns true if the specfied position is a boundary position. As a side |
| * effect, leaves the iterator pointing to the first boundary position at |
| * or after "offset". |
| * |
| * @param offset the offset to check. |
| * @return True if "offset" is a boundary position. |
| */ |
| UBool RuleBasedBreakIterator::isBoundary(int32_t offset) { |
| // out-of-range indexes are never boundary positions |
| if (offset < 0) { |
| first(); // For side effects on current position, tag values. |
| return FALSE; |
| } |
| |
| // Adjust offset to be on a code point boundary and not beyond the end of the text. |
| // Note that isBoundary() is always false for offsets that are not on code point boundaries. |
| // But we still need the side effect of leaving iteration at the following boundary. |
| |
| utext_setNativeIndex(&fText, offset); |
| int32_t adjustedOffset = static_cast<int32_t>(utext_getNativeIndex(&fText)); |
| |
| bool result = false; |
| UErrorCode status = U_ZERO_ERROR; |
| if (fBreakCache->seek(adjustedOffset) || fBreakCache->populateNear(adjustedOffset, status)) { |
| result = (fBreakCache->current() == offset); |
| } |
| |
| if (result && adjustedOffset < offset && utext_char32At(&fText, offset) == U_SENTINEL) { |
| // Original offset is beyond the end of the text. Return FALSE, it's not a boundary, |
| // but the iteration position remains set to the end of the text, which is a boundary. |
| return FALSE; |
| } |
| if (!result) { |
| // Not on a boundary. isBoundary() must leave iterator on the following boundary. |
| // Cache->seek(), above, left us on the preceding boundary, so advance one. |
| next(); |
| } |
| return result; |
| } |
| |
| |
| /** |
| * Returns the current iteration position. |
| * @return The current iteration position. |
| */ |
| int32_t RuleBasedBreakIterator::current(void) const { |
| return fPosition; |
| } |
| |
| |
| //======================================================================= |
| // implementation |
| //======================================================================= |
| |
| // |
| // RBBIRunMode - the state machine runs an extra iteration at the beginning and end |
| // of user text. A variable with this enum type keeps track of where we |
| // are. The state machine only fetches user input while in the RUN mode. |
| // |
| enum RBBIRunMode { |
| RBBI_START, // state machine processing is before first char of input |
| RBBI_RUN, // state machine processing is in the user text |
| RBBI_END // state machine processing is after end of user text. |
| }; |
| |
| |
| // Wrapper functions to select the appropriate handleNext() or handleSafePrevious() |
| // instantiation, based on whether an 8 or 16 bit table is required. |
| // |
| // These Trie access functions will be inlined within the handleNext()/Previous() instantions. |
| static inline uint16_t TrieFunc8(const UCPTrie *trie, UChar32 c) { |
| return UCPTRIE_FAST_GET(trie, UCPTRIE_8, c); |
| } |
| |
| static inline uint16_t TrieFunc16(const UCPTrie *trie, UChar32 c) { |
| return UCPTRIE_FAST_GET(trie, UCPTRIE_16, c); |
| } |
| |
| int32_t RuleBasedBreakIterator::handleNext() { |
| const RBBIStateTable *statetable = fData->fForwardTable; |
| bool use8BitsTrie = ucptrie_getValueWidth(fData->fTrie) == UCPTRIE_VALUE_BITS_8; |
| if (statetable->fFlags & RBBI_8BITS_ROWS) { |
| if (use8BitsTrie) { |
| return handleNext<RBBIStateTableRow8, TrieFunc8>(); |
| } else { |
| return handleNext<RBBIStateTableRow8, TrieFunc16>(); |
| } |
| } else { |
| if (use8BitsTrie) { |
| return handleNext<RBBIStateTableRow16, TrieFunc8>(); |
| } else { |
| return handleNext<RBBIStateTableRow16, TrieFunc16>(); |
| } |
| } |
| } |
| |
| int32_t RuleBasedBreakIterator::handleSafePrevious(int32_t fromPosition) { |
| const RBBIStateTable *statetable = fData->fReverseTable; |
| bool use8BitsTrie = ucptrie_getValueWidth(fData->fTrie) == UCPTRIE_VALUE_BITS_8; |
| if (statetable->fFlags & RBBI_8BITS_ROWS) { |
| if (use8BitsTrie) { |
| return handleSafePrevious<RBBIStateTableRow8, TrieFunc8>(fromPosition); |
| } else { |
| return handleSafePrevious<RBBIStateTableRow8, TrieFunc16>(fromPosition); |
| } |
| } else { |
| if (use8BitsTrie) { |
| return handleSafePrevious<RBBIStateTableRow16, TrieFunc8>(fromPosition); |
| } else { |
| return handleSafePrevious<RBBIStateTableRow16, TrieFunc16>(fromPosition); |
| } |
| } |
| } |
| |
| |
| //----------------------------------------------------------------------------------- |
| // |
| // handleNext() |
| // Run the state machine to find a boundary |
| // |
| //----------------------------------------------------------------------------------- |
| template <typename RowType, RuleBasedBreakIterator::PTrieFunc trieFunc> |
| int32_t RuleBasedBreakIterator::handleNext() { |
| int32_t state; |
| uint16_t category = 0; |
| RBBIRunMode mode; |
| |
| RowType *row; |
| UChar32 c; |
| int32_t result = 0; |
| int32_t initialPosition = 0; |
| const RBBIStateTable *statetable = fData->fForwardTable; |
| const char *tableData = statetable->fTableData; |
| uint32_t tableRowLen = statetable->fRowLen; |
| uint32_t dictStart = statetable->fDictCategoriesStart; |
| #ifdef RBBI_DEBUG |
| if (gTrace) { |
| RBBIDebugPuts("Handle Next pos char state category"); |
| } |
| #endif |
| |
| // handleNext always sets the break tag value. |
| // Set the default for it. |
| fRuleStatusIndex = 0; |
| |
| fDictionaryCharCount = 0; |
| |
| // if we're already at the end of the text, return DONE. |
| initialPosition = fPosition; |
| UTEXT_SETNATIVEINDEX(&fText, initialPosition); |
| result = initialPosition; |
| c = UTEXT_NEXT32(&fText); |
| if (c==U_SENTINEL) { |
| fDone = TRUE; |
| return UBRK_DONE; |
| } |
| |
| // Set the initial state for the state machine |
| state = START_STATE; |
| row = (RowType *) |
| //(statetable->fTableData + (statetable->fRowLen * state)); |
| (tableData + tableRowLen * state); |
| |
| |
| mode = RBBI_RUN; |
| if (statetable->fFlags & RBBI_BOF_REQUIRED) { |
| category = 2; |
| mode = RBBI_START; |
| } |
| |
| |
| // loop until we reach the end of the text or transition to state 0 |
| // |
| for (;;) { |
| if (c == U_SENTINEL) { |
| // Reached end of input string. |
| if (mode == RBBI_END) { |
| // We have already run the loop one last time with the |
| // character set to the psueudo {eof} value. Now it is time |
| // to unconditionally bail out. |
| break; |
| } |
| // Run the loop one last time with the fake end-of-input character category. |
| mode = RBBI_END; |
| category = 1; |
| } |
| |
| // |
| // Get the char category. An incoming category of 1 or 2 means that |
| // we are preset for doing the beginning or end of input, and |
| // that we shouldn't get a category from an actual text input character. |
| // |
| if (mode == RBBI_RUN) { |
| // look up the current character's character category, which tells us |
| // which column in the state table to look at. |
| category = trieFunc(fData->fTrie, c); |
| fDictionaryCharCount += (category >= dictStart); |
| } |
| |
| #ifdef RBBI_DEBUG |
| if (gTrace) { |
| RBBIDebugPrintf(" %4" PRId64 " ", utext_getNativeIndex(&fText)); |
| if (0x20<=c && c<0x7f) { |
| RBBIDebugPrintf("\"%c\" ", c); |
| } else { |
| RBBIDebugPrintf("%5x ", c); |
| } |
| RBBIDebugPrintf("%3d %3d\n", state, category); |
| } |
| #endif |
| |
| // State Transition - move machine to its next state |
| // |
| |
| // fNextState is a variable-length array. |
| U_ASSERT(category<fData->fHeader->fCatCount); |
| state = row->fNextState[category]; /*Not accessing beyond memory*/ |
| row = (RowType *) |
| // (statetable->fTableData + (statetable->fRowLen * state)); |
| (tableData + tableRowLen * state); |
| |
| |
| uint16_t accepting = row->fAccepting; |
| if (accepting == ACCEPTING_UNCONDITIONAL) { |
| // Match found, common case. |
| if (mode != RBBI_START) { |
| result = (int32_t)UTEXT_GETNATIVEINDEX(&fText); |
| } |
| fRuleStatusIndex = row->fTagsIdx; // Remember the break status (tag) values. |
| } else if (accepting > ACCEPTING_UNCONDITIONAL) { |
| // Lookahead match is completed. |
| U_ASSERT(accepting < fData->fForwardTable->fLookAheadResultsSize); |
| int32_t lookaheadResult = fLookAheadMatches[accepting]; |
| if (lookaheadResult >= 0) { |
| fRuleStatusIndex = row->fTagsIdx; |
| fPosition = lookaheadResult; |
| return lookaheadResult; |
| } |
| } |
| |
| // If we are at the position of the '/' in a look-ahead (hard break) rule; |
| // record the current position, to be returned later, if the full rule matches. |
| // TODO: Move this check before the previous check of fAccepting. |
| // This would enable hard-break rules with no following context. |
| // But there are line break test failures when trying this. Investigate. |
| // Issue ICU-20837 |
| uint16_t rule = row->fLookAhead; |
| U_ASSERT(rule == 0 || rule > ACCEPTING_UNCONDITIONAL); |
| U_ASSERT(rule == 0 || rule < fData->fForwardTable->fLookAheadResultsSize); |
| if (rule > ACCEPTING_UNCONDITIONAL) { |
| int32_t pos = (int32_t)UTEXT_GETNATIVEINDEX(&fText); |
| fLookAheadMatches[rule] = pos; |
| } |
| |
| if (state == STOP_STATE) { |
| // This is the normal exit from the lookup state machine. |
| // We have advanced through the string until it is certain that no |
| // longer match is possible, no matter what characters follow. |
| break; |
| } |
| |
| // Advance to the next character. |
| // If this is a beginning-of-input loop iteration, don't advance |
| // the input position. The next iteration will be processing the |
| // first real input character. |
| if (mode == RBBI_RUN) { |
| c = UTEXT_NEXT32(&fText); |
| } else { |
| if (mode == RBBI_START) { |
| mode = RBBI_RUN; |
| } |
| } |
| } |
| |
| // The state machine is done. Check whether it found a match... |
| |
| // If the iterator failed to advance in the match engine, force it ahead by one. |
| // (This really indicates a defect in the break rules. They should always match |
| // at least one character.) |
| if (result == initialPosition) { |
| utext_setNativeIndex(&fText, initialPosition); |
| utext_next32(&fText); |
| result = (int32_t)utext_getNativeIndex(&fText); |
| fRuleStatusIndex = 0; |
| } |
| |
| // Leave the iterator at our result position. |
| fPosition = result; |
| #ifdef RBBI_DEBUG |
| if (gTrace) { |
| RBBIDebugPrintf("result = %d\n\n", result); |
| } |
| #endif |
| return result; |
| } |
| |
| |
| //----------------------------------------------------------------------------------- |
| // |
| // handleSafePrevious() |
| // |
| // Iterate backwards using the safe reverse rules. |
| // The logic of this function is similar to handleNext(), but simpler |
| // because the safe table does not require as many options. |
| // |
| //----------------------------------------------------------------------------------- |
| template <typename RowType, RuleBasedBreakIterator::PTrieFunc trieFunc> |
| int32_t RuleBasedBreakIterator::handleSafePrevious(int32_t fromPosition) { |
| |
| int32_t state; |
| uint16_t category = 0; |
| RowType *row; |
| UChar32 c; |
| int32_t result = 0; |
| |
| const RBBIStateTable *stateTable = fData->fReverseTable; |
| UTEXT_SETNATIVEINDEX(&fText, fromPosition); |
| #ifdef RBBI_DEBUG |
| if (gTrace) { |
| RBBIDebugPuts("Handle Previous pos char state category"); |
| } |
| #endif |
| |
| // if we're already at the start of the text, return DONE. |
| if (fData == NULL || UTEXT_GETNATIVEINDEX(&fText)==0) { |
| return BreakIterator::DONE; |
| } |
| |
| // Set the initial state for the state machine |
| c = UTEXT_PREVIOUS32(&fText); |
| state = START_STATE; |
| row = (RowType *) |
| (stateTable->fTableData + (stateTable->fRowLen * state)); |
| |
| // loop until we reach the start of the text or transition to state 0 |
| // |
| for (; c != U_SENTINEL; c = UTEXT_PREVIOUS32(&fText)) { |
| |
| // look up the current character's character category, which tells us |
| // which column in the state table to look at. |
| // |
| // Off the dictionary flag bit. For reverse iteration it is not used. |
| category = trieFunc(fData->fTrie, c); |
| |
| #ifdef RBBI_DEBUG |
| if (gTrace) { |
| RBBIDebugPrintf(" %4d ", (int32_t)utext_getNativeIndex(&fText)); |
| if (0x20<=c && c<0x7f) { |
| RBBIDebugPrintf("\"%c\" ", c); |
| } else { |
| RBBIDebugPrintf("%5x ", c); |
| } |
| RBBIDebugPrintf("%3d %3d\n", state, category); |
| } |
| #endif |
| |
| // State Transition - move machine to its next state |
| // |
| // fNextState is a variable-length array. |
| U_ASSERT(category<fData->fHeader->fCatCount); |
| state = row->fNextState[category]; /*Not accessing beyond memory*/ |
| row = (RowType *) |
| (stateTable->fTableData + (stateTable->fRowLen * state)); |
| |
| if (state == STOP_STATE) { |
| // This is the normal exit from the lookup state machine. |
| // Transistion to state zero means we have found a safe point. |
| break; |
| } |
| } |
| |
| // The state machine is done. Check whether it found a match... |
| result = (int32_t)UTEXT_GETNATIVEINDEX(&fText); |
| #ifdef RBBI_DEBUG |
| if (gTrace) { |
| RBBIDebugPrintf("result = %d\n\n", result); |
| } |
| #endif |
| return result; |
| } |
| |
| |
| //------------------------------------------------------------------------------- |
| // |
| // getRuleStatus() Return the break rule tag associated with the current |
| // iterator position. If the iterator arrived at its current |
| // position by iterating forwards, the value will have been |
| // cached by the handleNext() function. |
| // |
| //------------------------------------------------------------------------------- |
| |
| int32_t RuleBasedBreakIterator::getRuleStatus() const { |
| |
| // fLastRuleStatusIndex indexes to the start of the appropriate status record |
| // (the number of status values.) |
| // This function returns the last (largest) of the array of status values. |
| int32_t idx = fRuleStatusIndex + fData->fRuleStatusTable[fRuleStatusIndex]; |
| int32_t tagVal = fData->fRuleStatusTable[idx]; |
| |
| return tagVal; |
| } |
| |
| |
| int32_t RuleBasedBreakIterator::getRuleStatusVec( |
| int32_t *fillInVec, int32_t capacity, UErrorCode &status) { |
| if (U_FAILURE(status)) { |
| return 0; |
| } |
| |
| int32_t numVals = fData->fRuleStatusTable[fRuleStatusIndex]; |
| int32_t numValsToCopy = numVals; |
| if (numVals > capacity) { |
| status = U_BUFFER_OVERFLOW_ERROR; |
| numValsToCopy = capacity; |
| } |
| int i; |
| for (i=0; i<numValsToCopy; i++) { |
| fillInVec[i] = fData->fRuleStatusTable[fRuleStatusIndex + i + 1]; |
| } |
| return numVals; |
| } |
| |
| |
| |
| //------------------------------------------------------------------------------- |
| // |
| // getBinaryRules Access to the compiled form of the rules, |
| // for use by build system tools that save the data |
| // for standard iterator types. |
| // |
| //------------------------------------------------------------------------------- |
| const uint8_t *RuleBasedBreakIterator::getBinaryRules(uint32_t &length) { |
| const uint8_t *retPtr = NULL; |
| length = 0; |
| |
| if (fData != NULL) { |
| retPtr = (const uint8_t *)fData->fHeader; |
| length = fData->fHeader->fLength; |
| } |
| return retPtr; |
| } |
| |
| |
| RuleBasedBreakIterator *RuleBasedBreakIterator::createBufferClone( |
| void * /*stackBuffer*/, int32_t &bufferSize, UErrorCode &status) { |
| if (U_FAILURE(status)){ |
| return NULL; |
| } |
| |
| if (bufferSize == 0) { |
| bufferSize = 1; // preflighting for deprecated functionality |
| return NULL; |
| } |
| |
| BreakIterator *clonedBI = clone(); |
| if (clonedBI == NULL) { |
| status = U_MEMORY_ALLOCATION_ERROR; |
| } else { |
| status = U_SAFECLONE_ALLOCATED_WARNING; |
| } |
| return (RuleBasedBreakIterator *)clonedBI; |
| } |
| |
| U_NAMESPACE_END |
| |
| |
| static icu::UStack *gLanguageBreakFactories = nullptr; |
| static const icu::UnicodeString *gEmptyString = nullptr; |
| static icu::UInitOnce gLanguageBreakFactoriesInitOnce = U_INITONCE_INITIALIZER; |
| static icu::UInitOnce gRBBIInitOnce = U_INITONCE_INITIALIZER; |
| |
| /** |
| * Release all static memory held by breakiterator. |
| */ |
| U_CDECL_BEGIN |
| UBool U_CALLCONV rbbi_cleanup(void) { |
| delete gLanguageBreakFactories; |
| gLanguageBreakFactories = nullptr; |
| delete gEmptyString; |
| gEmptyString = nullptr; |
| gLanguageBreakFactoriesInitOnce.reset(); |
| gRBBIInitOnce.reset(); |
| return TRUE; |
| } |
| U_CDECL_END |
| |
| U_CDECL_BEGIN |
| static void U_CALLCONV _deleteFactory(void *obj) { |
| delete (icu::LanguageBreakFactory *) obj; |
| } |
| U_CDECL_END |
| U_NAMESPACE_BEGIN |
| |
| static void U_CALLCONV rbbiInit() { |
| gEmptyString = new UnicodeString(); |
| ucln_common_registerCleanup(UCLN_COMMON_RBBI, rbbi_cleanup); |
| } |
| |
| static void U_CALLCONV initLanguageFactories() { |
| UErrorCode status = U_ZERO_ERROR; |
| U_ASSERT(gLanguageBreakFactories == NULL); |
| gLanguageBreakFactories = new UStack(_deleteFactory, NULL, status); |
| if (gLanguageBreakFactories != NULL && U_SUCCESS(status)) { |
| ICULanguageBreakFactory *builtIn = new ICULanguageBreakFactory(status); |
| gLanguageBreakFactories->push(builtIn, status); |
| #ifdef U_LOCAL_SERVICE_HOOK |
| LanguageBreakFactory *extra = (LanguageBreakFactory *)uprv_svc_hook("languageBreakFactory", &status); |
| if (extra != NULL) { |
| gLanguageBreakFactories->push(extra, status); |
| } |
| #endif |
| } |
| ucln_common_registerCleanup(UCLN_COMMON_RBBI, rbbi_cleanup); |
| } |
| |
| |
| static const LanguageBreakEngine* |
| getLanguageBreakEngineFromFactory(UChar32 c) |
| { |
| umtx_initOnce(gLanguageBreakFactoriesInitOnce, &initLanguageFactories); |
| if (gLanguageBreakFactories == NULL) { |
| return NULL; |
| } |
| |
| int32_t i = gLanguageBreakFactories->size(); |
| const LanguageBreakEngine *lbe = NULL; |
| while (--i >= 0) { |
| LanguageBreakFactory *factory = (LanguageBreakFactory *)(gLanguageBreakFactories->elementAt(i)); |
| lbe = factory->getEngineFor(c); |
| if (lbe != NULL) { |
| break; |
| } |
| } |
| return lbe; |
| } |
| |
| |
| //------------------------------------------------------------------------------- |
| // |
| // getLanguageBreakEngine Find an appropriate LanguageBreakEngine for the |
| // the character c. |
| // |
| //------------------------------------------------------------------------------- |
| const LanguageBreakEngine * |
| RuleBasedBreakIterator::getLanguageBreakEngine(UChar32 c) { |
| const LanguageBreakEngine *lbe = NULL; |
| UErrorCode status = U_ZERO_ERROR; |
| |
| if (fLanguageBreakEngines == NULL) { |
| fLanguageBreakEngines = new UStack(status); |
| if (fLanguageBreakEngines == NULL || U_FAILURE(status)) { |
| delete fLanguageBreakEngines; |
| fLanguageBreakEngines = 0; |
| return NULL; |
| } |
| } |
| |
| int32_t i = fLanguageBreakEngines->size(); |
| while (--i >= 0) { |
| lbe = (const LanguageBreakEngine *)(fLanguageBreakEngines->elementAt(i)); |
| if (lbe->handles(c)) { |
| return lbe; |
| } |
| } |
| |
| // No existing dictionary took the character. See if a factory wants to |
| // give us a new LanguageBreakEngine for this character. |
| lbe = getLanguageBreakEngineFromFactory(c); |
| |
| // If we got one, use it and push it on our stack. |
| if (lbe != NULL) { |
| fLanguageBreakEngines->push((void *)lbe, status); |
| // Even if we can't remember it, we can keep looking it up, so |
| // return it even if the push fails. |
| return lbe; |
| } |
| |
| // No engine is forthcoming for this character. Add it to the |
| // reject set. Create the reject break engine if needed. |
| if (fUnhandledBreakEngine == NULL) { |
| fUnhandledBreakEngine = new UnhandledEngine(status); |
| if (U_SUCCESS(status) && fUnhandledBreakEngine == NULL) { |
| status = U_MEMORY_ALLOCATION_ERROR; |
| return nullptr; |
| } |
| // Put it last so that scripts for which we have an engine get tried |
| // first. |
| fLanguageBreakEngines->insertElementAt(fUnhandledBreakEngine, 0, status); |
| // If we can't insert it, or creation failed, get rid of it |
| if (U_FAILURE(status)) { |
| delete fUnhandledBreakEngine; |
| fUnhandledBreakEngine = 0; |
| return NULL; |
| } |
| } |
| |
| // Tell the reject engine about the character; at its discretion, it may |
| // add more than just the one character. |
| fUnhandledBreakEngine->handleCharacter(c); |
| |
| return fUnhandledBreakEngine; |
| } |
| |
| void RuleBasedBreakIterator::dumpCache() { |
| fBreakCache->dumpCache(); |
| } |
| |
| void RuleBasedBreakIterator::dumpTables() { |
| fData->printData(); |
| } |
| |
| /** |
| * Returns the description used to create this iterator |
| */ |
| |
| const UnicodeString& |
| RuleBasedBreakIterator::getRules() const { |
| if (fData != NULL) { |
| return fData->getRuleSourceString(); |
| } else { |
| umtx_initOnce(gRBBIInitOnce, &rbbiInit); |
| return *gEmptyString; |
| } |
| } |
| |
| U_NAMESPACE_END |
| |
| #endif /* #if !UCONFIG_NO_BREAK_ITERATION */ |
