/*
 *******************************************************************************
 * Copyright (C) 2005-2011 International Business Machines Corporation and          *
 * others. All Rights Reserved.                                                *
 *******************************************************************************
 */
package com.ibm.icu.text;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.text.CharacterIterator;

import com.ibm.icu.impl.Assert;
import com.ibm.icu.impl.ICUDebug;


/**
 * Rule Based Break Iterator 
 * This is a port of the C++ class RuleBasedBreakIterator from ICU4C.
 * 
 * @stable ICU 2.0
 */
public class RuleBasedBreakIterator extends BreakIterator {

    
    //=======================================================================
    // Constructors & Factories
    //=======================================================================
    
    /** 
     * @internal 
     * @deprecated This API is ICU internal only.
     */
    public RuleBasedBreakIterator() {
    }

    /**
     * Create a break iterator from a precompiled set of break rules.
     * 
     * Creating a break iterator from the binary rules is much faster than
     * creating one from source rules. 
     * 
     * The binary rules are generated by the RuleBasedBreakIterator.compileRules() function.
     * Binary break iterator rules are not guaranteed to be compatible between
     * different versions of ICU.
     * 
     * @param is an input stream supplying the compiled binary rules.
     * @throws IOException if there is an error while reading the rules from the InputStream.
     * @see    #compileRules(String, OutputStream)
     * @draft ICU 4.8
     */
    public static RuleBasedBreakIterator getInstanceFromCompiledRules(InputStream is) throws IOException {
        RuleBasedBreakIterator  This = new RuleBasedBreakIterator();
        This.fRData = RBBIDataWrapper.get(is);
        return This;   
    }
    
    /*private RuleBasedBreakIterator(RuleBasedBreakIterator other) {
        // TODO: check types.
        fRData = other.fRData;
        if (fText != null) {
            fText = (CharacterIterator)(other.fText.clone());   
        }
    }*/

    /**
     * Construct a RuleBasedBreakIterator from a set of rules supplied as a string.
     * @param rules The break rules to be used.
     * @stable ICU 2.2
     */
    public RuleBasedBreakIterator(String rules)  {
        init();
        try {
            ByteArrayOutputStream ruleOS = new ByteArrayOutputStream();
            compileRules(rules, ruleOS);
            byte [] ruleBA = ruleOS.toByteArray();
            InputStream ruleIS = new ByteArrayInputStream(ruleBA);
            fRData = RBBIDataWrapper.get(ruleIS);
        } catch (IOException e) {
            ///CLOVER:OFF
            // An IO exception can only arrive here if there is a bug in the RBBI Rule compiler,
            //  causing bogus compiled rules to be produced, but with no compile error raised.
            RuntimeException rte = new RuntimeException("RuleBasedBreakIterator rule compilation internal error: "
                    + e.getMessage());
            throw rte;
            ///CLOVER:ON
        }
    }
    
    
    //=======================================================================
    // Boilerplate
    //=======================================================================
    
    /**
     * Clones this iterator.
     * @return A newly-constructed RuleBasedBreakIterator with the same
     * behavior as this one.
     * @stable ICU 2.0
     */
    public Object clone()
    {
        RuleBasedBreakIterator result = (RuleBasedBreakIterator)super.clone();
        if (fText != null) {
            result.fText = (CharacterIterator)(fText.clone());   
        }
        return result;
    }

    /**
     * Returns true if both BreakIterators are of the same class, have the same
     * rules, and iterate over the same text.
     * @stable ICU 2.0
     */
    public boolean equals(Object that) {
        try {
            RuleBasedBreakIterator other = (RuleBasedBreakIterator) that;
            if (fRData != other.fRData && (fRData == null || other.fRData == null)) {System.out.println("GOT HERE");
                return false;
            }
            if (fRData != null && other.fRData != null && 
                    (!fRData.fRuleSource.equals(other.fRData.fRuleSource))) {
                return false;
            }
            if (fText == null && other.fText == null) {
                return true;   
            }
            if (fText == null || other.fText == null) {
                return false;   
            }
            return fText.equals(other.fText);
        }
        catch(ClassCastException e) {
            return false;
        }
     }

    /**
     * Returns the description (rules) used to create this iterator.
     * (In ICU4C, the same function is RuleBasedBreakIterator::getRules())
     * @stable ICU 2.0
     */
    public String toString() {
        String   retStr = null;
        if (fRData != null) {
            retStr =  fRData.fRuleSource;
        }
        return retStr;
    }

    /**
     * Compute a hashcode for this BreakIterator
     * @return A hash code
     * @stable ICU 2.0
     */
    public int hashCode()
    {
        return fRData.fRuleSource.hashCode(); 
    }

    
    /** 
     * Tag value for "words" that do not fit into any of other categories. 
     * Includes spaces and most punctuation. 
     * @draft ICU 3.0 
     * @provisional This is a draft API and might change in a future release of ICU.
     */
    public static final int WORD_NONE           = 0;

    /**
     * Upper bound for tags for uncategorized words. 
     * @draft ICU 3.0 
     * @provisional This is a draft API and might change in a future release of ICU.
     */
    public static final int WORD_NONE_LIMIT     = 100;

    /**
     * Tag value for words that appear to be numbers, lower limit. 
     * @draft ICU 3.0 
     * @provisional This is a draft API and might change in a future release of ICU.
     */
    public static final int WORD_NUMBER         = 100;

    /** 
     * Tag value for words that appear to be numbers, upper limit.
     * @draft ICU 3.0 
     * @provisional This is a draft API and might change in a future release of ICU.
     */
    public static final int WORD_NUMBER_LIMIT   = 200;

    /** 
     * Tag value for words that contain letters, excluding
     * hiragana, katakana or ideographic characters, lower limit. 
     * @draft ICU 3.0 
     * @provisional This is a draft API and might change in a future release of ICU.
     */
    public static final int WORD_LETTER         = 200;

    /** 
     * Tag value for words containing letters, upper limit 
     * @draft ICU 3.0 
     * @provisional This is a draft API and might change in a future release of ICU.
     */
    public static final int WORD_LETTER_LIMIT   = 300;

    /** 
     * Tag value for words containing kana characters, lower limit
     * @draft ICU 3.0 
     * @provisional This is a draft API and might change in a future release of ICU.
     */
    public static final int WORD_KANA           = 300;

    /** 
     * Tag value for words containing kana characters, upper limit
     * @draft ICU 3.0 
     * @provisional This is a draft API and might change in a future release of ICU.
     */
    public static final int WORD_KANA_LIMIT     = 400;

    /**
     * Tag value for words containing ideographic characters, lower limit
     * @draft ICU 3.0 
     * @provisional This is a draft API and might change in a future release of ICU.
     */
    public static final int WORD_IDEO           = 400;

    /**
     * Tag value for words containing ideographic characters, upper limit
     * @draft ICU 3.0 
     * @provisional This is a draft API and might change in a future release of ICU.
     */
    public static final int WORD_IDEO_LIMIT     = 500;

   
    private static final int  START_STATE = 1;     // The state number of the starting state
    private static final int  STOP_STATE  = 0;     // The state-transition value indicating "stop"
    
    // RBBIRunMode - the state machine runs an extra iteration at the beginning and end
    //               of user text.  A variable with this enum type keeps track of where we
    //               are.  The state machine only fetches user text input while in RUN mode.
    private static final int  RBBI_START  = 0;
    private static final int  RBBI_RUN    = 1;
    private static final int  RBBI_END   = 2;

    /*
     * The character iterator through which this BreakIterator accesses the text.
     */
    private CharacterIterator   fText = new java.text.StringCharacterIterator("");
    
    /**
     * The rule data for this BreakIterator instance
     * @internal
     * @deprecated This API is ICU internal only.
     */
    protected RBBIDataWrapper     fRData;
    
    /*
     * Index of the Rule {tag} values for the most recent match. 
     */
    private int                 fLastRuleStatusIndex;

    /*
     * Rule tag value valid flag.
     * Some iterator operations don't intrinsically set the correct tag value.
     * This flag lets us lazily compute the value if we are ever asked for it.
     */
    private boolean             fLastStatusIndexValid;

    /**
     * Counter for the number of characters encountered with the "dictionary"
     *   flag set.  Normal RBBI iterators don't use it, although the code
     *   for updating it is live.  Dictionary Based break iterators (a subclass
     *   of us) access this field directly.
     * @internal
     * @deprecated This API is ICU internal only.
     */
     protected int fDictionaryCharCount;

    /**
     * Debugging flag.  Trace operation of state machine when true.
     * @internal
     * @deprecated This API is ICU internal only.
     */
    public static boolean       fTrace;

    /*
     * ICU debug argument name for RBBI
     */
    private static final String RBBI_DEBUG_ARG = "rbbi";

    /**
     * Dump the contents of the state table and character classes for this break iterator.
     * For debugging only.
     * @internal
     * @deprecated This API is ICU internal only.
     */
    public void dump() {
        this.fRData.dump();   
    }

    private static boolean debugInitDone = false;
    
    private void init() {
        fLastStatusIndexValid = true;
        fDictionaryCharCount  = 0;

 
        if (debugInitDone == false) {
            fTrace = ICUDebug.enabled(RBBI_DEBUG_ARG)
                && ICUDebug.value(RBBI_DEBUG_ARG).indexOf("trace") >= 0;
            debugInitDone = true;
        }
    }

    /**
     * Compile a set of source break rules into the binary state tables used
     * by the break iterator engine.  Creating a break iterator from precompiled
     * rules is much faster than creating one from source rules.
     * 
     * Binary break rules are not guaranteed to be compatible between different
     * versions of ICU.
     * 
     * 
     * @param rules  The source form of the break rules
     * @param ruleBinary  An output stream to receive the compiled rules.
     * @throws IOException If there is an error writing the output.
     * @see #getInstanceFromCompiledRules(InputStream)
     * @draft ICU 4.8
     */
    public static void compileRules(String rules, OutputStream ruleBinary) throws IOException {
        RBBIRuleBuilder.compileRules(rules, ruleBinary);
    }
    
    //=======================================================================
    // BreakIterator overrides
    //=======================================================================

    /**
     * Sets the current iteration position to the beginning of the text.
     * (i.e., the CharacterIterator's starting offset).
     * @return The offset of the beginning of the text.
     * @stable ICU 2.0
     */
    public int first() {
        fLastRuleStatusIndex  = 0;
        fLastStatusIndexValid = true;
        if (fText == null) {
            return BreakIterator.DONE;
        }
        fText.first();
        return fText.getIndex();
    }
    
    
    /**
     * Sets the current iteration position to the end of the text.
     * (i.e., the CharacterIterator's ending offset).
     * @return The text's past-the-end offset.
     * @stable ICU 2.0
     */
    public int last() {
        if (fText == null) {
            fLastRuleStatusIndex  = 0;
            fLastStatusIndexValid = true;
            return BreakIterator.DONE;
        }

        // I'm not sure why, but t.last() returns the offset of the last character,
        // rather than the past-the-end offset
        //
        //   (It's so a loop like for(p=it.last(); p!=DONE; p=it.previous()) ...
        //     will work correctly.)


        fLastStatusIndexValid = false;
        int pos = fText.getEndIndex();
        fText.setIndex(pos);
        return pos;
    }
    
    
    /**
     * Advances the iterator either forward or backward the specified number of steps.
     * Negative values move backward, and positive values move forward.  This is
     * equivalent to repeatedly calling next() or previous().
     * @param n The number of steps to move.  The sign indicates the direction
     * (negative is backwards, and positive is forwards).
     * @return The character offset of the boundary position n boundaries away from
     * the current one.
     * @stable ICU 2.0
     */
    public int next(int n) {
        int result = current();
        while (n > 0) {
            result = handleNext();
            --n;
        }
        while (n < 0) {
            result = previous();
            ++n;
        }
        return result;
    }
    
    
    /**
     * Advances the iterator to the next boundary position.
     * @return The position of the first boundary after this one.
     * @stable ICU 2.0
     */
    public int next() {
        return handleNext();
    }
    
    
    /**
     * Moves the iterator backwards, to the last boundary preceding this one.
     * @return The position of the last boundary position preceding this one.
     * @stable ICU 2.0
     */
    public int previous() {
        // if we're already sitting at the beginning of the text, return DONE
        if (fText == null || current() == fText.getBeginIndex()) {
            fLastRuleStatusIndex  = 0;
            fLastStatusIndexValid = true;
            return BreakIterator.DONE;
        }

        if (fRData.fSRTable != null || fRData.fSFTable != null) {
            return handlePrevious(fRData.fRTable);
        }

        // old rule syntax
        // set things up.  handlePrevious() will back us up to some valid
        // break position before the current position (we back our internal
        // iterator up one step to prevent handlePrevious() from returning
        // the current position), but not necessarily the last one before
        // where we started

        int       start = current();

        CIPrevious32(fText);
        int       lastResult    = handlePrevious(fRData.fRTable);
        if (lastResult == BreakIterator.DONE) {
            lastResult = fText.getBeginIndex();
            fText.setIndex(lastResult);
        }
        int       result        = lastResult;
        int       lastTag       = 0;
        boolean   breakTagValid = false;

        // iterate forward from the known break position until we pass our
        // starting point.  The last break position before the starting
        // point is our return value

        for (;;) {
            result         = handleNext();
            if (result == BreakIterator.DONE || result >= start) {
                break;
            }
            lastResult     = result;
            lastTag        = fLastRuleStatusIndex;
            breakTagValid  = true;
        }

        // fLastBreakTag wants to have the value for section of text preceding
        // the result position that we are to return (in lastResult.)  If
        // the backwards rules overshot and the above loop had to do two or more
        // handleNext()s to move up to the desired return position, we will have a valid
        // tag value. But, if handlePrevious() took us to exactly the correct result positon,
        // we wont have a tag value for that position, which is only set by handleNext().

        // set the current iteration position to be the last break position
        // before where we started, and then return that value
        fText.setIndex(lastResult);
        fLastRuleStatusIndex  = lastTag;       // for use by getRuleStatus()
        fLastStatusIndexValid = breakTagValid;
        return lastResult;
    }
    /**
     * Sets the iterator to refer to the first boundary position following
     * the specified position.
     * @param offset The position from which to begin searching for a break position.
     * @return The position of the first break after the current position.
     * @stable ICU 2.0
     */
    public int following(int offset) {
        // if the offset passed in is already past the end of the text,
        // just return DONE; if it's before the beginning, return the
        // text's starting offset
        fLastRuleStatusIndex  = 0;
        fLastStatusIndexValid = true;
        if (fText == null || offset >= fText.getEndIndex()) {
            last();
            return next();
        }
        else if (offset < fText.getBeginIndex()) {
            return first();
        }

        // otherwise, set our internal iteration position (temporarily)
        // to the position passed in.  If this is the _beginning_ position,
        // then we can just use next() to get our return value

        int result = 0;

        if (fRData.fSRTable != null) {
            // Safe Point Reverse rules exist.
            //   This allows us to use the optimum algorithm.
            fText.setIndex(offset);
            // move forward one codepoint to prepare for moving back to a
            // safe point.
            // this handles offset being between a supplementary character
            CINext32(fText);
            // handlePrevious will move most of the time to < 1 boundary away
            handlePrevious(fRData.fSRTable);
            result = next();
            while (result <= offset) {
                result = next();
            }
            return result;
        }
        if (fRData.fSFTable != null) {
            // No Safe point reverse table, but there is a safe pt forward table.
            // 
            fText.setIndex(offset);
            CIPrevious32(fText);
            // handle next will give result >= offset
            handleNext(fRData.fSFTable);
            // previous will give result 0 or 1 boundary away from offset,
            // most of the time
            // we have to
            int oldresult = previous();
            while (oldresult > offset) {
                result = previous();
                if (result <= offset) {
                    return oldresult;
                }
                oldresult = result;
            }
            result = next();
            if (result <= offset) {
                return next();
            }
            return result;
        }
        // otherwise, we have to sync up first.  Use handlePrevious() to back
        // us up to a known break position before the specified position (if
        // we can determine that the specified position is a break position,
        // we don't back up at all).  This may or may not be the last break
        // position at or before our starting position.  Advance forward
        // from here until we've passed the starting position.  The position
        // we stop on will be the first break position after the specified one.
        // old rule syntax

        fText.setIndex(offset);
        if (offset == fText.getBeginIndex()) {
            return handleNext();
        }
        result = previous();

        while (result != BreakIterator.DONE && result <= offset) {
            result = next();
        }

        return result;
    }
    /**
     * Sets the iterator to refer to the last boundary position before the
     * specified position.
     * @param offset The position to begin searching for a break from.
     * @return The position of the last boundary before the starting position.
     * @stable ICU 2.0
     */
    public int preceding(int offset) {
        // if the offset passed in is already past the end of the text,
        // just return DONE; if it's before the beginning, return the

        // text's starting offset
        if (fText == null || offset > fText.getEndIndex()) {
            // return BreakIterator::DONE;
            return last();
        }
        else if (offset < fText.getBeginIndex()) {
            return first();
        }

        // if we start by updating the current iteration position to the
        // position specified by the caller, we can just use previous()
        // to carry out this operation

        int  result;
        if (fRData.fSFTable != null) {
            /// todo synwee
            // new rule syntax
            fText.setIndex(offset);
            // move backwards one codepoint to prepare for moving forwards to a
            // safe point.
            // this handles offset being between a supplementary character
            CIPrevious32(fText);
            handleNext(fRData.fSFTable);
            result = previous();
            while (result >= offset) {
                result = previous();
            }
            return result;
        }
        if (fRData.fSRTable != null) {
            // backup plan if forward safe table is not available
            fText.setIndex(offset);
            CINext32(fText);
            // handle previous will give result <= offset
            handlePrevious(fRData.fSRTable);

            // next will give result 0 or 1 boundary away from offset,
            // most of the time
            // we have to
            int oldresult = next();
            while (oldresult < offset) {
                result = next();
                if (result >= offset) {
                    return oldresult;
                }
                oldresult = result;
            }
            result = previous();
            if (result >= offset) {
                return previous();
            }
            return result;
        }

        // old rule syntax
        fText.setIndex(offset);
        return previous();
    }

    /**
     * Throw IllegalArgumentException unless begin <= offset < end.
     * @stable ICU 2.0
     */
    protected static final void checkOffset(int offset, CharacterIterator text) {
        if (offset < text.getBeginIndex() || offset > text.getEndIndex()) {
            throw new IllegalArgumentException("offset out of bounds");
        }
    }


/**
 * Returns true if the specfied position is a boundary position.  As a side
 * effect, leaves the iterator pointing to the first boundary position at
 * or after "offset".
 * @param offset the offset to check.
 * @return True if "offset" is a boundary position.
 * @stable ICU 2.0
 */
public boolean isBoundary(int offset) {
    checkOffset(offset, fText);
    
    // the beginning index of the iterator is always a boundary position by definition
    if (offset == fText.getBeginIndex()) {
        first();       // For side effects on current position, tag values.
        return true;
    }

    if (offset == fText.getEndIndex()) {
        last();       // For side effects on current position, tag values.
        return true;
    }

    // otherwise, we can use following() on the position before the specified
    // one and return true if the position we get back is the one the user
    // specified
    
    // return following(offset - 1) == offset;
    // TODO:  check whether it is safe to revert to the simpler offset-1 code
    //         The safe rules may take care of unpaired surrogates ok.
    fText.setIndex(offset);
    CIPrevious32(fText);
    int  pos = fText.getIndex();
    boolean result = following(pos) == offset;
    return result;
}

/**
 * Returns the current iteration position.
 * @return The current iteration position.
 * @stable ICU 2.0
 */
public int current() {
    return (fText != null) ? fText.getIndex() : BreakIterator.DONE;
    }


private void makeRuleStatusValid() {
    if (fLastStatusIndexValid == false) {
        //  No cached status is available.
        if (fText == null || current() == fText.getBeginIndex()) {
            //  At start of text, or there is no text.  Status is always zero.
            fLastRuleStatusIndex = 0;
            fLastStatusIndexValid = true;
        } else {
            //  Not at start of text.  Find status the tedious way.
            int pa = current();
            previous();
            int pb = next();
            Assert.assrt (pa == pb);
        }
        Assert.assrt(fLastStatusIndexValid == true);
        Assert.assrt(fLastRuleStatusIndex >= 0  &&  fLastRuleStatusIndex < fRData.fStatusTable.length);
    }
}


/**
 * Return the status tag from the break rule that determined the most recently
 * returned break position.  The values appear in the rule source
 * within brackets, {123}, for example.  For rules that do not specify a
 * status, a default value of 0 is returned.  If more than one rule applies,
 * the numerically largest of the possible status values is returned.
 * <p>
 * Of the standard types of ICU break iterators, only the word break
 * iterator provides status values.  The values are defined in
 * class RuleBasedBreakIterator, and allow distinguishing between words
 * that contain alphabetic letters, "words" that appear to be numbers,
 * punctuation and spaces, words containing ideographic characters, and
 * more.  Call <code>getRuleStatus</code> after obtaining a boundary
 * position from <code>next()<code>, <code>previous()</code>, or 
 * any other break iterator functions that returns a boundary position.
 * <p>
 * @return the status from the break rule that determined the most recently
 * returned break position.
 *
 * @draft ICU 3.0
 * @provisional This is a draft API and might change in a future release of ICU.
 */

public int  getRuleStatus() {
    makeRuleStatusValid();
    //   Status records have this form:
    //           Count N         <--  fLastRuleStatusIndex points here.
    //           Status val 0
    //           Status val 1
    //              ...
    //           Status val N-1  <--  the value we need to return
    //   The status values are sorted in ascending order.
    //   This function returns the last (largest) of the array of status values.
    int  idx = fLastRuleStatusIndex + fRData.fStatusTable[fLastRuleStatusIndex];
    int  tagVal = fRData.fStatusTable[idx];

    return tagVal;
}


/**
 * Get the status (tag) values from the break rule(s) that determined the most 
 * recently returned break position.  The values appear in the rule source
 * within brackets, {123}, for example.  The default status value for rules
 * that do not explicitly provide one is zero.
 * <p>
 * The status values used by the standard ICU break rules are defined
 * as public constants in class RuleBasedBreakIterator.
 * <p>
 * If the size  of the output array is insufficient to hold the data,
 *  the output will be truncated to the available length.  No exception
 *  will be thrown.
 *
 * @param fillInArray an array to be filled in with the status values.  
 * @return          The number of rule status values from rules that determined 
 *                  the most recent boundary returned by the break iterator.
 *                  In the event that the array is too small, the return value
 *                  is the total number of status values that were available,
 *                  not the reduced number that were actually returned.
 * @draft ICU 3.0
 * @provisional This is a draft API and might change in a future release of ICU.
 */
public int getRuleStatusVec(int[] fillInArray) {
    makeRuleStatusValid();
    int numStatusVals = fRData.fStatusTable[fLastRuleStatusIndex];
    if (fillInArray != null) {  
        int numToCopy = Math.min(numStatusVals, fillInArray.length);
        for (int i=0; i<numToCopy; i++) {
            fillInArray[i] = fRData.fStatusTable[fLastRuleStatusIndex + i + 1];
        }
    }
    return numStatusVals;
 }


/**
 * Return a CharacterIterator over the text being analyzed.  This version
 * of this method returns the actual CharacterIterator we're using internally.
 * Changing the state of this iterator can have undefined consequences.  If
 * you need to change it, clone it first.
 * @return An iterator over the text being analyzed.
 * @stable ICU 2.0
 */
    public CharacterIterator getText() {
        return fText;
    }


    /**
     * Set the iterator to analyze a new piece of text.  This function resets
     * the current iteration position to the beginning of the text.
     * @param newText An iterator over the text to analyze.
     * @stable ICU 2.0
     */
    public void setText(CharacterIterator newText) {
        fText = newText;
        this.first();
    }
    
    /**
     * Control debug, trace and dump options.
     * @internal
     * @deprecated This API is ICU internal only.
     */
    protected static String fDebugEnv = ICUDebug.enabled(RBBI_DEBUG_ARG) ?
                                        ICUDebug.value(RBBI_DEBUG_ARG) : null;

    
    // 32 bit Char value returned from when an iterator has run out of range.
    //     Positive value so fast case (not end, not surrogate) can be checked
    //     with a single test.
    private static int CI_DONE32 = 0x7fffffff;
    
    /**
     * Move the iterator forward to the next code point, and return that code point,
     *   leaving the iterator positioned at char returned.
     *   For Supplementary chars, the iterator is left positioned at the lead surrogate.
     * @param ci  The character iterator
     * @return    The next code point.
     */
     static int CINext32(CharacterIterator ci) {
        // If the current position is at a surrogate pair, move to the trail surrogate
        //   which leaves it in positon for underlying iterator's next() to work.
        int c= ci.current();
        if (c >= UTF16.LEAD_SURROGATE_MIN_VALUE && c<=UTF16.LEAD_SURROGATE_MAX_VALUE) {
            c = ci.next();   
            if (c<UTF16.TRAIL_SURROGATE_MIN_VALUE || c>UTF16.TRAIL_SURROGATE_MAX_VALUE) {
               c = ci.previous();   
            }
        }

        // For BMP chars, this next() is the real deal.
        c = ci.next();
        
        // If we might have a lead surrogate, we need to peak ahead to get the trail 
        //  even though we don't want to really be positioned there.
        if (c >= UTF16.LEAD_SURROGATE_MIN_VALUE) {
            c = CINextTrail32(ci, c);   
        }
        
        if (c >= UTF16.SUPPLEMENTARY_MIN_VALUE && c != CI_DONE32) {
            // We got a supplementary char.  Back the iterator up to the postion
            // of the lead surrogate.
            ci.previous();   
        }
        return c;
   }

    
    // Out-of-line portion of the in-line Next32 code.
    // The call site does an initial ci.next() and calls this function
    //    if the 16 bit value it gets is >= LEAD_SURROGATE_MIN_VALUE.
    // NOTE:  we leave the underlying char iterator positioned in the
    //        middle of a surroage pair.  ci.next() will work correctly
    //        from there, but the ci.getIndex() will be wrong, and needs
    //        adjustment.
    private static int CINextTrail32(CharacterIterator ci, int lead) {
        int retVal = lead;
        if (lead <= UTF16.LEAD_SURROGATE_MAX_VALUE) {
            char  cTrail = ci.next();
            if (UTF16.isTrailSurrogate(cTrail)) {
                retVal = ((lead  - UTF16.LEAD_SURROGATE_MIN_VALUE) << 10) +
                            (cTrail - UTF16.TRAIL_SURROGATE_MIN_VALUE) +
                            UTF16.SUPPLEMENTARY_MIN_VALUE;
            } else {
                ci.previous();
            }
        } else {
            if (lead == CharacterIterator.DONE && ci.getIndex() >= ci.getEndIndex()) {
                retVal = CI_DONE32;
            }
        }
        return retVal;
    }
       
    private static int CIPrevious32(CharacterIterator ci) {
        if (ci.getIndex() <= ci.getBeginIndex()) {
            return CI_DONE32;   
        }
        char trail = ci.previous();
        int retVal = trail;
        if (UTF16.isTrailSurrogate(trail) && ci.getIndex()>ci.getBeginIndex()) {
            char lead = ci.previous();
            if (UTF16.isLeadSurrogate(lead)) {
                retVal = (((int)lead  - UTF16.LEAD_SURROGATE_MIN_VALUE) << 10) +
                          ((int)trail - UTF16.TRAIL_SURROGATE_MIN_VALUE) +
                          UTF16.SUPPLEMENTARY_MIN_VALUE;
            } else {
                ci.next();
            }           
        }
        return retVal;
    }
   
    static int CICurrent32(CharacterIterator ci) {
        char  lead   = ci.current();
        int   retVal = lead;
        if (retVal < UTF16.LEAD_SURROGATE_MIN_VALUE) {
            return retVal;   
        }
        if (UTF16.isLeadSurrogate(lead)) {
            int  trail = (int)ci.next();
            ci.previous();
            if (UTF16.isTrailSurrogate((char)trail)) {
                retVal = ((lead  - UTF16.LEAD_SURROGATE_MIN_VALUE) << 10) +
                         (trail - UTF16.TRAIL_SURROGATE_MIN_VALUE) +
                         UTF16.SUPPLEMENTARY_MIN_VALUE;
            }
         } else {
            if (lead == CharacterIterator.DONE) {
                if (ci.getIndex() >= ci.getEndIndex())   {
                    retVal = CI_DONE32;   
                }
            }
         }
        return retVal;
    }
    

    //-----------------------------------------------------------------------------------
    //
    //      handleNext(void)    All forward iteration vectors through this function.
    //                          NOTE:  This function is overridden by the dictionary base break iterator.
    //                                 User level API functions go to the dbbi implementation
    //                                     when the break iterator type is dbbi.
    //                                 The DBBI implementation sometimes explicitly calls back to here, 
    //                                     its inherited handleNext().
    //                      
    //-----------------------------------------------------------------------------------
    int handleNext() {
        return handleNext(fRData.fFTable);
    }

    /**
     * The State Machine Engine for moving forward is here.
     * This function is the heart of the RBBI run time engine.
     * 
     * @param stateTable
     * @return the new iterator position
     * 
     * A note on supplementary characters and the position of underlying
     * Java CharacterIterator:   Normally, a character iterator is positioned at
     * the char most recently returned by next().  Within this function, when
     * a supplementary char is being processed, the char iterator is left
     * sitting on the trail surrogate, in the middle of the code point.
     * This is different from everywhere else, where an iterator always
     * points at the lead surrogate of a supplementary.
     */
    private int handleNext(short stateTable[]) {
        int               state;
        short             category        = 0;
        int               mode;
        int               row;
        int               c;
        int               lookaheadStatus = 0;
        int               lookaheadTagIdx = 0;
        int               result          = 0;
        int               initialPosition = 0;
        int               lookaheadResult = 0;
        boolean          lookAheadHardBreak = 
            (stateTable[RBBIDataWrapper.FLAGS+1] & RBBIDataWrapper.RBBI_LOOKAHEAD_HARD_BREAK) != 0;
        
        if (fTrace) {
            System.out.println("Handle Next   pos      char  state category");
        }

        // No matter what, handleNext alway correctly sets the break tag value.
        fLastStatusIndexValid = true;
        fLastRuleStatusIndex  = 0;

        // if we're already at the end of the text, return DONE.
        if (fText == null) {
            fLastRuleStatusIndex = 0;
            return BreakIterator.DONE;
        }

        // Set up the starting char
        initialPosition = fText.getIndex();
        result          = initialPosition;
        c               = fText.current();
        if (c >= UTF16.LEAD_SURROGATE_MIN_VALUE) {
            c = CINextTrail32(fText, c);
            if (c == CI_DONE32) {
                fLastRuleStatusIndex = 0;
                return BreakIterator.DONE;
            }
        }

        // Set the initial state for the state machine
        state           = START_STATE;
        row             = fRData.getRowIndex(state); 
        category        = 3;
        mode            = RBBI_RUN;
        if ((stateTable[RBBIDataWrapper.FLAGS+1] & RBBIDataWrapper.RBBI_BOF_REQUIRED) != 0) {
            category = 2;
            mode     = RBBI_START;
        }


        // loop until we reach the end of the text or transition to state 0
        while (state != STOP_STATE) {
            if (c == CI_DONE32) {
                // Reached end of input string.
                if (mode == RBBI_END) {
                    // We have already run the loop one last time with the
                    // character set to the pseudo {eof} value. Now it is time
                    // to unconditionally bail out.

                    if (lookaheadResult > result) {
                        // We ran off the end of the string with a pending
                        // look-ahead match.
                        // Treat this as if the look-ahead condition had been
                        // met, and return
                        // the match at the / position from the look-ahead rule.
                        result = lookaheadResult;
                        fLastRuleStatusIndex = lookaheadTagIdx;
                        lookaheadStatus = 0;
                    } else if (result == initialPosition) {
                        // Ran off end, no match found.
                        // move forward one
                        fText.setIndex(initialPosition);
                        CINext32(fText);
                    }
                    break;
                }
                // Run the loop one last time with the fake end-of-input character category
                mode = RBBI_END;
                category = 1;
            }
            
            // Get the char category.  An incoming category of 1 or 2 mens that
            //      we are preset for doing the beginning or end of input, and
            //      that we shouldn't get a category from an actual text input character.
            //
            if (mode == RBBI_RUN) {
                // look up the current character's character category, which tells us
                // which column in the state table to look at.
                //
                category = (short) fRData.fTrie.getCodePointValue(c);
                
                // Check the dictionary bit in the character's category.
                //    Counter is only used by dictionary based iterators (subclasses).
                //    Chars that need to be handled by a dictionary have a flag bit set
                //    in their category values.
                //
                if ((category & 0x4000) != 0)  {
                    fDictionaryCharCount++;
                    //  And off the dictionary flag bit.
                    category &= ~0x4000;
                }
           }

            if (fTrace) {
                System.out.print("            " +  RBBIDataWrapper.intToString(fText.getIndex(), 5)); 
                System.out.print(RBBIDataWrapper.intToHexString(c, 10));
                System.out.println(RBBIDataWrapper.intToString(state,7) + RBBIDataWrapper.intToString(category,6));
            }

            // look up a state transition in the state table
            //     state = row->fNextState[category];
            state = stateTable[row + RBBIDataWrapper.NEXTSTATES + category];
            row   = fRData.getRowIndex(state);  

            // Advance to the next character.  
            // If this is a beginning-of-input loop iteration, don't advance.
            //    The next iteration will be processing the first real input character.
            if (mode == RBBI_RUN) {
                c = (int)fText.next(); 
                if (c >= UTF16.LEAD_SURROGATE_MIN_VALUE) {
                    c = CINextTrail32(fText, c);
                }
            } else {
                if (mode == RBBI_START) {
                    mode = RBBI_RUN;
                }
            }
             
            if (stateTable[row + RBBIDataWrapper.ACCEPTING] == -1) {
                // Match found, common case
                result = fText.getIndex();
                if (c >= UTF16.SUPPLEMENTARY_MIN_VALUE && c != CI_DONE32) {
                    // The iterator has been left in the middle of a surrogate pair.
                    // We want the start of it.
                    result--;
                }

                //  Remember the break status (tag) values.
                fLastRuleStatusIndex = stateTable[row + RBBIDataWrapper.TAGIDX];
            }

            if (stateTable[row + RBBIDataWrapper.LOOKAHEAD] != 0) {
                if (lookaheadStatus != 0
                    && stateTable[row + RBBIDataWrapper.ACCEPTING] == lookaheadStatus) {
                    // Lookahead match is completed.  Set the result accordingly, but only
                    // if no other rule has matched further in the mean time.
                    result               = lookaheadResult;
                    fLastRuleStatusIndex = lookaheadTagIdx;
                    lookaheadStatus      = 0;
                    // TODO: make a standalone hard break in a rule work.
                    if (lookAheadHardBreak) {
                        return result;
                    }
                    // Look-ahead completed, but other rules may match further.  Continue on.
                    //   TODO:  junk this feature?  I don't think it's used anywhere.
                    continue;
                }

                lookaheadResult = fText.getIndex();
                if (c>=UTF16.SUPPLEMENTARY_MIN_VALUE && c!=CI_DONE32) {
                    // The iterator has been left in the middle of a surrogate pair.
                    // We want the beginning  of it.
                    lookaheadResult--;
                }
                lookaheadStatus = stateTable[row + RBBIDataWrapper.LOOKAHEAD];
                lookaheadTagIdx = stateTable[row + RBBIDataWrapper.TAGIDX];
                continue;
            }


            if (stateTable[row + RBBIDataWrapper.ACCEPTING] != 0) {
                // Because this is an accepting state, any in-progress look-ahead match
                //   is no longer relavant.  Clear out the pending lookahead status.
                lookaheadStatus = 0; 
            }
            
         }        // End of state machine main loop

        // The state machine is done.  Check whether it found a match...

        // If the iterator failed to advance in the match engine, force it ahead by one.
        //   (This really indicates a defect in the break rules.  They should always match
        //    at least one character.)
        if (result == initialPosition) {
            result = fText.setIndex(initialPosition);
            CINext32(fText);
            result = fText.getIndex();
        }

        // Leave the iterator at our result position.
        //   (we may have advanced beyond the last accepting position chasing after
        //    longer matches that never completed.)
        fText.setIndex(result);
        if (fTrace) {
            System.out.println("result = " + result);
        }
        return result;
    }

    
    private int handlePrevious(short stateTable[]) {
        int            state;
        int            category           = 0;
        int            mode;
        int            row;        
        int            c;
        int            lookaheadStatus    = 0;
        int            result             = 0;
        int            initialPosition    = 0;
        int            lookaheadResult    = 0;
        boolean        lookAheadHardBreak = 
            (stateTable[RBBIDataWrapper.FLAGS+1] & RBBIDataWrapper.RBBI_LOOKAHEAD_HARD_BREAK) != 0;
        
        
        if (fText == null || stateTable == null) {
            return 0;
        }
        // handlePrevious() never gets the rule status.
        // Flag the status as invalid; if the user ever asks for status, we will need
        // to back up, then re-find the break position using handleNext(), which does
        // get the status value.
        fLastStatusIndexValid = false;
        fLastRuleStatusIndex  = 0;
        
        // set up the starting char
        initialPosition = fText.getIndex();
        result          = initialPosition;
        c               = CIPrevious32(fText);
        
        // Set up the initial state for the state machine
        state = START_STATE;
        row = fRData.getRowIndex(state);
        category = 3;   // TODO:  obsolete?  from the old start/run mode scheme?
        mode     = RBBI_RUN;
        if ((stateTable[RBBIDataWrapper.FLAGS+1] & RBBIDataWrapper.RBBI_BOF_REQUIRED) != 0) {
            category = 2;
            mode     = RBBI_START;
        }
        
        if (fTrace) {
            System.out.println("Handle Prev   pos   char  state category ");
        }
        
        // loop until we reach the beginning of the text or transition to state 0
        //
        mainLoop: for (;;) {
            innerBlock: {
                if (c == CI_DONE32) {
                    // Reached end of input string.
                    if (mode == RBBI_END || fRData.fHeader.fVersion == 1) {
                        // Either this is the old (ICU 3.2 and earlier) format data which
                        // does not support explicit support for matching {eof}, or
                        // we have already done the {eof} iteration.  Now is the time
                        // to unconditionally bail out.
                        if (lookaheadResult < result) {
                            // We ran off the end of the string with a pending look-ahead match.
                            // Treat this as if the look-ahead condition had been met, and return
                            //  the match at the / position from the look-ahead rule.
                            result = lookaheadResult;
                            lookaheadStatus = 0;
                        } else if (result == initialPosition) {
                            // Ran off start, no match found.
                            // Move one position (towards the start, since we are doing previous.)
                            fText.setIndex(initialPosition);
                            CIPrevious32(fText);
                        }
                        break mainLoop;
                    }
                    mode = RBBI_END;
                    category = 1;
                }
                
                if (mode == RBBI_RUN) {
                    // look up the current character's category, which tells us
                    // which column in the state table to look at.
                    //
                    category = (short) fRData.fTrie.getCodePointValue(c);
                    
                    // Check the dictionary bit in the character's category.
                    //    Counter is only used by dictionary based iterators (subclasses).
                    //    Chars that need to be handled by a dictionary have a flag bit set
                    //    in their category values.
                    //
                    if ((category & 0x4000) != 0)  {
                        fDictionaryCharCount++;
                        //  And off the dictionary flag bit.
                        category &= ~0x4000;
                    }
                }
                
                
                if (fTrace) {
                    System.out.print("             " + fText.getIndex() + "   ");
                    if (0x20 <= c && c < 0x7f) {
                        System.out.print("  " + c + "  ");
                    } else {
                        System.out.print(" " + Integer.toHexString(c) + " ");
                    }
                    System.out.println(" " + state + "  " + category + " ");
                }
                
                // State Transition - move machine to its next state
                //
                state = stateTable[row + RBBIDataWrapper.NEXTSTATES + category];
                row = fRData.getRowIndex(state);
                
                if (stateTable[row + RBBIDataWrapper.ACCEPTING] == -1) {
                    // Match found, common case, could have lookahead so we move
                    // on to check it
                    result = fText.getIndex();
                }
                
                if (stateTable[row + RBBIDataWrapper.LOOKAHEAD] != 0) {
                    if (lookaheadStatus != 0
                            && stateTable[row + RBBIDataWrapper.ACCEPTING] == lookaheadStatus) {
                        // Lookahead match is completed. Set the result
                        // accordingly, but only
                        // if no other rule has matched further in the mean
                        // time.
                        result = lookaheadResult;
                        lookaheadStatus = 0;
                        // TODO: make a standalone hard break in a rule work.
                        
                        if (lookAheadHardBreak) {
                            break mainLoop;
                        }
                        // Look-ahead completed, but other rules may match further.
                        // Continue on.
                        // TODO: junk this feature?  I don't think that it's used anywhere.
                        break innerBlock;
                    }
                    // Hit a possible look-ahead match. We are at the
                    // position of the '/'. Remember this position.
                    lookaheadResult = fText.getIndex();
                    lookaheadStatus = stateTable[row + RBBIDataWrapper.LOOKAHEAD];
                    break innerBlock;
                } 
                
                // not lookahead...
                if (stateTable[row + RBBIDataWrapper.ACCEPTING] != 0) {
                    // This is a plain (non-look-ahead) accepting state.
                    if (!lookAheadHardBreak) {
                        // Clear out any pending look-ahead matches,
                        // but only if not doing the lookAheadHardBreak option
                        // which needs to force a break no matter what is going
                        // on with the rest of the match, i.e. we can't abandon
                        // a partially completed look-ahead match because
                        // some other rule matched further than the '/' position
                        // in the look-ahead match.
                        lookaheadStatus = 0; 
                    }
                }
                
            } // end of innerBlock.  "break innerBlock" in above code comes out here.
        
        
            if (state == STOP_STATE) {
                // Normal loop exit is here
                break mainLoop;
            }
        
            // then move iterator position backwards one character
            //
            if (mode == RBBI_RUN) {
                c = CIPrevious32(fText);
            } else {
                if (mode == RBBI_START) {
                    mode = RBBI_RUN;
                }
            }
        
        
        }   // End of the main loop.
        
        // The state machine is done.  Check whether it found a match...
        //
        // If the iterator failed to advance in the match engine, force it ahead by one.
        //   (This really indicates a defect in the break rules.  They should always match
        //    at least one character.)
        if (result == initialPosition) {
            result = fText.setIndex(initialPosition);
            CIPrevious32(fText);
            result = fText.getIndex();
        }
        
        fText.setIndex(result);
        if (fTrace) {
            System.out.println("Result = " + result);
        }
        
        return result;
    }


    //-------------------------------------------------------------------------------
    
    //
    
    //  isDictionaryChar      Return true if the category lookup for this char
    
    //                        indicates that it is in the set of dictionary lookup
    
    //                        chars.
    
    //
    
    //                        This function is intended for use by dictionary based
    
    //                        break iterators.
    
    //
    
    //-------------------------------------------------------------------------------
    
    boolean isDictionaryChar(int c) {
    
        short  category = (short) fRData.fTrie.getCodePointValue(c);
    
        return (category & 0x4000) != 0;
    
    }

}
//eof