/**
*******************************************************************************
* Copyright (C) 1996-2003, International Business Machines Corporation and    *
* others. All Rights Reserved.                                                *
*******************************************************************************
*
* $Source: /xsrl/Nsvn/icu/icu4j/src/com/ibm/icu/text/CollationRuleParser.java,v $ 
* $Date: 2003/06/03 18:49:33 $ 
* $Revision: 1.15 $
*
*******************************************************************************
*/
package com.ibm.icu.text;

import java.text.ParseException;
import java.util.Hashtable;
import java.util.Arrays;
import com.ibm.icu.lang.UCharacter;
import com.ibm.icu.impl.UCharacterProperty;

/**
* Class for parsing collation rules, produces a list of tokens that will be 
* turned into collation elements
* @author Syn Wee Quek
* @since release 2.2, June 7 2002
* @draft 2.2
*/
final class CollationRuleParser
{     
	// public data members ---------------------------------------------------
	
	// package private constructors ------------------------------------------
	
	/**
     * <p>RuleBasedCollator constructor that takes the rules. 
     * Please see RuleBasedCollator class description for more details on the 
     * collation rule syntax.</p>
     * @see java.util.Locale
     * @param rules the collation rules to build the collation table from.
     * @exception ParseException thrown when argument rules have an invalid 
     *            syntax.
     * @draft 2.2
     */
    CollationRuleParser(String rules) throws ParseException
    {
        extractSetsFromRules(rules);
        m_source_ = new StringBuffer(Normalizer.decompose(rules, false).trim());
        m_rules_ = m_source_.toString();
        m_current_ = 0;
        m_extraCurrent_ = m_source_.length();
        m_variableTop_ = null;
        m_parsedToken_ = new ParsedToken();
        m_hashTable_ = new Hashtable();
        m_options_ = new OptionSet(RuleBasedCollator.UCA_);
        m_listHeader_ = new TokenListHeader[512];
        m_resultLength_ = 0;
        // call assembleTokenList() manually, so that we can 
        // init a parser and manually parse tokens
		//assembleTokenList();
    }
    
	// package private inner classes -----------------------------------------
    
    /**
     * Collation options set
     */
    static class OptionSet 
    {
	    // package private constructor ---------------------------------------
	    
	    /**
	     * Initializes the option set with the argument collators
	     * @param collator option to use
	     */
	    OptionSet(RuleBasedCollator collator) 
	    {
		    m_variableTopValue_ = collator.m_variableTopValue_;
		    m_isFrenchCollation_ = collator.isFrenchCollation();
		    m_isAlternateHandlingShifted_ 
		                           = collator.isAlternateHandlingShifted();
		    m_caseFirst_ = collator.m_caseFirst_;
		    m_isCaseLevel_ = collator.isCaseLevel();
		    m_decomposition_ = collator.getDecomposition();
		    m_strength_ = collator.getStrength();
		    m_isHiragana4_ = collator.m_isHiragana4_;
	    }
	    
	    // package private data members --------------------------------------
	    
	    int m_variableTopValue_;
	    boolean m_isFrenchCollation_;
	    /** 
	     * Attribute for handling variable elements
	     */
	    boolean m_isAlternateHandlingShifted_; 
	    /** 
	     * who goes first, lower case or uppercase 
	     */
	    int m_caseFirst_;         
	    /** 
	     * do we have an extra case level 
	     */
	    boolean m_isCaseLevel_;         
	    /** 
	     * attribute for normalization 
	     */
	    int m_decomposition_; 
	    /** 
	     * attribute for strength
	     */
	    int m_strength_;          
	    /** 
	     * attribute for special Hiragana 
	     */
	    boolean m_isHiragana4_;         
    };
    
    /**
     * List of tokens used by the collation rules
     */
    static class TokenListHeader
    {
        Token m_first_;
        Token m_last_;
        Token m_reset_;
        boolean m_indirect_;
        int m_baseCE_;
        int m_baseContCE_;
        int m_nextCE_;
        int m_nextContCE_;
        int m_previousCE_;
        int m_previousContCE_;
        int m_pos_[] = new int[Collator.IDENTICAL + 1];
        int m_gapsLo_[] = new int[3 * (Collator.TERTIARY + 1)];
        int m_gapsHi_[] = new int[3 * (Collator.TERTIARY + 1)];
        int m_numStr_[] = new int[3 * (Collator.TERTIARY + 1)];
        Token m_fStrToken_[] = new Token[Collator.TERTIARY + 1];
        Token m_lStrToken_[] = new Token[Collator.TERTIARY + 1];
    };
    
    /**
     * Token wrapper for collation rules
     */   
    static class Token 
    {
       // package private data members ---------------------------------------
       
       int m_CE_[];
       int m_CELength_;
       int m_expCE_[];
       int m_expCELength_;
       int m_source_;
       int m_expansion_;
       int m_prefix_;
       int m_strength_;
       int m_toInsert_;
       int m_polarity_; // 1 for <, <<, <<<, , ; and 0 for >, >>, >>>
       TokenListHeader m_listHeader_;
       Token m_previous_;
       Token m_next_;
       StringBuffer m_rules_;
       
       // package private constructors ---------------------------------------
       
       Token() 
       {
           m_CE_ = new int[128];
           m_expCE_ = new int[128];
           // TODO: this should also handle reverse
           m_polarity_ = TOKEN_POLARITY_POSITIVE_; 
           m_next_ = null;
           m_previous_ = null;
           m_CELength_ = 0;
           m_expCELength_ = 0;
       }
                
       // package private methods --------------------------------------------
       
       /**
        * Hashcode calculation for token
        * @return the hashcode
        */
       public int hashCode()
       {
           int result = 0;
	       int len = (m_source_ & 0xFF000000) >>> 24;
	       int inc = ((len - 32) / 32) + 1;
	        
	       int start = m_source_ & 0x00FFFFFF;
	       int limit = start + len;    
	
	       while (start < limit) {
	           result = (result * 37) + m_rules_.charAt(start);
	           start += inc;
	       }
	       return result;
	   }
	
	   /**
	    * Equals calculation
	    * @param target object to compare
	    * @return true if target is the same as this object
	    */
	   public boolean equals(Object target)
	   {
	       if (target == this) {
	           return true;
	       }
	       if (target instanceof Token) {
	           Token t = (Token)target;
			   int sstart = m_source_ & 0x00FFFFFF;
			   int tstart = t.m_source_ & 0x00FFFFFF;
			   int slimit = (m_source_ & 0xFF000000) >> 24;
			   int tlimit = (m_source_ & 0xFF000000) >> 24;
			   
			   int end = sstart + slimit - 1;
			
			   if (m_source_ == 0 || t.m_source_ == 0) {
			       return false;
			   }
			   if (slimit != tlimit) {
			       return false;
			   }
			   if (m_source_ == t.m_source_) {
			       return true;
			   }
               
			   while (sstart < end 
			          && m_rules_.charAt(sstart) == t.m_rules_.charAt(tstart)) 
	           {
	               ++ sstart;
	               ++ tstart;
	           }
			   if (m_rules_.charAt(sstart) == t.m_rules_.charAt(tstart)) {
			       return true;
			   }
	       }    
	       return false;
	    }
    };
    
    // package private data member -------------------------------------------
    
    /**
     * Indicator that the token is resetted yet, ie & in the rules
     */
    static final int TOKEN_RESET_ = 0xDEADBEEF;
    
    /**
     * Size of the number of tokens
     */
    int m_resultLength_;
    /**
     * List of parsed tokens
     */
    TokenListHeader m_listHeader_[];
    /**
     * Variable top token
     */
    Token m_variableTop_;
    /**
     * Collation options
     */
    OptionSet m_options_;
    /**
     * Normalized collation rules with some extra characters
     */
    StringBuffer m_source_;
    /**
     * Hash table to keep all tokens
     */
    Hashtable m_hashTable_;
    
    // package private method ------------------------------------------------
    
    void setDefaultOptionsInCollator(RuleBasedCollator collator)
    {
        collator.m_defaultStrength_ = m_options_.m_strength_;
        collator.m_defaultDecomposition_ = m_options_.m_decomposition_;
        collator.m_defaultIsFrenchCollation_ = m_options_.m_isFrenchCollation_;
        collator.m_defaultIsAlternateHandlingShifted_ 
                                    = m_options_.m_isAlternateHandlingShifted_;
        collator.m_defaultIsCaseLevel_ = m_options_.m_isCaseLevel_;
        collator.m_defaultCaseFirst_ = m_options_.m_caseFirst_;
        collator.m_defaultIsHiragana4_ = m_options_.m_isHiragana4_;
        collator.m_defaultVariableTopValue_ = m_options_.m_variableTopValue_;
    }
    
    // private inner classes -------------------------------------------------
    
    /** 
     * This is a token that has been parsed but not yet processed. Used to 
     * reduce the number of arguments in the parser
     */
    private static class ParsedToken 
    {
        // private constructor ----------------------------------------------
        
        /**
         * Empty constructor
         */
        ParsedToken()
        {
            m_charsLen_ = 0;
            m_charsOffset_ = 0;
            m_extensionLen_ = 0;
            m_extensionOffset_ = 0;
            m_prefixLen_ = 0;
            m_prefixOffset_ = 0;
            m_flags_ = 0;
            m_strength_ = TOKEN_UNSET_;
        }
        
        // private data members ---------------------------------------------
        
        int m_strength_;
        int m_charsOffset_;
        int m_charsLen_;
        int m_extensionOffset_;
        int m_extensionLen_;
        int m_prefixOffset_;
        int m_prefixLen_;
        char m_flags_;
        char m_indirectIndex_;
    };
    
    /**
     * Boundary wrappers
     */
    private static class IndirectBoundaries 
    {
        // package private constructor ---------------------------------------
        
        IndirectBoundaries(int startce[], int limitce[]) 
        {
            // Set values for the top - TODO: once we have values for all the 
            // indirects, we are going to initalize here. 
            m_startCE_ = startce[0];
            m_startContCE_ = startce[1];
            if (limitce != null) {
                m_limitCE_ = limitce[0];
                m_limitContCE_ = limitce[1];
            } 
            else { 
                m_limitCE_ = 0;
                m_limitContCE_ = 0; 
            } 
        }
        
        // package private data members --------------------------------------
        
	    int m_startCE_;
	    int m_startContCE_;
	    int m_limitCE_;
	    int m_limitContCE_;
    };

    /**
     * Collation option rule tag
     */
    private static class TokenOption
    {
    	// package private constructor ---------------------------------------
    	
    	TokenOption(String name, int attribute, String suboptions[],
    	            int suboptionattributevalue[]) 
    	{
    		m_name_ = name;
    		m_attribute_ = attribute;
        	m_subOptions_ = suboptions;
        	m_subOptionAttributeValues_ = suboptionattributevalue;		
    	}
    	
    	// package private data member ---------------------------------------
    	
    	private String m_name_;
    	private int m_attribute_;
        private String m_subOptions_[];
        private int m_subOptionAttributeValues_[];
    };
    
    // private variables -----------------------------------------------------

    /**
     * Current parsed token
     */
    private ParsedToken m_parsedToken_;
    /**
     * Collation rule
     */
    private String m_rules_;
    private int m_current_;
    /**
     * End of the option while reading. 
     * Need it for UnicodeSet reading support. 
     */
    private int m_optionEnd_;
    /**
     * Current offset in m_source
     */
    private int m_sourceLimit_;
    /**
     * Offset to m_source_ ofr the extra expansion characters
     */
    private int m_extraCurrent_;
    
    /** 
     * UnicodeSet that contains code points to be copied from the UCA
     */
    UnicodeSet m_copySet_;
    
    /**
     * UnicodeSet that contains code points for which we want to remove
     * UCA contractions. It implies copying of these code points from 
     * the UCA.
     */
    UnicodeSet m_removeSet_;
    /** 
     * This is space for the extra strings that need to be unquoted during the 
     * parsing of the rules 
     */
    private static final int TOKEN_EXTRA_RULE_SPACE_SIZE_ = 2048;
    /**
     * Indicator that the token is not set yet
     */
    private static final int TOKEN_UNSET_ = 0xFFFFFFFF;
    /**
     * Indicator that the rule is in the > polarity, ie everything on the 
     * right of the rule is less than
     */
    private static final int TOKEN_POLARITY_NEGATIVE_ = 0;
    /**
     * Indicator that the rule is in the < polarity, ie everything on the 
     * right of the rule is greater than
     */
    private static final int TOKEN_POLARITY_POSITIVE_ = 1;
    /**
     * Flag mask to determine if top is set
     */
    private static final int TOKEN_TOP_MASK_ = 0x04;
    /**
     * Flag mask to determine if variable top is set
     */
    private static final int TOKEN_VARIABLE_TOP_MASK_ = 0x08;
    /**
     * Flag mask to determine if a before attribute is set
     */
    private static final int TOKEN_BEFORE_ = 0x03;
    /**
     * For use in parsing token options
     */
    private static final int TOKEN_SUCCESS_MASK_ = 0x10;
    
    /** 
     * These values are used for finding CE values for indirect positioning. 
     * Indirect positioning is a mechanism for allowing resets on symbolic 
     * values. It only works for resets and you cannot tailor indirect names.
     * An indirect name can define either an anchor point or a range. An anchor 
     * point behaves in exactly the same way as a code point in reset would, 
     * except that it cannot be tailored. A range (we currently only know for 
     * the [top] range will explicitly set the upper bound for generated CEs, 
     * thus allowing for better control over how many CEs can be squeezed 
     * between in the range without performance penalty. In that respect, we use 
     * [top] for tailoring of locales that use CJK characters. Other indirect 
     * values are currently a pure convenience, they can be used to assure that 
     * the CEs will be always positioned in the same place relative to a point 
     * with known properties (e.g. first primary ignorable). 
     */
    private static final IndirectBoundaries INDIRECT_BOUNDARIES_[];
   
    /**
     * Inverse UCA constants
     */
    private static final int INVERSE_SIZE_MASK_ = 0xFFF00000;
    private static final int INVERSE_OFFSET_MASK_ = 0x000FFFFF;
    private static final int INVERSE_SHIFT_VALUE_ = 20;
    
    /**
     * Collation option tags
     * [last variable] last variable value 
     * [last primary ignorable] largest CE for primary ignorable 
     * [last secondary ignorable] largest CE for secondary ignorable 
     * [last tertiary ignorable] largest CE for tertiary ignorable 
     * [top] guaranteed to be above all implicit CEs, for now and in the future (in 1.8) 
	 */
	private static final TokenOption RULES_OPTIONS_[];
	
	static 
	{
        INDIRECT_BOUNDARIES_ = new IndirectBoundaries[15];
        // UCOL_RESET_TOP_VALUE
        INDIRECT_BOUNDARIES_[0] = new IndirectBoundaries(
                        RuleBasedCollator.UCA_CONSTANTS_.LAST_NON_VARIABLE_, 
                        RuleBasedCollator.UCA_CONSTANTS_.FIRST_IMPLICIT_);
        // UCOL_FIRST_PRIMARY_IGNORABLE
        INDIRECT_BOUNDARIES_[1] = new IndirectBoundaries(
                    RuleBasedCollator.UCA_CONSTANTS_.FIRST_PRIMARY_IGNORABLE_, 
                    null);
        // UCOL_LAST_PRIMARY_IGNORABLE
        INDIRECT_BOUNDARIES_[2] = new IndirectBoundaries(
                    RuleBasedCollator.UCA_CONSTANTS_.LAST_PRIMARY_IGNORABLE_, 
                    null);
          
        // UCOL_FIRST_SECONDARY_IGNORABLE
        INDIRECT_BOUNDARIES_[3] = new IndirectBoundaries(
                   RuleBasedCollator.UCA_CONSTANTS_.FIRST_SECONDARY_IGNORABLE_, 
                   null);
        // UCOL_LAST_SECONDARY_IGNORABLE
        INDIRECT_BOUNDARIES_[4] = new IndirectBoundaries(
                   RuleBasedCollator.UCA_CONSTANTS_.LAST_SECONDARY_IGNORABLE_, 
                   null);
        // UCOL_FIRST_TERTIARY_IGNORABLE 
        INDIRECT_BOUNDARIES_[5] = new IndirectBoundaries(
                   RuleBasedCollator.UCA_CONSTANTS_.FIRST_TERTIARY_IGNORABLE_, 
                   null);
        // UCOL_LAST_TERTIARY_IGNORABLE
        INDIRECT_BOUNDARIES_[6] = new IndirectBoundaries(
                   RuleBasedCollator.UCA_CONSTANTS_.LAST_TERTIARY_IGNORABLE_, 
                   null);
        // UCOL_FIRST_VARIABLE;
        INDIRECT_BOUNDARIES_[7] = new IndirectBoundaries(
                   RuleBasedCollator.UCA_CONSTANTS_.FIRST_VARIABLE_, 
                   null);  
        // UCOL_LAST_VARIABLE 
        INDIRECT_BOUNDARIES_[8] = new IndirectBoundaries(
                   RuleBasedCollator.UCA_CONSTANTS_.LAST_VARIABLE_, 
                   null);
        // UCOL_FIRST_NON_VARIABLE
        INDIRECT_BOUNDARIES_[9] = new IndirectBoundaries(
                   RuleBasedCollator.UCA_CONSTANTS_.FIRST_NON_VARIABLE_, 
                   null);
        // UCOL_LAST_NON_VARIABLE
        INDIRECT_BOUNDARIES_[10] = new IndirectBoundaries(
                   RuleBasedCollator.UCA_CONSTANTS_.LAST_NON_VARIABLE_, 
                   RuleBasedCollator.UCA_CONSTANTS_.FIRST_IMPLICIT_);
        // UCOL_FIRST_IMPLICIT 
        INDIRECT_BOUNDARIES_[11] = new IndirectBoundaries(
                   RuleBasedCollator.UCA_CONSTANTS_.FIRST_IMPLICIT_,
                   null); 
        // UCOL_LAST_IMPLICIT 
        INDIRECT_BOUNDARIES_[12] = new IndirectBoundaries(
                   RuleBasedCollator.UCA_CONSTANTS_.LAST_IMPLICIT_, 
                   RuleBasedCollator.UCA_CONSTANTS_.FIRST_TRAILING_); 
        // UCOL_FIRST_TRAILING 
        INDIRECT_BOUNDARIES_[13] = new IndirectBoundaries(
                   RuleBasedCollator.UCA_CONSTANTS_.FIRST_TRAILING_, 
                   null); 
        // UCOL_LAST_TRAILING 
        INDIRECT_BOUNDARIES_[14] = new IndirectBoundaries(
                   RuleBasedCollator.UCA_CONSTANTS_.LAST_TRAILING_, 
                   null); 
        INDIRECT_BOUNDARIES_[14].m_limitCE_ 
                 = RuleBasedCollator.UCA_CONSTANTS_.PRIMARY_SPECIAL_MIN_ << 24; 

		RULES_OPTIONS_ = new TokenOption[19];
		String option[] = {"non-ignorable", "shifted"};
		int value[] = {RuleBasedCollator.AttributeValue.NON_IGNORABLE_, 
	                   RuleBasedCollator.AttributeValue.SHIFTED_};
	    RULES_OPTIONS_[0] = new TokenOption("alternate", 
	                          RuleBasedCollator.Attribute.ALTERNATE_HANDLING_, 
	                          option, value);
        option = new String[1];
        option[0] = "2";
        value = new int[1];
        value[0] = RuleBasedCollator.AttributeValue.ON_;
        RULES_OPTIONS_[1] = new TokenOption("backwards", 
	                             RuleBasedCollator.Attribute.FRENCH_COLLATION_,
                                 option, value);
	    String offonoption[] = new String[2];
        offonoption[0] = "off"; 
        offonoption[1] = "on";
        int offonvalue[] = new int[2];
        offonvalue[0] = RuleBasedCollator.AttributeValue.OFF_; 
        offonvalue[1] = RuleBasedCollator.AttributeValue.ON_;
        RULES_OPTIONS_[2] = new TokenOption("caseLevel", 
                                       RuleBasedCollator.Attribute.CASE_LEVEL_,
                                       offonoption, offonvalue);
	    option = new String[3];
        option[0] = "lower"; 
        option[1] = "upper";
        option[2] = "off";
        value = new int[3];
        value[0] = RuleBasedCollator.AttributeValue.LOWER_FIRST_;
        value[1] = RuleBasedCollator.AttributeValue.UPPER_FIRST_;
        value[2] = RuleBasedCollator.AttributeValue.OFF_;
        RULES_OPTIONS_[3] = new TokenOption("caseFirst", 
                                       RuleBasedCollator.Attribute.CASE_FIRST_, 
                                       option, value);
        RULES_OPTIONS_[4] = new TokenOption("normalization", 
                               RuleBasedCollator.Attribute.NORMALIZATION_MODE_,
        	                   offonoption, offonvalue);
	    RULES_OPTIONS_[5] = new TokenOption("hiraganaQ", 
	                     RuleBasedCollator.Attribute.HIRAGANA_QUATERNARY_MODE_,
	    	             offonoption, offonvalue);
        option = new String[5];
        option[0] = "1"; 
        option[1] = "2";
        option[2] = "3";
        option[3] = "4"; 
        option[4] = "I";
        value = new int[5];
        value[0] = RuleBasedCollator.AttributeValue.PRIMARY_;
        value[1] = RuleBasedCollator.AttributeValue.SECONDARY_;
        value[2] = RuleBasedCollator.AttributeValue.TERTIARY_;
        value[3] = RuleBasedCollator.AttributeValue.QUATERNARY_;
        value[4] = RuleBasedCollator.AttributeValue.IDENTICAL_;
	    RULES_OPTIONS_[6] = new TokenOption("strength",
                                         RuleBasedCollator.Attribute.STRENGTH_, 
                                         option, value);
        RULES_OPTIONS_[7] = new TokenOption("variable top", 
	                              RuleBasedCollator.Attribute.LIMIT_, 
                                  null, null);
	    RULES_OPTIONS_[8] = new TokenOption("rearrange", 
	                              RuleBasedCollator.Attribute.LIMIT_, 
                                  null, null);
        option = new String[3];
        option[0] = "1";
        option[1] = "2";
        option[2] = "3";
        value = new int[3];
        value[0] = RuleBasedCollator.AttributeValue.PRIMARY_;
        value[1] = RuleBasedCollator.AttributeValue.SECONDARY_;
        value[2] = RuleBasedCollator.AttributeValue.TERTIARY_;
	    RULES_OPTIONS_[9] = new TokenOption("before", 
                                  RuleBasedCollator.Attribute.LIMIT_, 
                                  option, value);
	    RULES_OPTIONS_[10] = new TokenOption("top", 
                                  RuleBasedCollator.Attribute.LIMIT_, 
                                  null, null);
        String firstlastoption[] = new String[7];
        firstlastoption[0] = "primary";
        firstlastoption[1] = "secondary";
        firstlastoption[2] = "tertiary";
        firstlastoption[3] = "variable";
        firstlastoption[4] = "regular";
        firstlastoption[5] = "implicit";
        firstlastoption[6] = "trailing";

        int firstlastvalue[] = new int[7];
        Arrays.fill(firstlastvalue, RuleBasedCollator.AttributeValue.PRIMARY_);

	    RULES_OPTIONS_[11] = new TokenOption("first", 
                                  RuleBasedCollator.Attribute.LIMIT_, 
	    	                      firstlastoption, firstlastvalue);
        RULES_OPTIONS_[12] = new TokenOption("last", 
                                  RuleBasedCollator.Attribute.LIMIT_, 
	    	                      firstlastoption, firstlastvalue);
        RULES_OPTIONS_[13] = new TokenOption("optimize", 
                                  RuleBasedCollator.Attribute.LIMIT_, 
                                  null, null);
        RULES_OPTIONS_[14] = new TokenOption("suppressContractions", 
                                  RuleBasedCollator.Attribute.LIMIT_, 
                                  null, null);                                  
	    RULES_OPTIONS_[15] = new TokenOption("undefined", 
	                              RuleBasedCollator.Attribute.LIMIT_, 
                                  null, null);
	    RULES_OPTIONS_[16] = new TokenOption("scriptOrder", 
	                              RuleBasedCollator.Attribute.LIMIT_, 
                                  null, null);
	    RULES_OPTIONS_[17] = new TokenOption("charsetname", 
	                              RuleBasedCollator.Attribute.LIMIT_, 
                                  null, null);
	    RULES_OPTIONS_[18] = new TokenOption("charset", 
	                              RuleBasedCollator.Attribute.LIMIT_, 
                                  null, null);
	};
    
    /**
     * Utility data members
     */
    private Token m_utilToken_ = new Token();
    private CollationElementIterator m_UCAColEIter_ 
                      = RuleBasedCollator.UCA_.getCollationElementIterator("");
    private int m_utilCEBuffer_[] = new int[2];
    
    // private methods -------------------------------------------------------

    /**
     * Assembles the token list
     * @param
     * @exception ParseException thrown when rules syntax fails
     */
    int assembleTokenList() throws ParseException
    {
		Token lastToken = null;
	    m_parsedToken_.m_strength_ = TOKEN_UNSET_; 
        int sourcelimit = m_source_.length();
        int expandNext = 0;
		
        while (m_current_ < sourcelimit) {
            m_parsedToken_.m_prefixOffset_ = 0;
		    if (parseNextToken(lastToken == null) < 0) {
                // we have reached the end
                continue;
            }
		    char specs = m_parsedToken_.m_flags_;
		    boolean variableTop = ((specs & TOKEN_VARIABLE_TOP_MASK_) != 0);
		    boolean top = ((specs & TOKEN_TOP_MASK_) != 0);
		    int lastStrength = TOKEN_UNSET_;
            if (lastToken != null) {
                lastStrength = lastToken.m_strength_;
            }
            m_utilToken_.m_source_ = m_parsedToken_.m_charsLen_ << 24 
                                             | m_parsedToken_.m_charsOffset_;
            m_utilToken_.m_rules_ = m_source_;
            // 4 Lookup each source in the CharsToToken map, and find a 
            // sourcetoken
            Token sourceToken = (Token)m_hashTable_.get(m_utilToken_);
            if (m_parsedToken_.m_strength_ != TOKEN_RESET_) {
		        if (lastToken == null) { 
		        	// this means that rules haven't started properly
		            throwParseException(m_source_.toString(), 0);
		        }
                //  6 Otherwise (when relation != reset)
                if (sourceToken == null) {
                    // If sourceToken is null, create new one
		            sourceToken = new Token();
     		        sourceToken.m_rules_ = m_source_;
		            sourceToken.m_source_ = m_parsedToken_.m_charsLen_ << 24 
		                                   | m_parsedToken_.m_charsOffset_;
		            sourceToken.m_prefix_ = m_parsedToken_.m_prefixLen_ << 24 
		                                   | m_parsedToken_.m_prefixOffset_;
		            // TODO: this should also handle reverse
		            sourceToken.m_polarity_ = TOKEN_POLARITY_POSITIVE_; 
    		        sourceToken.m_next_ = null;
 		            sourceToken.m_previous_ = null;
		            sourceToken.m_CELength_ = 0;
		            sourceToken.m_expCELength_ = 0;
		            m_hashTable_.put(sourceToken, sourceToken);
		        } 
		        else {
		            // we could have fished out a reset here
		            if (sourceToken.m_strength_ != TOKEN_RESET_ 
		                && lastToken != sourceToken) {
		                // otherwise remove sourceToken from where it was.
		                if (sourceToken.m_next_ != null) {
		                    if (sourceToken.m_next_.m_strength_ 
		                                           > sourceToken.m_strength_) {
		                        sourceToken.m_next_.m_strength_ 
		                                           = sourceToken.m_strength_;
		                    }
		                    sourceToken.m_next_.m_previous_ 
		                                            = sourceToken.m_previous_;
		                } 
		                else {
		                    sourceToken.m_listHeader_.m_last_ 
		                                            = sourceToken.m_previous_;
		                }
		                if (sourceToken.m_previous_ != null) {
		                    sourceToken.m_previous_.m_next_ 
		                                                = sourceToken.m_next_;
		                } 
		                else {
		                    sourceToken.m_listHeader_.m_first_ 
		                                                = sourceToken.m_next_;
		                }
		                sourceToken.m_next_ = null;
		                sourceToken.m_previous_ = null;
		            }
		        }
        		sourceToken.m_strength_ = m_parsedToken_.m_strength_;
                sourceToken.m_listHeader_ = lastToken.m_listHeader_;
                
                // 1.  Find the strongest strength in each list, and set 
                // strongestP and strongestN accordingly in the headers. 
                if (lastStrength == TOKEN_RESET_
                    || sourceToken.m_listHeader_.m_first_ == null) {
                    // If LAST is a reset insert sourceToken in the list.
			        if (sourceToken.m_listHeader_.m_first_ == null) {
			            sourceToken.m_listHeader_.m_first_ = sourceToken;
			            sourceToken.m_listHeader_.m_last_ = sourceToken;
			        } 
			        else { // we need to find a place for us
			               // and we'll get in front of the same strength
			            if (sourceToken.m_listHeader_.m_first_.m_strength_ 
			                                     <= sourceToken.m_strength_) {
			                sourceToken.m_next_ 
			                              = sourceToken.m_listHeader_.m_first_;
			                sourceToken.m_next_.m_previous_ = sourceToken;
			                sourceToken.m_listHeader_.m_first_ = sourceToken;
			                sourceToken.m_previous_ = null;
			            } 
			            else {
			                lastToken = sourceToken.m_listHeader_.m_first_;
			                while (lastToken.m_next_ != null 
			                       && lastToken.m_next_.m_strength_ 
			                                     > sourceToken.m_strength_) {
			                    lastToken = lastToken.m_next_;
			                }
			                if (lastToken.m_next_ != null) {
			                    lastToken.m_next_.m_previous_ = sourceToken;
			                } 
			                else {
			                    sourceToken.m_listHeader_.m_last_ 
			                                                   = sourceToken;
			                }
			                sourceToken.m_previous_ = lastToken;
			                sourceToken.m_next_ = lastToken.m_next_;
			                lastToken.m_next_ = sourceToken;
			            }
			        }
                } 
                else {
                    // Otherwise (when LAST is not a reset) 
                    // if polarity (LAST) == polarity(relation), insert 
                    // sourceToken after LAST, otherwise insert before. 
                    // when inserting after or before, search to the next 
                    // position with the same strength in that direction. 
                    // (This is called postpone insertion).        
			        if (sourceToken != lastToken) { 
			            if (lastToken.m_polarity_ == sourceToken.m_polarity_) {
			                while (lastToken.m_next_ != null 
			                       && lastToken.m_next_.m_strength_ 
			                                       > sourceToken.m_strength_) {
			                    lastToken = lastToken.m_next_;
			                }
			                sourceToken.m_previous_ = lastToken;
			                if (lastToken.m_next_ != null) {
			                    lastToken.m_next_.m_previous_ = sourceToken;
			                } 
			                else {
			                    sourceToken.m_listHeader_.m_last_ = sourceToken;
			                }
			                sourceToken.m_next_ = lastToken.m_next_;
			                lastToken.m_next_ = sourceToken;
			            } 
			            else {
			                while (lastToken.m_previous_ != null 
			                       && lastToken.m_previous_.m_strength_ 
		                                        > sourceToken.m_strength_) {
			                    lastToken = lastToken.m_previous_;
			                }
			                sourceToken.m_next_ = lastToken;
			                if (lastToken.m_previous_ != null) {
			                    lastToken.m_previous_.m_next_ = sourceToken;
			                } 
			                else {
			                    sourceToken.m_listHeader_.m_first_ 
			                                                     = sourceToken;
			                }
			                sourceToken.m_previous_ = lastToken.m_previous_;
			                lastToken.m_previous_ = sourceToken;
			            }
			        } 
			        else { // repeated one thing twice in rules, stay with the 
			        	   // stronger strength 
			            if (lastStrength < sourceToken.m_strength_) {
			                sourceToken.m_strength_ = lastStrength;
			            }
			        }
                }
                // if the token was a variable top, we're gonna put it in 
                if (variableTop == true && m_variableTop_ == null) {
                    variableTop = false;
                    m_variableTop_ = sourceToken;
                }
                // Treat the expansions.
                // There are two types of expansions: explicit (x / y) and 
                // reset based propagating expansions 
                // (&abc * d * e <=> &ab * d / c * e / c) 
                // if both of them are in effect for a token, they are combined.
               sourceToken.m_expansion_ = m_parsedToken_.m_extensionLen_ << 24 
                                          | m_parsedToken_.m_extensionOffset_;
               if (expandNext != 0) {
		           if (sourceToken.m_strength_ == RuleBasedCollator.PRIMARY) { 
		               // primary strength kills off the implicit expansion 
		               expandNext = 0;
		           } 
		           else if (sourceToken.m_expansion_ == 0) { 
		               // if there is no expansion, implicit is just added to 
		               // the token 
		               sourceToken.m_expansion_ = expandNext;
		           } 
		           else { 
		               // there is both explicit and implicit expansion. 
		               // We need to make a combination
		               int start = expandNext & 0xFFFFFF;
                       int size = expandNext >>> 24;
                       if (size > 0) {
		                  m_source_.append(m_source_.substring(start,
                                                               start + size));
                       }
   		               start = m_parsedToken_.m_extensionOffset_;
		               m_source_.append(m_source_.substring(start, 
		                              start + m_parsedToken_.m_extensionLen_));
		               sourceToken.m_expansion_ = (size 
		                               + m_parsedToken_.m_extensionLen_) << 24 
		                               | m_extraCurrent_;
		               m_extraCurrent_ += size + m_parsedToken_.m_extensionLen_;
		           }
		        }
            } 
            else {
	            if (lastToken != null && lastStrength == TOKEN_RESET_) {
	                // if the previous token was also a reset, this means that 
	                // we have two consecutive resets and we want to remove the 
	                // previous one if empty
	                if (m_listHeader_[m_resultLength_ - 1].m_first_ == null) {
	                    m_resultLength_ --;
	                }
	            }
	            if (sourceToken == null) { 
	                // this is a reset, but it might still be somewhere in the 
	                // tailoring, in shorter form
	                int searchCharsLen = m_parsedToken_.m_charsLen_;
	                while (searchCharsLen > 1 && sourceToken == null) {
	                    searchCharsLen --;
	                    // key = searchCharsLen << 24 | charsOffset;
			            m_utilToken_.m_source_ = searchCharsLen << 24 
			                                 | m_parsedToken_.m_charsOffset_;
			            m_utilToken_.m_rules_ = m_source_;
			            sourceToken = (Token)m_hashTable_.get(m_utilToken_);
			        }
			        if (sourceToken != null) {
			            expandNext = (m_parsedToken_.m_charsLen_ 
			                                          - searchCharsLen) << 24 
			                            | (m_parsedToken_.m_charsOffset_ 
			                               + searchCharsLen);
			        }
			    }
		        if ((specs & TOKEN_BEFORE_) != 0) {
                    if (top == false) { 
    		            // we're doing before & there is no indirection
    	                int strength = (specs & TOKEN_BEFORE_) - 1;
    	                if (sourceToken != null 
    	                    && sourceToken.m_strength_ != TOKEN_RESET_) { 
    	                    // this is a before that is already ordered in the UCA 
    	                    // - so we need to get the previous with good strength
    			            while (sourceToken.m_strength_ > strength 
    			                   && sourceToken.m_previous_ != null) {
    			                sourceToken = sourceToken.m_previous_;
    			            }
    			            // here, either we hit the strength or NULL
    			            if (sourceToken.m_strength_ == strength) {
    			                if (sourceToken.m_previous_ != null) {
    			                    sourceToken = sourceToken.m_previous_;
    			                } 
    			                else { // start of list
    			                    sourceToken 
    			                         = sourceToken.m_listHeader_.m_reset_;
    			                }              
    			            } 
    			            else { // we hit NULL, we should be doing the else part 
    			                sourceToken 
                                         = sourceToken.m_listHeader_.m_reset_;
    			                sourceToken = getVirginBefore(sourceToken, 
    			                                              strength);
    			            }
    			        } 
    			        else {
    			            sourceToken 
                                      = getVirginBefore(sourceToken, strength);
                        } 
                    }
                    else { 
                        // this is both before and indirection
                        top = false; 
                        m_listHeader_[m_resultLength_] = new TokenListHeader();
                        m_listHeader_[m_resultLength_].m_previousCE_ = 0; 
                        m_listHeader_[m_resultLength_].m_previousContCE_ = 0; 
                        m_listHeader_[m_resultLength_].m_indirect_ = true; 
                        // we need to do slightly more work. we need to get the 
                        // baseCE using the inverse UCA & getPrevious. The next 
                        // bound is not set, and will be decided in ucol_bld
                        int strength = (specs & TOKEN_BEFORE_) - 1; 
                        int baseCE = INDIRECT_BOUNDARIES_[
                                   m_parsedToken_.m_indirectIndex_].m_startCE_;
                        int baseContCE = INDIRECT_BOUNDARIES_[
                               m_parsedToken_.m_indirectIndex_].m_startContCE_;
                        int ce[] = new int[2]; 
                        CollationParsedRuleBuilder.InverseUCA invuca
                                     = CollationParsedRuleBuilder.INVERSE_UCA_;
                        invuca.getInversePrevCE(baseCE, baseContCE, strength, 
                                                ce);
                        m_listHeader_[m_resultLength_].m_baseCE_ = ce[0];
                        m_listHeader_[m_resultLength_].m_baseContCE_ = ce[1];
                        m_listHeader_[m_resultLength_].m_nextCE_ = 0;
                        m_listHeader_[m_resultLength_].m_nextContCE_ = 0;
                        
                        sourceToken = new Token();
                        expandNext = initAReset(0, sourceToken);    
    			    }
			    }
	            // 5 If the relation is a reset: 
	            // If sourceToken is null 
	            // Create new list, create new sourceToken, make the baseCE 
	            // from source, put the sourceToken in ListHeader of the new 
	            // list
	            if (sourceToken == null) {
                    if (m_listHeader_[m_resultLength_] == null) {
	            	    m_listHeader_[m_resultLength_] = new TokenListHeader();
                    }
	                // 3 Consider each item: relation, source, and expansion: 
	                // e.g. ...< x / y ... 
	                // First convert all expansions into normal form. 
	                // Examples: 
	                // If "xy" doesn't occur earlier in the list or in the UCA, 
	                // convert &xy * c * d * ... into &x * c/y * d * ... 
	                // Note: reset values can never have expansions, although 
	                // they can cause the very next item to have one. They may 
	                // be contractions, if they are found earlier in the list. 
			        if (top == false) {
			            CollationElementIterator coleiter 
                        = RuleBasedCollator.UCA_.getCollationElementIterator(
                        	m_source_.substring(m_parsedToken_.m_charsOffset_, 
                                            	m_parsedToken_.m_charsOffset_ 
                                            	+ m_parsedToken_.m_charsLen_));
			
			            int CE = coleiter.next();
			            // offset to the character in the full rule string
			            int expand = coleiter.getOffset() 
			                         + m_parsedToken_.m_charsOffset_;
			            int SecondCE = coleiter.next();
			            
			            m_listHeader_[m_resultLength_].m_baseCE_ 
			                                                 = CE & 0xFFFFFF3F;
			            if (RuleBasedCollator.isContinuation(SecondCE)) {
			                m_listHeader_[m_resultLength_].m_baseContCE_ 
			                                                        = SecondCE;
			            } 
			            else {
			                m_listHeader_[m_resultLength_].m_baseContCE_ = 0;
			            }
			            m_listHeader_[m_resultLength_].m_nextCE_ = 0;
			            m_listHeader_[m_resultLength_].m_nextContCE_ = 0;
			            m_listHeader_[m_resultLength_].m_previousCE_ = 0;
			            m_listHeader_[m_resultLength_].m_previousContCE_ = 0;
			            m_listHeader_[m_resultLength_].m_indirect_ = false;
			            sourceToken = new Token();
			            expandNext = initAReset(expand, sourceToken);
			        } 
			        else { // top == TRUE
			            top = false;
			            m_listHeader_[m_resultLength_].m_previousCE_ = 0;
			            m_listHeader_[m_resultLength_].m_previousContCE_ = 0;
			            m_listHeader_[m_resultLength_].m_indirect_ = true;
			            IndirectBoundaries ib = INDIRECT_BOUNDARIES_[
                                              m_parsedToken_.m_indirectIndex_];
			            m_listHeader_[m_resultLength_].m_baseCE_ 
			                                                   = ib.m_startCE_;
			            m_listHeader_[m_resultLength_].m_baseContCE_ 
		                                                   = ib.m_startContCE_;
			            m_listHeader_[m_resultLength_].m_nextCE_ 
			                                                   = ib.m_limitCE_;
			            m_listHeader_[m_resultLength_].m_nextContCE_ 
			                                               = ib.m_limitContCE_;
			            sourceToken = new Token();
			            expandNext = initAReset(0, sourceToken);
	                }
		        } 
		        else { // reset to something already in rules
		            top = false;
		        }
		    }
		    // 7 After all this, set LAST to point to sourceToken, and goto 
		    // step 3.
		    lastToken = sourceToken;
		}
		  
		if (m_resultLength_ > 0 
            && m_listHeader_[m_resultLength_ - 1].m_first_ == null) {
		    m_resultLength_ --;
		}
		return m_resultLength_;
	}
     
    /**
     * Formats and throws a ParseException
     * @param rules collation rule that failed
     * @param offset failed offset in rules
     * @throws ParseException with failure information
     */
    private static final void throwParseException(String rules, int offset)
                                                          throws ParseException
    {
        // for pre-context
        String precontext = rules.substring(0, offset);
        String postcontext = rules.substring(offset, rules.length());
        StringBuffer error = new StringBuffer(
                                    "Parse error occurred in rule at offset "); 
        error.append(offset); 
        error.append("\n after the prefix \"");
        error.append(precontext);
        error.append("\" before the suffix \"");
        error.append(postcontext);
        throw new ParseException(error.toString(), offset);
    }
    
    private final boolean doSetTop() {
        m_parsedToken_.m_charsOffset_ = m_extraCurrent_;
        m_source_.append((char)0xFFFE);
        IndirectBoundaries ib = 
                  INDIRECT_BOUNDARIES_[m_parsedToken_.m_indirectIndex_];
        m_source_.append((char)(ib.m_startCE_ >> 16));
        m_source_.append((char)(ib.m_startCE_ & 0xFFFF));
        m_extraCurrent_ += 3;
        if (INDIRECT_BOUNDARIES_[m_parsedToken_.m_indirectIndex_
                                                       ].m_startContCE_ == 0) {
            m_parsedToken_.m_charsLen_ = 3;
        }
        else { 
            m_source_.append((char)(INDIRECT_BOUNDARIES_[
                                        m_parsedToken_.m_indirectIndex_
                                    ].m_startContCE_ >> 16)); 
            m_source_.append((char)(INDIRECT_BOUNDARIES_[
                                        m_parsedToken_.m_indirectIndex_
                                    ].m_startContCE_ & 0xFFFF)); 
            m_extraCurrent_ += 2;
            m_parsedToken_.m_charsLen_ = 5;
        } 
        return true;
    }
    
    /**
     * Getting the next token
     * @param startofrules flag indicating if we are at the start of rules
     * @return the offset of the rules
     * @exception ParseException thrown when rule parsing fails
     */
    private int parseNextToken(boolean startofrules) throws ParseException
    { 
	    // parsing part
	    boolean variabletop = false;
	    boolean top = false;
	    boolean inchars = true;
	    boolean inquote = false;
	    boolean wasinquote = false;
	    byte before = 0;
	    boolean isescaped = false;
	    int /*newcharslen = 0,*/ newextensionlen = 0;
	    int /*charsoffset = 0,*/ extensionoffset = 0;
	    int newstrength = TOKEN_UNSET_; 
	
        m_parsedToken_.m_charsLen_ = 0;
	    m_parsedToken_.m_charsOffset_ = 0;
        m_parsedToken_.m_prefixOffset_ = 0; 
        m_parsedToken_.m_prefixLen_ = 0;
	    m_parsedToken_.m_indirectIndex_ = 0;
	
	    int limit = m_rules_.length();
	    while (m_current_ < limit) {
	        char ch = m_source_.charAt(m_current_);
	        if (inquote) {
		        if (ch == 0x0027) { // '\''
		            inquote = false;
		        } 
                else {
		            if ((m_parsedToken_.m_charsLen_ == 0) || inchars) {
		                 if (m_parsedToken_.m_charsLen_ == 0) {
		                     m_parsedToken_.m_charsOffset_ = m_extraCurrent_;
		                 }
		                 m_parsedToken_.m_charsLen_ ++;
		            } 
                    else {
		                if (newextensionlen == 0) {
		                    extensionoffset = m_extraCurrent_;
		                }
		                newextensionlen ++;
		            }
		        }
	        }
            else if (isescaped) {
	            isescaped = false;
	            if (newstrength == TOKEN_UNSET_) {
	                throwParseException(m_rules_, m_current_);
	            }
	            if (ch != 0 && m_current_ != limit) {
	                if (inchars) {
	                    if (m_parsedToken_.m_charsLen_ == 0) {
	                        m_parsedToken_.m_charsOffset_ = m_current_;
	                    }
	                    m_parsedToken_.m_charsLen_ ++;
	                } 
                    else {
	                    if (newextensionlen == 0) {
	                        extensionoffset = m_current_;
	                    }
	                    newextensionlen ++;
	                }
	            }
	        }
            else {
                if (!UCharacterProperty.isRuleWhiteSpace(ch)) {
                    // Sets the strength for this entry
	                switch (ch) {
	                case 0x003D : // '='
	                    if (newstrength != TOKEN_UNSET_) {
	                        return doEndParseNextToken(newstrength, 
                                                       top, 
                                                       extensionoffset,
                                                       newextensionlen, 
                                                       variabletop, before);
	                    }
	                    // if we start with strength, we'll reset to top
	                    if (startofrules == true) {
                            m_parsedToken_.m_indirectIndex_ = 5;
                            top = doSetTop();
	                        return doEndParseNextToken(TOKEN_RESET_, 
                                                       top, 
                                                       extensionoffset, 
                                                       newextensionlen, 
                                                       variabletop, before);
	                    }
	                    newstrength = Collator.IDENTICAL;
	                    break;
	                case 0x002C : // ','
	                    if (newstrength != TOKEN_UNSET_) {
	                        return doEndParseNextToken(newstrength, 
                                                       top, 
                                                       extensionoffset, 
                                                       newextensionlen, 
                                                       variabletop, before);
	                    }
	                    // if we start with strength, we'll reset to top
	                    if (startofrules == true) {
                            m_parsedToken_.m_indirectIndex_ = 5;
                            top = doSetTop();
	                        return doEndParseNextToken(TOKEN_RESET_, 
                                                       top, 
                                                       extensionoffset, 
                                                       newextensionlen, 
                                                       variabletop, before);
	                    }
	                    newstrength = Collator.TERTIARY;
	                    break;
	                case 0x003B : // ';'
	                    if (newstrength != TOKEN_UNSET_) {
	                        return doEndParseNextToken(newstrength, 
                                                       top, 
                                                       extensionoffset, 
                                                       newextensionlen, 
                                                       variabletop, before);   
	                    }
	                    // if we start with strength, we'll reset to top
	                    if (startofrules == true) {
                            m_parsedToken_.m_indirectIndex_ = 5;
                            top = doSetTop();
                            return doEndParseNextToken(TOKEN_RESET_, 
                                                       top, 
                                                       extensionoffset, 
                                                       newextensionlen, 
                                                       variabletop, before);
	                    }
	                    newstrength = Collator.SECONDARY;
	                    break;
	                case 0x003C : // '<'
	                    if (newstrength != TOKEN_UNSET_) {
	                        return doEndParseNextToken(newstrength, 
                                                       top, 
                                                       extensionoffset, 
                                                       newextensionlen, 
                                                       variabletop, before);
	                    }
	                    // if we start with strength, we'll reset to top
		                if (startofrules == true) {
                            m_parsedToken_.m_indirectIndex_ = 5;
                            top = doSetTop();
                            return doEndParseNextToken(TOKEN_RESET_, 
                                                       top, 
                                                       extensionoffset, 
                                                       newextensionlen, 
                                                       variabletop, before);
		                }
		                // before this, do a scan to verify whether this is 
		                // another strength
		                if (m_source_.charAt(m_current_ + 1) == 0x003C) {
		                    m_current_ ++;
		                    if (m_source_.charAt(m_current_ + 1) == 0x003C) {
		                        m_current_ ++; // three in a row!
		                        newstrength = Collator.TERTIARY;
		                    } 
		                    else { // two in a row 
		                        newstrength = Collator.SECONDARY;
		                    }
		                } 
		                else { // just one 
		                    newstrength = Collator.PRIMARY;
		                }
		                break;
		            case 0x0026 : // '&'
		                if (newstrength != TOKEN_UNSET_) {
		                    return doEndParseNextToken(newstrength, 
                                                       top, 
                                                       extensionoffset, 
                                                       newextensionlen, 
                                                       variabletop, before);
		                }
		                newstrength = TOKEN_RESET_; // PatternEntry::RESET = 0
		                break;
		            case 0x005b : // '['
		                // options - read an option, analyze it
		                m_optionEnd_ = m_rules_.indexOf(0x005d, m_current_);
		                if (m_optionEnd_ != -1) { // ']'
		                    byte result = readAndSetOption();
		                    m_current_ = m_optionEnd_;
		                    if ((result & TOKEN_TOP_MASK_) != 0) {
		                        if (newstrength == TOKEN_RESET_) { 
                                    top = doSetTop();
                                    if (before != 0) { 
                                        // This is a combination of before and 
                                        // indirection like 
                                        // '&[before 2][first regular]<b' 
                                        m_source_.append((char)0x002d); 
                                        m_source_.append((char)before); 
                                        m_extraCurrent_ += 2; 
                                        m_parsedToken_.m_charsLen_ += 2;
                                    } 
		                            m_current_ ++;
		                            return doEndParseNextToken(newstrength, 
		                                               true, 
                                                       extensionoffset, 
                                                       newextensionlen, 
                                                       variabletop, before);
		                        } 
		                        else {
		                            throwParseException(m_rules_, m_current_);
		                        }
		                    } 
		                    else if ((result & TOKEN_VARIABLE_TOP_MASK_) != 0) {
		                        if (newstrength != TOKEN_RESET_ 
		                            && newstrength != TOKEN_UNSET_) {
		                            variabletop = true;
		                            m_parsedToken_.m_charsOffset_ 
                                                             = m_extraCurrent_;
		                            m_source_.append((char)0xFFFF);
		                            m_extraCurrent_ ++;
		                            m_current_ ++;
                                    m_parsedToken_.m_charsLen_ = 1;
		                            return doEndParseNextToken(newstrength, 
		                                               top, 
                                                       extensionoffset, 
                                                       newextensionlen, 
                                                       variabletop, before);
		                        } 
		                        else {
		                            throwParseException(m_rules_, m_current_);
		                        }
		                    } 
		                    else if ((result & TOKEN_BEFORE_) != 0){
		                        if (newstrength == TOKEN_RESET_) {
		                            before = (byte)(result & TOKEN_BEFORE_);
		                        } 
		                        else {
		                            throwParseException(m_rules_, m_current_);		
		                        }
		                    }  
		                }
		                break;
		            case 0x002F : // '/'
		                wasinquote = false; // if we were copying source 
		                                    // characters, we want to stop now
		                inchars = false; // we're now processing expansion 
		                break;
		            case 0x005C : // back slash for escaped chars
		                isescaped = true;
		                break;
		            // found a quote, we're gonna start copying
		            case 0x0027 : //'\''
		                if (newstrength == TOKEN_UNSET_) { 
		                    // quote is illegal until we have a strength
		                    throwParseException(m_rules_, m_current_);
                        }
		                inquote = true;
		                if (inchars) { // we're doing characters 
		                    if (wasinquote == false) {
		                        m_parsedToken_.m_charsOffset_ = m_extraCurrent_;
		                    }
		                    if (m_parsedToken_.m_charsLen_ != 0) {
		                    	m_source_.append(m_source_.substring(
                                       m_current_ - m_parsedToken_.m_charsLen_, 
                                       m_current_));
		                        m_extraCurrent_ += m_parsedToken_.m_charsLen_;
		                    }
		                    m_parsedToken_.m_charsLen_ ++;
		                } 
		                else { // we're doing an expansion
		                    if (wasinquote == false) {
		                        extensionoffset = m_extraCurrent_;
		                    }
		                    if (newextensionlen != 0) {
		                        m_source_.append(m_source_.substring(
		                                           m_current_ - newextensionlen,
		                                           m_current_));
		                        m_extraCurrent_ += newextensionlen;
		                    }
		                    newextensionlen ++;
		                }
		                wasinquote = true;
		                m_current_ ++;
		                ch = m_source_.charAt(m_current_); 
		                if (ch == 0x0027) { // copy the double quote
		                    m_source_.append(ch);
		                    m_extraCurrent_ ++;
		                    inquote = false;
		                }
		                break;
		            // '@' is french only if the strength is not currently set
		            // if it is, it's just a regular character in collation  
		            case 0x0040 : // '@'
		                if (newstrength == TOKEN_UNSET_) {
		                    m_options_.m_isFrenchCollation_ = true;
		                break;
		            }
		            case 0x007C : //|
		                // this means we have actually been reading prefix part 
		                // we want to store read characters to the prefix part 
		                // and continue reading the characters (proper way 
		                // would be to restart reading the chars, but in that 
		                // case we would have to complicate the token hasher, 
		                // which I do not intend to play with. Instead, we will 
		                // do prefixes when prefixes are due (before adding the 
		                // elements).
		                m_parsedToken_.m_prefixOffset_ 
                                                = m_parsedToken_.m_charsOffset_;
		                m_parsedToken_.m_prefixLen_ 
                                                = m_parsedToken_.m_charsLen_;
                        if (inchars) { // we're doing characters
		                    if (wasinquote == false) {
		                        m_parsedToken_.m_charsOffset_ = m_extraCurrent_;
		                    }
		                    if (m_parsedToken_.m_charsLen_ != 0) {
                                String prefix = m_source_.substring(
                                       m_current_ - m_parsedToken_.m_charsLen_, 
                                       m_current_);
		                    	m_source_.append(prefix);
		                        m_extraCurrent_ += m_parsedToken_.m_charsLen_;
		                    }
		                    m_parsedToken_.m_charsLen_ ++;
		                }
		                wasinquote = true;
                        do {
                            m_current_ ++;
                            ch = m_source_.charAt(m_current_); 
                            // skip whitespace between '|' and the character
                        } while (UCharacterProperty.isRuleWhiteSpace(ch));
		                break;
                    case 0x0021: // '!' // ignoring java set thai reordering
                        break;
	                default :
	                    if (newstrength == TOKEN_UNSET_) {
	                        throwParseException(m_rules_, m_current_);
	                    }
	                    if (isSpecialChar(ch) && (inquote == false)) {
	                        throwParseException(m_rules_, m_current_);
	                    }
	                    if (ch == 0x0000 && m_current_ + 1 == limit) {
	                        break;
	                    }
	                    if (inchars) {
	                        if (m_parsedToken_.m_charsLen_ == 0) {
	                            m_parsedToken_.m_charsOffset_ = m_current_;
	                        }
	                        m_parsedToken_.m_charsLen_++;
	                    } 
	                    else {
	                        if (newextensionlen == 0) {
	                            extensionoffset = m_current_;
	                        }
	                        newextensionlen ++;
	                    }	
	                    break;
	                }
	            }   
            }
	        if (wasinquote) {
	            if (ch != 0x27) {
	              	m_source_.append(ch);
	                m_extraCurrent_ ++;
	            }
	        }   
	        m_current_ ++;
	    }  
	    return doEndParseNextToken(newstrength, top, 
                                   extensionoffset, newextensionlen, 
                                   variabletop, before);
	}

    /**
     * End the next parse token
     * @param newstrength new strength
     * @return offset in rules, -1 for end of rules
     */
    private int doEndParseNextToken(int newstrength, /*int newcharslen,*/
                                    boolean top, /*int charsoffset,*/
                                    int extensionoffset, int newextensionlen, 
                                    boolean variabletop, int before)
                                    throws ParseException
    {
        if (newstrength == TOKEN_UNSET_) {
            return -1;
        }
        if (m_parsedToken_.m_charsLen_ == 0 && top == false) {
		    throwParseException(m_rules_, m_current_); 
		}
		
		m_parsedToken_.m_strength_ = newstrength; 
		//m_parsedToken_.m_charsOffset_ = charsoffset;
		//m_parsedToken_.m_charsLen_ = newcharslen;
		m_parsedToken_.m_extensionOffset_ = extensionoffset;
	    m_parsedToken_.m_extensionLen_ = newextensionlen;
		m_parsedToken_.m_flags_ = (char)
		                          ((variabletop ? TOKEN_VARIABLE_TOP_MASK_ : 0)
                                  | (top ? TOKEN_TOP_MASK_ : 0) | before);
		return m_current_;
    }
    
    /**
     * Token before this element
     * @param sourcetoken 
     * @param strength collation strength
     * @return the token before source token
     * @exception ParseException thrown when rules have the wrong syntax
     */
    private Token getVirginBefore(Token sourcetoken, int strength) 
                                                          throws ParseException
    {
	    // this is a virgin before - we need to fish the anchor from the UCA
	    if (sourcetoken != null) {
            int offset = sourcetoken.m_source_ & 0xFFFFFF;
            m_UCAColEIter_.setText(m_source_.substring(offset, offset + 1));
	    } 
	    else {
            m_UCAColEIter_.setText(
                             m_source_.substring(m_parsedToken_.m_charsOffset_, 
                             m_parsedToken_.m_charsOffset_ + 1));
	    }
	      	
	    int basece = m_UCAColEIter_.next() & 0xFFFFFF3F;
	    int basecontce = m_UCAColEIter_.next();
	    if (basecontce == CollationElementIterator.NULLORDER) {
	        basecontce = 0;
	    }
	    // first ce and second ce m_utilCEBuffer_
	    int invpos = CollationParsedRuleBuilder.INVERSE_UCA_.getInversePrevCE(
                                                     basece, basecontce, 
                                                     strength, m_utilCEBuffer_);
	    int ch = CollationParsedRuleBuilder.INVERSE_UCA_.m_table_[3 * invpos 
                                                                  + 2];
        if ((ch &  INVERSE_SIZE_MASK_) != 0) {
	        int offset = ch & INVERSE_OFFSET_MASK_;
	        ch = CollationParsedRuleBuilder.INVERSE_UCA_.m_continuations_[
                                                                       offset];
	    }      
	    m_source_.append((char)ch);
	    m_extraCurrent_ ++;
	    m_parsedToken_.m_charsOffset_ = m_extraCurrent_ - 1;
	    m_parsedToken_.m_charsLen_ = 1;
	
	    // We got an UCA before. However, this might have been tailored.
	    // example:
	    // &\u30ca = \u306a
	    // &[before 3]\u306a<<<\u306a|\u309d
	  
	    m_utilToken_.m_source_ = (m_parsedToken_.m_charsLen_ << 24) 
	                                         | m_parsedToken_.m_charsOffset_;
	    m_utilToken_.m_rules_ = m_source_;
	    sourcetoken = (Token)m_hashTable_.get(m_utilToken_);
	  
	    // if we found a tailored thing, we have to use the UCA value and 
	    // construct a new reset token with constructed name
	    if (sourcetoken != null && sourcetoken.m_strength_ != TOKEN_RESET_) {
	        // character to which we want to anchor is already tailored. 
	        // We need to construct a new token which will be the anchor point
            m_source_.setCharAt(m_extraCurrent_ - 1, '\uFFFE');
	        m_source_.append(ch);
	        m_extraCurrent_ ++;
	        m_parsedToken_.m_charsLen_ ++;
            m_listHeader_[m_resultLength_] = new TokenListHeader();
	        m_listHeader_[m_resultLength_].m_baseCE_ 
                                             = m_utilCEBuffer_[0] & 0xFFFFFF3F;
	        if (RuleBasedCollator.isContinuation(m_utilCEBuffer_[1])) {
	            m_listHeader_[m_resultLength_].m_baseContCE_ 
                                                          = m_utilCEBuffer_[1];
	        } 
	        else {
	            m_listHeader_[m_resultLength_].m_baseContCE_ = 0;
	        }
	        m_listHeader_[m_resultLength_].m_nextCE_ = 0;
	        m_listHeader_[m_resultLength_].m_nextContCE_ = 0;
	        m_listHeader_[m_resultLength_].m_previousCE_ = 0;
	        m_listHeader_[m_resultLength_].m_previousContCE_ = 0;
	        m_listHeader_[m_resultLength_].m_indirect_ = false;
	        sourcetoken = new Token();
	        initAReset(-1, sourcetoken);   
	    }
	    return sourcetoken;
	}
	
	/**
	 * Processing Description.
	 * 1. Build a m_listHeader_. Each list has a header, which contains two lists 
	 * (positive and negative), a reset token, a baseCE, nextCE, and 
	 * previousCE. The lists and reset may be null. 
	 * 2. As you process, you keep a LAST pointer that points to the last token 
	 * you handled. 
	 * @param expand string offset, -1 for null strings
	 * @param targetToken token to update
	 * @return expandnext offset
	 * @throws ParseException thrown when rules syntax failed
	 */
	private int initAReset(int expand, Token targetToken) throws ParseException
	{
        if (m_resultLength_ == m_listHeader_.length - 1) {
            // Unfortunately, this won't work, as we store addresses of lhs in 
            // token
            TokenListHeader temp[] = new TokenListHeader[m_resultLength_ << 1];
            System.arraycopy(m_listHeader_, 0, temp, 0, m_resultLength_ + 1);
            m_listHeader_ = temp;
        }
	    // do the reset thing
	    targetToken.m_rules_ = m_source_;
	    targetToken.m_source_ = m_parsedToken_.m_charsLen_ << 24 
	                            | m_parsedToken_.m_charsOffset_;
	    targetToken.m_expansion_ = m_parsedToken_.m_extensionLen_ << 24 
	                               | m_parsedToken_.m_extensionOffset_;
	    if (m_parsedToken_.m_prefixOffset_ != 0) {
	        throwParseException(m_rules_, m_parsedToken_.m_charsOffset_ - 1);
	    } 
	    
        targetToken.m_prefix_ = 0;
	    // TODO: this should also handle reverse
	    targetToken.m_polarity_ = TOKEN_POLARITY_POSITIVE_; 
	    targetToken.m_strength_ = TOKEN_RESET_;
	    targetToken.m_next_ = null;
	    targetToken.m_previous_ = null;
	    targetToken.m_CELength_ = 0;
	    targetToken.m_expCELength_ = 0;
	    targetToken.m_listHeader_ = m_listHeader_[m_resultLength_];
	    m_listHeader_[m_resultLength_].m_first_ = null;
	    m_listHeader_[m_resultLength_].m_last_ = null;
	    m_listHeader_[m_resultLength_].m_first_ = null;
	    m_listHeader_[m_resultLength_].m_last_ = null;
	    m_listHeader_[m_resultLength_].m_reset_ = targetToken;
	
	    /* 3 Consider each item: relation, source, and expansion: 
	     * e.g. ...< x / y ... 
	     * First convert all expansions into normal form. Examples: 
	     * If "xy" doesn't occur earlier in the list or in the UCA, convert 
	     * &xy * c * d * ... into &x * c/y * d * ... 
	     * Note: reset values can never have expansions, although they can 
	     * cause the very next item to have one. They may be contractions, if 
	     * they are found earlier in the list. 
	     */
	    int result = 0;
	    if (expand > 0) {
	        // check to see if there is an expansion
	        if (m_parsedToken_.m_charsLen_ > 1) {
	            targetToken.m_source_ = ((expand 
	                                      - m_parsedToken_.m_charsOffset_ ) 
	                                      << 24) 
	                                      | m_parsedToken_.m_charsOffset_;
	            result = ((m_parsedToken_.m_charsLen_ 
	                           + m_parsedToken_.m_charsOffset_ - expand) << 24) 
	                           | expand;
	        }
	    }   
	
	    m_resultLength_ ++;
	    m_hashTable_.put(targetToken, targetToken);
	    return result;
	}
	
	/**
	 * Checks if an character is special
	 * @param ch character to test
	 * @return true if the character is special
	 */
	private static final boolean isSpecialChar(char ch)
	{
        return (ch <= 0x002F && ch >= 0x0020) || (ch <= 0x003F && ch >= 0x003A) 
               || (ch <= 0x0060 && ch >= 0x005B) 
               || (ch <= 0x007E && ch >= 0x007D) || ch == 0x007B;
	}
    
    private
    UnicodeSet readAndSetUnicodeSet(String source, int start) throws ParseException 
    {
      while(source.charAt(start) != '[') { /* advance while we find the first '[' */
        start++;
      }
      // now we need to get a balanced set of '[]'. The problem is that a set can have 
      // many, and *end point to the first closing '['
      int noOpenBraces = 1;
      int current = 1; // skip the opening brace
      while(start+current < source.length() && noOpenBraces != 0) {
        if(source.charAt(start+current) == '[') {
          noOpenBraces++;
        } else if(source.charAt(start+current) == ']') { // closing brace
          noOpenBraces--;
        }
        current++;
      }
      //int nextBrace = -1;
    
      if(noOpenBraces != 0 || (/*nextBrace =*/ source.indexOf("]", start+current) /*']'*/) == -1) {
        throwParseException(m_rules_, start);
      }
      return new UnicodeSet(source.substring(start, start+current)); //uset_openPattern(start, current);
    }
    
    
    /** in C, optionarg is passed by reference to function.
     *  We use a private int to simulate this.
     */
    private int m_optionarg_ = 0;

	private int readOption(String rules, int start, int optionend) 
    {
        m_optionarg_ = 0;
        int i = 0;
        while (i < RULES_OPTIONS_.length) {
            String option = RULES_OPTIONS_[i].m_name_;
            int optionlength = option.length();
            if (rules.length() > start + optionlength
                && option.equalsIgnoreCase(rules.substring(start, 
                                                      start + optionlength))) {
                if (optionend - start > optionlength) {
                    m_optionarg_ = start + optionlength + 1; 
                    // start of the options, skip space
                    while (m_optionarg_ < optionend && UCharacter.isWhitespace(rules.charAt(m_optionarg_))) 
                    {   // eat whitespace
                        m_optionarg_ ++;
                    }
                }     
                break;
            }
            i ++;
        }
        if(i == RULES_OPTIONS_.length) {
            i = -1;
        }
        return i;
    }
	/**
	 * Reads and set collation options
	 * @param optionend offset to the end of the option in rules
	 * @return TOKEN_SUCCESS if option is set correct, 0 otherwise
     * @exception ParseException thrown when options in rules are wrong
	 */
	private byte readAndSetOption() throws ParseException
	{
		int start = m_current_ + 1; // skip opening '['
	    int i = readOption(m_rules_, start, m_optionEnd_);
        
        int optionarg = m_optionarg_;
	
	    if (i < 0) {
	        throwParseException(m_rules_, start);
	    }
	
	    if (i < 7) {
            if (optionarg != 0) {
	            for (int j = 0; j < RULES_OPTIONS_[i].m_subOptions_.length; 
                                                                        j ++) {
                     String subname = RULES_OPTIONS_[i].m_subOptions_[j];
                     int size = optionarg + subname.length();
                     if (m_rules_.length() > size 
                         && subname.equalsIgnoreCase(m_rules_.substring(
                                                           optionarg, size))) {
	                     setOptions(m_options_, RULES_OPTIONS_[i].m_attribute_, 
                             RULES_OPTIONS_[i].m_subOptionAttributeValues_[j]);
	                     return TOKEN_SUCCESS_MASK_;
                     }
	            }
	        }
	        throwParseException(m_rules_, optionarg);
	    } 
        else if (i == 7) { // variable top
	        return TOKEN_SUCCESS_MASK_ | TOKEN_VARIABLE_TOP_MASK_;
	    } 
        else if (i == 8) { // rearange
	        return TOKEN_SUCCESS_MASK_;
	    } 
        else if (i == 9) { // before
	        if (optionarg != 0) {
	            for (int j = 0; j < RULES_OPTIONS_[i].m_subOptions_.length; 
                                                                        j ++) {
                     String subname = RULES_OPTIONS_[i].m_subOptions_[j];
                     int size = optionarg + subname.length();
                     if (m_rules_.length() > size 
                         && subname.equalsIgnoreCase(
                                               m_rules_.substring(optionarg, 
                                              optionarg + subname.length()))) {
	                     return (byte)(TOKEN_SUCCESS_MASK_ 
                            | RULES_OPTIONS_[i].m_subOptionAttributeValues_[j] 
                            + 1);
                     }
	            }
	        }
	        throwParseException(m_rules_, optionarg);
	    } 
        else if (i == 10) {  // top, we are going to have an array with 
            // structures of limit CEs index to this array will be 
            // src->parsedToken.indirectIndex
	        m_parsedToken_.m_indirectIndex_ = 0;
	        return TOKEN_SUCCESS_MASK_ | TOKEN_TOP_MASK_;
	    } 
        else if (i < 13) { // first, last 
	        for (int j = 0; j < RULES_OPTIONS_[i].m_subOptions_.length; j ++) {     
	            String subname = RULES_OPTIONS_[i].m_subOptions_[j];
                int size = optionarg + subname.length();
                if (m_rules_.length() > size 
                    && subname.equalsIgnoreCase(m_rules_.substring(optionarg, 
                                                                   size))) {
	                m_parsedToken_.m_indirectIndex_ = (char)(i - 10 + (j << 1));         
	                return TOKEN_SUCCESS_MASK_ | TOKEN_TOP_MASK_;
	            }
	        }	        
            throwParseException(m_rules_, optionarg);
	    } 
        else if(i == 13 || i == 14) { // copy and remove are handled before normalization
            // we need to move end here
            int noOpenBraces = 1;
            m_current_++; // skip opening brace
            while(m_current_ < m_source_.length() && noOpenBraces != 0) {
                if(m_source_.charAt(m_current_) == '[') {
                  noOpenBraces++;
                } else if(m_source_.charAt(m_current_) == ']') { // closing brace
                  noOpenBraces--;
                }
                m_current_++;
            }
            m_optionEnd_ = m_current_-1;
            return TOKEN_SUCCESS_MASK_; 
        }
        else {
	        throwParseException(m_rules_, optionarg);
	    }
        return TOKEN_SUCCESS_MASK_; // we will never reach here.
	}

    /**
     * Set collation option
     * @param optionset option set to set
     * @param attribute type to set
     * @param value attribute value
     */
    private void setOptions(OptionSet optionset, int attribute, int value) 
    {
	    switch (attribute) {
	        case RuleBasedCollator.Attribute.HIRAGANA_QUATERNARY_MODE_ :
	            optionset.m_isHiragana4_ 
                            = (value == RuleBasedCollator.AttributeValue.ON_);
	            break;
	        case RuleBasedCollator.Attribute.FRENCH_COLLATION_ :
	            optionset.m_isFrenchCollation_ 
                             = (value == RuleBasedCollator.AttributeValue.ON_);
	            break;
	        case RuleBasedCollator.Attribute.ALTERNATE_HANDLING_ :
	            optionset.m_isAlternateHandlingShifted_ 
                             = (value 
                                == RuleBasedCollator.AttributeValue.SHIFTED_);
	            break;
	        case RuleBasedCollator.Attribute.CASE_FIRST_ :
	            optionset.m_caseFirst_ = value;
	            break;
	        case RuleBasedCollator.Attribute.CASE_LEVEL_ :
	            optionset.m_isCaseLevel_ 
                             = (value == RuleBasedCollator.AttributeValue.ON_);
	            break;
	        case RuleBasedCollator.Attribute.NORMALIZATION_MODE_ :
                if (value == RuleBasedCollator.AttributeValue.ON_) {
                    value = Collator.CANONICAL_DECOMPOSITION;
                }
	            optionset.m_decomposition_ = value;
	            break;
	        case RuleBasedCollator.Attribute.STRENGTH_ :
	            optionset.m_strength_ = value;
	            break;
	        default :
	            break;
	    }
  	}
  	
    UnicodeSet getTailoredSet() throws ParseException
    {	
        boolean startOfRules = true;
        UnicodeSet tailored = new UnicodeSet();
        String pattern;
        CanonicalIterator it = new CanonicalIterator("");
        
        m_parsedToken_.m_strength_ = TOKEN_UNSET_; 
        int sourcelimit = m_source_.length();
        //int expandNext = 0;
        
        while (m_current_ < sourcelimit) {
        m_parsedToken_.m_prefixOffset_ = 0;
        if (parseNextToken(startOfRules) < 0) {
            // we have reached the end
            continue;
        }
        startOfRules = false;
        // The idea is to tokenize the rule set. For each non-reset token,
        // we add all the canonicaly equivalent FCD sequences 
            if(m_parsedToken_.m_strength_ != TOKEN_RESET_) {		    	
                it.setSource(m_source_.substring(
                  	m_parsedToken_.m_charsOffset_, 
                  	m_parsedToken_.m_charsOffset_+m_parsedToken_.m_charsLen_));
                pattern = it.next();
                while(pattern != null) {
                  	if(Normalizer.quickCheck(pattern, Normalizer.FCD,0) != Normalizer.NO) {
                        tailored.add(pattern);
                    }
                    pattern = it.next();
                }
            }
        }
        return tailored;
    }
    
    final private void extractSetsFromRules(String rules) throws ParseException {
      int optionNumber = -1;
      int setStart = 0;
      int i = 0;
      while(i < rules.length()) {
        if(rules.charAt(i) == 0x005B) {
          optionNumber = readOption(rules, i+1, rules.length());
          setStart = m_optionarg_;
          if(optionNumber == 13) { /* copy - parts of UCA to tailoring */
            UnicodeSet newSet = readAndSetUnicodeSet(rules, setStart);
              if(m_copySet_ == null) {
                m_copySet_ = newSet;
              } else {
                m_copySet_.addAll(newSet);
              }
          } else if(optionNumber == 14) {
            UnicodeSet newSet = readAndSetUnicodeSet(rules, setStart);
              if(m_removeSet_ == null) {
                m_removeSet_ = newSet;
              } else {
                m_removeSet_.addAll(newSet);
              }
          }
        }
        i++;
      }        
    }
}