/*
**********************************************************************
*   Copyright (c) 2001, International Business Machines
*   Corporation and others.  All Rights Reserved.
**********************************************************************
* $Source: /xsrl/Nsvn/icu/icu4j/src/com/ibm/text/Attic/TransliteratorParser.java,v $
* $Date: 2001/10/24 00:03:38 $
* $Revision: 1.7 $
**********************************************************************
*/
package com.ibm.text;

import com.ibm.text.resources.ResourceReader;
import com.ibm.util.Utility;
import java.util.Stack;
import java.util.Vector;
import java.text.ParsePosition;

class TransliteratorParser {

    //----------------------------------------------------------------------
    // Data members
    //----------------------------------------------------------------------

    /**
     * PUBLIC data member containing the parsed data object, or null if
     * there were no rules.
     */
    public RuleBasedTransliterator.Data data;

    /**
     * PUBLIC data member.
     * The block of ::IDs, both at the top and at the bottom.
     * Inserted into these may be additional rules at the
     * idSplitPoint.
     */
    public String idBlock;

    /**
     * PUBLIC data member.
     * In a compound RBT, the index at which the RBT rules are
     * inserted into the ID block.  Index 0 means before any IDs
     * in the block.  Index idBlock.length() means after all IDs
     * in the block.  Index is a string index.
     */
    public int idSplitPoint;

    /**
     * PUBLIC data member containing the parsed compound filter, if any.
     */
    public UnicodeSet compoundFilter;


    // The number of rules parsed.  This tells us if there were
    // any actual transliterator rules, or if there were just ::ID
    // block IDs.
    private int ruleCount;

    private int direction;

    /**
     * Temporary symbol table used during parsing.
     */
    private ParseData parseData;

    /**
     * Temporary vector of set variables.  When parsing is complete, this
     * is copied into the array data.variables.  As with data.variables,
     * element 0 corresponds to character data.variablesBase.
     */
    private Vector variablesVector;

    /**
     * The next available stand-in for variables.  This starts at some point in
     * the private use area (discovered dynamically) and increments up toward
     * <code>variableLimit</code>.  At any point during parsing, available
     * variables are <code>variableNext..variableLimit-1</code>.
     */
    private char variableNext;

    /**
     * The last available stand-in for variables.  This is discovered
     * dynamically.  At any point during parsing, available variables are
     * <code>variableNext..variableLimit-1</code>.  During variable definition
     * we use the special value variableLimit-1 as a placeholder.
     */
    private char variableLimit;

    /**
     * When we encounter an undefined variable, we do not immediately signal
     * an error, in case we are defining this variable, e.g., "$a = [a-z];".
     * Instead, we save the name of the undefined variable, and substitute
     * in the placeholder char variableLimit - 1, and decrement
     * variableLimit.
     */
    private String undefinedVariableName;

    /**
     * The stand-in character for the 'dot' set, represented by '.' in
     * patterns.  This is allocated the first time it is needed, and
     * reused thereafter.
     */
    private int dotStandIn = -1;

    //----------------------------------------------------------------------
    // Constants
    //----------------------------------------------------------------------

    // Indicator for ID blocks
    private static final String ID_TOKEN = "::";
    private static final int ID_TOKEN_LEN = 2;

    // Operators
    private static final char VARIABLE_DEF_OP   = '=';
    private static final char FORWARD_RULE_OP   = '>';
    private static final char REVERSE_RULE_OP   = '<';
    private static final char FWDREV_RULE_OP    = '~'; // internal rep of <> op

    private static final String OPERATORS = "=><";

    // Other special characters
    private static final char QUOTE               = '\'';
    private static final char ESCAPE              = '\\';
    private static final char END_OF_RULE         = ';';
    private static final char RULE_COMMENT_CHAR   = '#';

    private static final char CONTEXT_ANTE        = '{'; // ante{key
    private static final char CONTEXT_POST        = '}'; // key}post
    private static final char CURSOR_POS          = '|';
    private static final char CURSOR_OFFSET       = '@';
    private static final char ANCHOR_START        = '^';

    private static final char KLEENE_STAR         = '*';
    private static final char ONE_OR_MORE         = '+';
    private static final char ZERO_OR_ONE         = '?';

    private static final char DOT                 = '.';
    private static final String DOT_SET           = "[^[:Zp:][:Zl:]\r\n$]";

    // By definition, the ANCHOR_END special character is a
    // trailing SymbolTable.SYMBOL_REF character.
    // private static final char ANCHOR_END       = '$';

    // Segments of the input string are delimited by "(" and ")".  In the
    // output string these segments are referenced as "$1" through "$9".
    private static final char SEGMENT_OPEN        = '(';
    private static final char SEGMENT_CLOSE       = ')';

    //----------------------------------------------------------------------
    // class ParseData
    //----------------------------------------------------------------------

    /**
     * This class implements the SymbolTable interface.  It is used
     * during parsing to give UnicodeSet access to variables that
     * have been defined so far.  Note that it uses variablesVector,
     * _not_ data.variables.
     */
    private class ParseData implements SymbolTable {

        /**
         * Implement SymbolTable API.
         */
        public char[] lookup(String name) {
            return (char[]) data.variableNames.get(name);
        }

        /**
         * Implement SymbolTable API.
         */
        public UnicodeSet lookupSet(int ch) {
            // Note that we cannot use data.lookupSet() because the
            // set array has not been constructed yet.
            int i = ch - data.variablesBase;
            if (i >= 0 && i < variablesVector.size()) {
                return (UnicodeSet) variablesVector.elementAt(i);
            }
            return null;
        }

        /**
         * Implement SymbolTable API.  Parse out a symbol reference
         * name.
         */
        public String parseReference(String text, ParsePosition pos, int limit) {
            int start = pos.getIndex();
            int i = start;
            while (i < limit) {
                char c = text.charAt(i);
                if ((i==start && !Character.isUnicodeIdentifierStart(c)) ||
                    !Character.isUnicodeIdentifierPart(c)) {
                    break;
                }
                ++i;
            }
            if (i == start) { // No valid name chars
                return null;
            }
            pos.setIndex(i);
            return text.substring(start, i);
        }
    }

    //----------------------------------------------------------------------
    // classes RuleBody, RuleArray, and RuleReader
    //----------------------------------------------------------------------

    /**
     * A private abstract class representing the interface to rule
     * source code that is broken up into lines.  Handles the
     * folding of lines terminated by a backslash.  This folding
     * is limited; it does not account for comments, quotes, or
     * escapes, so its use to be limited.
     */
    private static abstract class RuleBody {

        /**
         * Retrieve the next line of the source, or return null if
         * none.  Folds lines terminated by a backslash into the
         * next line, without regard for comments, quotes, or
         * escapes.
         */
        String nextLine() {
            String s = handleNextLine();
            if (s != null &&
                s.length() > 0 &&
                s.charAt(s.length() - 1) == '\\') {

                StringBuffer b = new StringBuffer(s);
                do {
                    b.deleteCharAt(b.length()-1);
                    s = handleNextLine();
                    if (s == null) {
                        break;
                    }
                    b.append(s);
                } while (s.length() > 0 &&
                         s.charAt(s.length() - 1) == '\\');

                s = b.toString();
            }
            return s;
        }

        /**
         * Reset to the first line of the source.
         */
        abstract void reset();

        /**
         * Subclass method to return the next line of the source.
         */
        abstract String handleNextLine();
    };

    /**
     * RuleBody subclass for a String[] array.
     */
    private static class RuleArray extends RuleBody {
        String[] array;
        int i;
        public RuleArray(String[] array) { this.array = array; i = 0; }
        public String handleNextLine() {
            return (i < array.length) ? array[i++] : null;
        }
        public void reset() {
            i = 0;
        }
    };

    /**
     * RuleBody subclass for a ResourceReader.
     */
    private static class RuleReader extends RuleBody {
        ResourceReader reader;
        public RuleReader(ResourceReader reader) { this.reader = reader; }
        public String handleNextLine() {
            try {
                return reader.readLine();
            } catch (java.io.IOException e) {}
            return null;
        }
        public void reset() {
            reader.reset();
        }
    };

    //----------------------------------------------------------------------
    // class Segments
    //----------------------------------------------------------------------

    /**
     * Segments are parentheses-enclosed regions of the input string.
     * These are referenced in the output string using the notation $1,
     * $2, etc.  Numbering is in order of appearance of the left
     * parenthesis.  Number is one-based.  Segments are defined as start,
     * limit pairs.  Segments may nest.
     *
     * During parsing, segment data is encoded in an object of class
     * Segments.  At runtime, the same data is encoded in compact form as
     * an array of integers in a TransliterationRule.  The runtime encoding
     * must satisfy three goals:
     *
     * 1. Iterate over the offsets in a pattern, from left to right,
     *    and indicate all segment boundaries, in order.  This is done
     *    during matching.
     *
     * 2. Given a reference $n, produce the start and limit offsets
     *    for that segment.  This is done during replacement.
     *
     * 3. Similar to goal 1, but in addition, indicate whether each
     *    segment boundary is a start or a limit, in other words, whether
     *    each is an open paren or a close paren.  This is required by
     *    the toRule() method.
     *
     * Goal 1 must be satisfied at high speed since this is done during
     * matching.  Goal 2 is next most important.  Goal 3 is not performance
     * critical since it is only needed by toRule().
     *
     * The array of integers is actually two arrays concatenated.  The
     * first gives the index values of the open and close parentheses in
     * the order they appear.  The second maps segment numbers to the
     * indices of the first array.  The two arrays have the same length.
     * Iterating over the first array satisfies goal 1.  Indexing into the
     * second array satisfies goal 2.  Goal 3 is satisfied by iterating
     * over the second array and constructing the required data when
     * needed.  This is what toRule() does.
     *
     * Example:  (a b(c d)e f)
     *            0 1 2 3 4 5 6
     *
     * First array: Indices are 0, 2, 4, and 6.

     * Second array: $1 is at 0 and 6, and $2 is at 2 and 4, so the
     * second array is 0, 3, 1 2 -- these give the indices in the
     * first array at which $1:open, $1:close, $2:open, and $2:close
     * occur.
     *
     * The final array is: 2, 7, 0, 2, 4, 6, -1, 2, 5, 3, 4, -1
     *
     * Each subarray is terminated with a -1, and two leading entries
     * give the number of segments and the offset to the first entry
     * of the second array.  In addition, the second array value are
     * all offset by 2 so they index directly into the final array.
     * The total array size is 4*segments[0] + 4.  The second index is
     * 2*segments[0] + 3.
     *
     * In the output string, a segment reference is indicated by a
     * character in a special range, as defined by
     * RuleBasedTransliterator.Data.
     *
     * Most rules have no segments, in which case segments is null, and the
     * output string need not be checked for segment reference characters.
     *
     * See also rbt_rule.h/cpp.
     */
    private static class Segments {

        private Vector offsets; // holds Integer objects

        private Vector isOpenParen; // holds Boolean objects

        private int offset(int i) {
            return ((Integer) offsets.elementAt(i)).intValue();
        }

        private boolean isOpen(int i) {
            return ((Boolean) isOpenParen.elementAt(i)).booleanValue();
        }

        // size of the Vectors
        private int size() {
            // assert(offset.size() == isOpenParen.size());
            return offsets.size();
        }

        public Segments() {
            offsets = new Vector();
            isOpenParen = new Vector();
        }

        public void addParenthesisAt(int offset, boolean isOpen) {
            offsets.addElement(new Integer(offset));
            isOpenParen.addElement(new Boolean(isOpen));
        }

        public int getLastParenOffset(boolean[] isOpenParen) {
            if (size() == 0) {
                return -1;
            }
            isOpenParen[0] = isOpen(size()-1);
            return offset(size()-1);
        }

        // Remove the last (rightmost) segment.  Store its offsets in start
        // and limit, and then convert all offsets at or after start to be
        // equal to start.  Upon failure, return FALSE.  Assume that the
        // caller has already called getLastParenOffset() and validated that
        // there is at least one parenthesis and that the last one is a close
        // paren.
        public boolean extractLastParenSubstring(int[] start, int[] limit) {
            // assert(offsets.size() > 0);
            // assert(isOpenParen.elementAt(isOpenParen.size()-1) == 0);
            int i = size() - 1;
            int n = 1; // count of close parens we need to match
            // Record position of the last close paren
            limit[0] = offset(i);
            --i; // back up to the one before the last one
            while (i >= 0 && n != 0) {
                n += isOpen(i) ? -1 : 1;
            }
            if (n != 0) {
                return false;
            }
            // assert(i>=0);
            start[0] = offset(i);
            // Reset all segment pairs from i to size() - 1 to [start, start+1).
            while (i<size()) {
                int o = isOpen(i) ? start[0] : (start[0]+1);
                offsets.setElementAt(new Integer(o), i);
                ++i;
            }
            return true;
        }

        // Assume caller has already gotten a TRUE validate().
        public int[] createArray() {
            int c = count(); // number of segments
            int arrayLen = 4*c + 4;
            int[] array = new int[arrayLen];
            int a2offset = 2*c + 3; // offset to array 2

            array[0] = c;
            array[1] = a2offset;
            int i;
            for (i=0; i<2*c; ++i) {
                array[2+i] = offset(i);
            }
            array[a2offset-1] = -1;
            array[arrayLen-1] = -1;
            // Now walk through and match up segment numbers with parentheses.
            // Number segments from 0.  We're going to offset all entries by 2
            // to skip the first two elements, array[0] and array[1].
            Stack stack = new Stack();
            int nextOpen = 0; // seg # of next open, 0-based
            for (i=0; i<2*c; ++i) {
                boolean open = isOpen(i);
                // Let seg be the zero-based segment number.
                // Open parens are at 2*seg in array 2.
                // Close parens are at 2*seg+1 in array 2.
                if (open) {
                    array[a2offset + 2*nextOpen] = 2+i;
                    stack.push(new Integer(nextOpen));
                    ++nextOpen;
                } else {
                    int nextClose = ((Integer) stack.pop()).intValue();
                    array[a2offset + 2*nextClose+1] = 2+i;
                }
            }
            // assert(stack.empty());

            return array;
        }

        public boolean validate() {
            // want number of parens >= 2
            // want number of parens to be even
            // want first paren '('
            // want parens to match up in the end
            if ((size() < 2) || (size() % 2 != 0) || !isOpen(0)) {
                return false;
            }
            int n = 0;
            for (int i=0; i<size(); ++i) {
                n += isOpen(i) ? 1 : -1;
                if (n < 0) {
                    return false;
                }
            }
            return n == 0;
        }

        // Number of segments
        // Assume caller has already gotten a TRUE validate().
        public int count() {
            // assert(validate());
            return size() / 2;
        }
    }

    //----------------------------------------------------------------------
    // class RuleHalf
    //----------------------------------------------------------------------

    /**
     * A class representing one side of a rule.  This class knows how to
     * parse half of a rule.  It is tightly coupled to the method
     * TransliteratorParser.parseRule().
     */
    private static class RuleHalf {

        public String text;

        public int cursor = -1; // position of cursor in text
        public int ante = -1;   // position of ante context marker '{' in text
        public int post = -1;   // position of post context marker '}' in text

        // Record the position of the segment substrings and references.  A
        // given side should have segments or segment references, but not
        // both.
        public Segments segments = null;
        public int maxRef = -1; // index of largest ref (1..9)

        // Record the offset to the cursor either to the left or to the
        // right of the key.  This is indicated by characters on the output
        // side that allow the cursor to be positioned arbitrarily within
        // the matching text.  For example, abc{def} > | @@@ xyz; changes
        // def to xyz and moves the cursor to before abc.  Offset characters
        // must be at the start or end, and they cannot move the cursor past
        // the ante- or postcontext text.  Placeholders are only valid in
        // output text.
        public int cursorOffset = 0; // only nonzero on output side

        public boolean anchorStart = false;
        public boolean anchorEnd   = false;

        /**
         * Parse one side of a rule, stopping at either the limit,
         * the END_OF_RULE character, or an operator.  Return
         * the pos of the terminating character (or limit).
         */
        public int parse(String rule, int pos, int limit,
                         TransliteratorParser parser) {
            int start = pos;
            StringBuffer buf = new StringBuffer();
            ParsePosition pp = null;
            int cursorOffsetPos = 0; // Position of first CURSOR_OFFSET on _right_
            boolean done = false;
            int quoteStart = -1; // Most recent 'single quoted string'
            int quoteLimit = -1;
            int varStart = -1; // Most recent $variableReference
            int varLimit = -1;
            int[] iref = new int[1];

        main:
            while (pos < limit && !done) {
                char c = rule.charAt(pos++);
                if (Character.isWhitespace(c)) {
                    // Ignore whitespace.  Note that this is not Unicode
                    // spaces, but Java spaces -- a subset, representing
                    // whitespace likely to be seen in code.
                    continue;
                }
                if (OPERATORS.indexOf(c) >= 0) {
                    --pos; // Backup to point to operator
                    break main;
                }
                if (anchorEnd) {
                    // Text after a presumed end anchor is a syntax err
                    syntaxError("Malformed variable reference", rule, start);
                }
                if (UnicodeSet.resemblesPattern(rule, pos-1)) {
                    if (pp == null) {
                        pp = new ParsePosition(0);
                    }
                    pp.setIndex(pos-1); // Backup to opening '['
                    buf.append(parser.parseSet(rule, pp));
                    pos = pp.getIndex();                    
                    continue;
                }
                // Handle escapes
                if (c == ESCAPE) {
                    if (pos == limit) {
                        syntaxError("Trailing backslash", rule, start);
                    }
                    iref[0] = pos;
                    int escaped = Utility.unescapeAt(rule, iref);
                    pos = iref[0];
                    if (escaped == -1) {
                        syntaxError("Malformed escape", rule, start);
                    }
                    UTF16.append(buf, escaped);
                    continue;
                }
                // Handle quoted matter
                if (c == QUOTE) {
                    int iq = rule.indexOf(QUOTE, pos);
                    if (iq == pos) {
                        buf.append(c); // Parse [''] outside quotes as [']
                        ++pos;
                    } else {
                        /* This loop picks up a segment of quoted text of the
                         * form 'aaaa' each time through.  If this segment
                         * hasn't really ended ('aaaa''bbbb') then it keeps
                         * looping, each time adding on a new segment.  When it
                         * reaches the final quote it breaks.
                         */
                        quoteStart = buf.length();
                        for (;;) {
                            if (iq < 0) {
                                syntaxError("Unterminated quote", rule, start);
                            }
                            buf.append(rule.substring(pos, iq));
                            pos = iq+1;
                            if (pos < limit && rule.charAt(pos) == QUOTE) {
                            // Parse [''] inside quotes as [']
                                iq = rule.indexOf(QUOTE, pos+1);
                            // Continue looping
                            } else {
                                break;
                            }
                        }
                        quoteLimit = buf.length();
                    }
                    continue;
                }
                switch (c) {
                case ANCHOR_START:
                    if (buf.length() == 0 && !anchorStart) {
                        anchorStart = true;
                    } else {
                        syntaxError("Misplaced anchor start",
                                    rule, start);
                    }
                    break;
                case SEGMENT_OPEN:
                case SEGMENT_CLOSE:
                    // Handle segment definitions "(" and ")"
                    // Parse "(", ")"
                    if (segments == null) {
                        segments = new Segments();
                    }
                    segments.addParenthesisAt(buf.length(), c == SEGMENT_OPEN);
                    break;
                case END_OF_RULE:
                    --pos; // Backup to point to END_OF_RULE
                    break main;
                case SymbolTable.SYMBOL_REF:
                    // Handle variable references and segment references "$1" .. "$9"
                    {
                        // A variable reference must be followed immediately
                        // by a Unicode identifier start and zero or more
                        // Unicode identifier part characters, or by a digit
                        // 1..9 if it is a segment reference.
                        if (pos == limit) {
                            // A variable ref character at the end acts as
                            // an anchor to the context limit, as in perl.
                            anchorEnd = true;
                            break;
                        }
                        // Parse "$1" "$2" .. "$9" .. (no upper limit)
                        c = rule.charAt(pos);
                        int r = Character.digit(c, 10);
                        if (r >= 1 && r <= 9) {
                            ++pos;
                            while (pos < limit) {
                                c = rule.charAt(pos);
                                int d = Character.digit(c, 10);
                                if (d < 0) {
                                    break;
                                }
                                if (r > 214748364 ||
                                    (r == 214748364 && d > 7)) {
                                    syntaxError("Undefined segment reference",
                                                rule, start);
                                }
                                r = 10*r + d;
                            }
                            if (r > maxRef) {
                                maxRef = r;
                            }
                            buf.append(parser.getSegmentStandin(r));
                        } else {
                            if (pp == null) { // Lazy create
                                pp = new ParsePosition(0);
                            }
                            pp.setIndex(pos);
                            String name = parser.parseData.
                                            parseReference(rule, pp, limit);
                            if (name == null) {
                                // This means the '$' was not followed by a
                                // valid name.  Try to interpret it as an
                                // end anchor then.  If this also doesn't work
                                // (if we see a following character) then signal
                                // an error.
                                anchorEnd = true;
                                break;
                            }
                            pos = pp.getIndex();
                            // If this is a variable definition statement,
                            // then the LHS variable will be undefined.  In
                            // that case appendVariableDef() will append the
                            // special placeholder char variableLimit-1.
                            varStart = buf.length();
                            parser.appendVariableDef(name, buf);
                            varLimit = buf.length();
                        }
                    }
                    break;
                case CONTEXT_ANTE:
                    if (ante >= 0) {
                        syntaxError("Multiple ante contexts", rule, start);
                    }
                    ante = buf.length();
                    break;
                case CONTEXT_POST:
                    if (post >= 0) {
                        syntaxError("Multiple post contexts", rule, start);
                    }
                    post = buf.length();
                    break;
                case CURSOR_POS:
                    if (cursor >= 0) {
                        syntaxError("Multiple cursors", rule, start);
                    }
                    cursor = buf.length();
                    break;
                case CURSOR_OFFSET:
                    if (cursorOffset < 0) {
                        if (buf.length() > 0) {
                            syntaxError("Misplaced " + c, rule, start);
                        }
                        --cursorOffset;
                    } else if (cursorOffset > 0) {
                        if (buf.length() != cursorOffsetPos || cursor >= 0) {
                            syntaxError("Misplaced " + c, rule, start);
                        }
                        ++cursorOffset;
                    } else {
                        if (cursor == 0 && buf.length() == 0) {
                            cursorOffset = -1;
                        } else if (cursor < 0) {
                            cursorOffsetPos = buf.length();
                            cursorOffset = 1;
                        } else {
                            syntaxError("Misplaced " + c, rule, start);
                        }
                    }
                    break;
                case DOT:
                    buf.append(parser.getDotStandIn());
                    break;
                case KLEENE_STAR:
                case ONE_OR_MORE:
                case ZERO_OR_ONE:
                    // Quantifiers.  We handle single characters, quoted strings,
                    // variable references, and segments.
                    //  a+      matches  aaa
                    //  'foo'+  matches  foofoofoo
                    //  $v+     matches  xyxyxy if $v == xy
                    //  (seg)+  matches  segsegseg
                    {
                        int qstart, qlimit;
                        boolean[] isOpenParen = new boolean[1];
                        boolean isSegment = false;
                        if (segments != null &&
                            segments.getLastParenOffset(isOpenParen) == buf.length()) {
                            // The */+ immediately follows a segment
                            if (isOpenParen[0]) {
                                syntaxError("Misplaced quantifier", rule, start);
                            }
                            int[] startparam = new int[1];
                            int[] limitparam = new int[1];
                            if (!segments.extractLastParenSubstring(startparam, limitparam)) {
                                syntaxError("Mismatched segment delimiters", rule, start);
                            }
                            qstart = startparam[0];
                            qlimit = limitparam[0];
                            isSegment = true;
                        } else {
                            // The */+ follows an isolated character or quote
                            // or variable reference
                            if (buf.length() == quoteLimit) {
                                // The */+ follows a 'quoted string'
                                qstart = quoteStart;
                                qlimit = quoteLimit;
                            } else if (buf.length() == varLimit) {
                                // The */+ follows a $variableReference
                                qstart = varStart;
                                qlimit = varLimit;
                            } else {
                                // The */+ follows a single character
                                qstart = buf.length() - 1;
                                qlimit = qstart + 1;
                            }
                        }
                        UnicodeMatcher m =
                            new StringMatcher(buf.toString(), qstart, qlimit,
                                              isSegment, parser.data);
                        int min = 0;
                        int max = Quantifier.MAX;
                        switch (c) {
                        case ONE_OR_MORE:
                            min = 1;
                            break;
                        case ZERO_OR_ONE:
                            min = 0;
                            max = 1;
                            break;
                            // case KLEENE_STAR:
                            //    do nothing -- min, max already set
                        }
                        m = new Quantifier(m, min, max);
                        buf.setLength(qstart);
                        buf.append(parser.generateStandInFor(m));
                    }
                    break;
                default:
                    // Disallow unquoted characters other than [0-9A-Za-z]
                    // in the printable ASCII range.  These characters are
                    // reserved for possible future use.
                    if (c >= 0x0021 && c <= 0x007E &&
                        !((c >= '0' && c <= '9') ||
                          (c >= 'A' && c <= 'Z') ||
                          (c >= 'a' && c <= 'z'))) {
                        syntaxError("Unquoted " + c, rule, start);
                    }
                    buf.append(c);
                    break;
                }
            }

            if (cursorOffset > 0 && cursor != cursorOffsetPos) {
                syntaxError("Misplaced " + CURSOR_POS, rule, start);
            }
            text = buf.toString();
            return pos;
        }

        /**
         * Remove context.
         */
        void removeContext() {
            text = text.substring(ante < 0 ? 0 : ante,
                                  post < 0 ? text.length() : post);
            ante = post = -1;
            anchorStart = anchorEnd = false;
        }

        /**
         * Create and return an int[] array of segments.
         */
        int[] createSegments() {
            return (segments == null) ? null : segments.createArray();
        }
    }

    //----------------------------------------------------------------------
    // PUBLIC methods
    //----------------------------------------------------------------------

    /**
     * Constructor.
     */
    public TransliteratorParser() {
    }

    /**
     * Parse a set of rules.  After the parse completes, examine the public
     * data members for results.
     */
    public void parse(String rules, int direction) {
        parseRules(new RuleArray(new String[] { rules }), direction);
    }
   
    /**
     * Parse a set of rules.  After the parse completes, examine the public
     * data members for results.
     */
    public void parse(ResourceReader rules, int direction) {
        parseRules(new RuleReader(rules), direction);
    }

    //----------------------------------------------------------------------
    // PRIVATE methods
    //----------------------------------------------------------------------

    /**
     * Parse an array of zero or more rules.  The strings in the array are
     * treated as if they were concatenated together, with rule terminators
     * inserted between array elements if not present already.
     *
     * Any previous rules are discarded.  Typically this method is called exactly
     * once, during construction.
     *
     * The member this.data will be set to null if there are no rules.
     *
     * @exception IllegalArgumentException if there is a syntax error in the
     * rules
     */
    void parseRules(RuleBody ruleArray, int dir) {
        data = new RuleBasedTransliterator.Data();
        direction = dir;
        ruleCount = 0;
        compoundFilter = null;

        // By default, rules use part of the private use area
        // E000..F8FF for variables and other stand-ins.  Currently
        // the range F000..F8FF is typically sufficient.  The 'use
        // variable range' pragma allows rule sets to modify this.
        setVariableRange(0xF000, 0xF8FF);

        variablesVector = new Vector();
        parseData = new ParseData();

        StringBuffer errors = null;
        int errorCount = 0;

        ruleArray.reset();

        StringBuffer idBlockResult = new StringBuffer();
        idSplitPoint = -1;
        // The mode marks whether we are in the header ::id block, the
        // rule block, or the footer ::id block.
        // mode == 0: start: rule->1, ::id->0
        // mode == 1: in rules: rule->1, ::id->2
        // mode == 2: in footer rule block: rule->ERROR, ::id->2
        int mode = 0;

        // The compound filter offset is an index into idBlockResult.
        // If it is 0, then the compound filter occurred at the start,
        // and it is the offset to the _start_ of the compound filter
        // pattern.  Otherwise it is the offset to the _limit_ of the
        // compound filter pattern within idBlockResult.
        this.compoundFilter = null;
        int compoundFilterOffset = -1;

    main:
        for (;;) {
            String rule = ruleArray.nextLine();
            if (rule == null) {
                break;
            }
            int pos = 0;
            int limit = rule.length();
            while (pos < limit) {
                char c = rule.charAt(pos++);
                if (Character.isWhitespace(c)) {
                    // Ignore leading whitespace.  Note that this is not
                    // Unicode spaces, but Java spaces -- a subset,
                    // representing whitespace likely to be seen in code.
                    continue;
                }
                // Skip lines starting with the comment character
                if (c == RULE_COMMENT_CHAR) {
                    pos = rule.indexOf("\n", pos) + 1;
                    if (pos == 0) {
                        break; // No "\n" found; rest of rule is a commnet
                    }
                    continue; // Either fall out or restart with next line
                }
                // Often a rule file contains multiple errors.  It's
                // convenient to the rule author if these are all reported
                // at once.  We keep parsing rules even after a failure, up
                // to a specified limit, and report all errors at once.
                try {
                    // We've found the start of a rule or ID.  c is its first
                    // character, and pos points past c.
                    --pos;
                    // Look for an ID token.  Must have at least ID_TOKEN_LEN + 1
                    // chars left.
                    if ((pos + ID_TOKEN_LEN + 1) <= limit &&
                        rule.regionMatches(pos, ID_TOKEN, 0, ID_TOKEN_LEN)) {
                        pos += ID_TOKEN_LEN;
                        c = rule.charAt(pos);
                        while (UCharacter.isWhitespace(c) && pos < limit) {
                            ++pos;
                            c = rule.charAt(pos);
                        }
                        int lengthBefore = idBlockResult.length();
                        if (mode == 1) {
                            mode = 2;
                            idSplitPoint = lengthBefore;
                        }
                        int[] p = new int[] { pos };
                        boolean[] sawDelim = new boolean[1];
                        UnicodeSet[] cpdFilter = new UnicodeSet[1];
                        Transliterator.parseID(rule, idBlockResult, p, sawDelim, cpdFilter, direction, false);
                        if (p[0] == pos || !sawDelim[0]) {
                            // Invalid ::id
                            int i1 = pos + 2;
                            while (i1 < rule.length() && rule.charAt(i1) != ';') {
                                ++i1;
                            }
                            throw new IllegalArgumentException("Invalid ::ID " +
                                                               rule.substring(pos, i1));
                        }
                        if (cpdFilter[0] != null) {
                            if (compoundFilter != null) {
                                // Multiple compound filters
                                throw new IllegalArgumentException("Multiple compound filters");
                            }
                            compoundFilter = cpdFilter[0];
                            compoundFilterOffset = (direction == Transliterator.FORWARD) ?
                                lengthBefore : idBlockResult.length();
                        }
                        pos = p[0];
                    } else if (resemblesPragma(rule, pos, limit)) {
                        int ppp = parsePragma(rule, pos, limit);
                        if (ppp < 0) {
                            throw new IllegalArgumentException("Unrecognized pragma: " +
                                                               rule.substring(pos));
                        }
                        pos = ppp;
                    } else {
                        // Parse a rule
                        pos = parseRule(rule, pos, limit);
                        ++ruleCount;
                        if (mode == 2) {
                            // ::id in illegal position (because a rule
                            // occurred after the ::id footer block)
                            throw new IllegalArgumentException("::ID in illegal position");
                        }
                        mode = 1;
                    }
                } catch (IllegalArgumentException e) {
                    if (errorCount == 30) {
                        errors.append("\nMore than 30 errors; further messages squelched");
                        break main;
                    }
                    if (errors == null) {
                        errors = new StringBuffer(e.getMessage());
                    } else {
                        errors.append("\n" + e.getMessage());
                    }
                    ++errorCount;
                    pos = ruleEnd(rule, pos, limit) + 1; // +1 advances past ';'
                }
            }
        }

        idBlock = idBlockResult.toString();

        // Convert the set vector to an array
        data.variables = new UnicodeMatcher[variablesVector.size()];
        variablesVector.copyInto(data.variables);
        variablesVector = null;

        // Do more syntax checking and index the rules
        try {
            if (compoundFilter != null) {
                if ((direction == Transliterator.FORWARD &&
                     compoundFilterOffset != 0) ||
                    (direction == Transliterator.REVERSE &&
                     compoundFilterOffset != idBlock.length())) {
                    throw new IllegalArgumentException("Compound filters misplaced");
                }
            }

            data.ruleSet.freeze();

            if (idSplitPoint < 0) {
                idSplitPoint = idBlock.length();
            }

            if (ruleCount == 0) {
                data = null;
            }
        } catch (IllegalArgumentException e) {
            if (errors == null) {
                errors = new StringBuffer(e.getMessage());
            } else {
                errors.append("\n").append(e.getMessage());
            }
        }

        if (errors != null) {
            throw new IllegalArgumentException(errors.toString());
        }
    }

    /**
     * MAIN PARSER.  Parse the next rule in the given rule string, starting
     * at pos.  Return the index after the last character parsed.  Do not
     * parse characters at or after limit.
     *
     * Important:  The character at pos must be a non-whitespace character
     * that is not the comment character.
     *
     * This method handles quoting, escaping, and whitespace removal.  It
     * parses the end-of-rule character.  It recognizes context and cursor
     * indicators.  Once it does a lexical breakdown of the rule at pos, it
     * creates a rule object and adds it to our rule list.
     *
     * This method is tightly coupled to the inner class RuleHalf.
     */
    private int parseRule(String rule, int pos, int limit) {
        // Locate the left side, operator, and right side
        int start = pos;
        char operator = 0;

        RuleHalf left  = new RuleHalf();
        RuleHalf right = new RuleHalf();

        undefinedVariableName = null;
        pos = left.parse(rule, pos, limit, this);

        if (pos == limit ||
            OPERATORS.indexOf(operator = rule.charAt(pos++)) < 0) {
            syntaxError("No operator", rule, start);
        }

        // Found an operator char.  Check for forward-reverse operator.
        if (operator == REVERSE_RULE_OP &&
            (pos < limit && rule.charAt(pos) == FORWARD_RULE_OP)) {
            ++pos;
            operator = FWDREV_RULE_OP;
        }

        pos = right.parse(rule, pos, limit, this);

        if (pos < limit) {
            if (rule.charAt(pos) == END_OF_RULE) {
                ++pos;
            } else {
                // RuleHalf parser must have terminated at an operator
                syntaxError("Unquoted operator", rule, start);
            }
        }

        if (operator == VARIABLE_DEF_OP) {
            // LHS is the name.  RHS is a single character, either a literal
            // or a set (already parsed).  If RHS is longer than one
            // character, it is either a multi-character string, or multiple
            // sets, or a mixture of chars and sets -- syntax error.

            // We expect to see a single undefined variable (the one being
            // defined).
            if (undefinedVariableName == null) {
                syntaxError("Missing '$' or duplicate definition", rule, start);
            }
            if (left.text.length() != 1 || left.text.charAt(0) != variableLimit) {
                syntaxError("Malformed LHS", rule, start);
            }
            if (left.anchorStart || left.anchorEnd ||
                right.anchorStart || right.anchorEnd) {
                syntaxError("Malformed variable def", rule, start);
            }
            // We allow anything on the right, including an empty string.
            int n = right.text.length();
            char[] value = new char[n];
            right.text.getChars(0, n, value, 0);
            data.variableNames.put(undefinedVariableName, value);

            ++variableLimit;
            return pos;
        }

        // If this is not a variable definition rule, we shouldn't have
        // any undefined variable names.
        if (undefinedVariableName != null) {
            syntaxError("Undefined variable $" + undefinedVariableName,
                        rule, start);
        }

        // If the direction we want doesn't match the rule
        // direction, do nothing.
        if (operator != FWDREV_RULE_OP &&
            ((direction == Transliterator.FORWARD) != (operator == FORWARD_RULE_OP))) {
            return pos;
        }

        // Transform the rule into a forward rule by swapping the
        // sides if necessary.
        if (direction == Transliterator.REVERSE) {
            RuleHalf temp = left;
            left = right;
            right = temp;
        }

        // Remove non-applicable elements in forward-reverse
        // rules.  Bidirectional rules ignore elements that do not
        // apply.
        if (operator == FWDREV_RULE_OP) {
            right.removeContext();
            right.segments = null;
            left.cursor = left.maxRef = -1;
            left.cursorOffset = 0;
        }

        // Normalize context
        if (left.ante < 0) {
            left.ante = 0;
        }
        if (left.post < 0) {
            left.post = left.text.length();
        }

        // Context is only allowed on the input side.  Cursors are only
        // allowed on the output side.  Segment delimiters can only appear
        // on the left, and references on the right.  Cursor offset
        // cannot appear without an explicit cursor.  Cursor offset
        // cannot place the cursor outside the limits of the context.
        // Anchors are only allowed on the input side.
        if (right.ante >= 0 || right.post >= 0 || left.cursor >= 0 ||
            right.segments != null || left.maxRef >= 0 ||
            (right.cursorOffset != 0 && right.cursor < 0) ||
            // - The following two checks were used to ensure that the
            // - the cursor offset stayed within the ante- or postcontext.
            // - However, with the addition of quantifiers, we have to
            // - allow arbitrary cursor offsets and do runtime checking.
            //(right.cursorOffset > (left.text.length() - left.post)) ||
            //(-right.cursorOffset > left.ante) ||
            right.anchorStart || right.anchorEnd) {
            syntaxError("Malformed rule", rule, start);
        }

        // Check integrity of segments and segment references.  Each
        // segment's start must have a corresponding limit, and the
        // references must not refer to segments that do not exist.
        if (left.segments != null) {
            if (!left.segments.validate()) {
                syntaxError("Missing segment close", rule, start);
            }
            int n = left.segments.count();
            if (right.maxRef > n) {
                syntaxError("Undefined segment reference", rule, start);
            }
        }

        data.ruleSet.addRule(new TransliterationRule(
                                     left.text, left.ante, left.post,
                                     right.text, right.cursor, right.cursorOffset,
                                     left.createSegments(),
                                     left.anchorStart, left.anchorEnd,
                                     data));

        return pos;
    }

    /**
     * Set the variable range to [start, end] (inclusive).
     */
    private void setVariableRange(int start, int end) {
        if (start > end || start < 0 || end > 0xFFFF) {
            throw new IllegalArgumentException("Invalid variable range " + start + ", " + end);
        }
        
        // Segment references work down; variables work up.  We don't
        // know how many of each we will need.
        data.segmentBase = (char) end;
        data.segmentCount = 0;
        data.variablesBase = variableNext = (char) start; // first private use
        variableLimit = (char) (end + 1);
    }

    /**
     * Set the maximum backup to 'backup', in response to a pragma
     * statement.
     */
    private void pragmaMaximumBackup(int backup) {
        //TODO Finish
    }

    /**
     * Begin normalizing all rules using the given mode, in response
     * to a pragma statement.
     */
    private void pragmaNormalizeRules(Normalizer.Mode mode) {
        //TODO Finish
    }

    /**
     * Return true if the given rule looks like a pragma.
     * @param pos offset to the first non-whitespace character
     * of the rule.
     * @param limit pointer past the last character of the rule.
     */
    static boolean resemblesPragma(String rule, int pos, int limit) {
        // Must start with /use\s/i
        return Utility.parsePattern(rule, pos, limit, "use ", null) >= 0;
    }

    /**
     * Parse a pragma.  This method assumes resemblesPragma() has
     * already returned true.
     * @param pos offset to the first non-whitespace character
     * of the rule.
     * @param limit pointer past the last character of the rule.
     * @return the position index after the final ';' of the pragma,
     * or -1 on failure.
     */
    private int parsePragma(String rule, int pos, int limit) {
        int[] array = new int[2];

        // resemblesPragma() has already returned true, so we
        // know that pos points to /use\s/i; we can skip 4 characters
        // immediately
        pos += 4;
        
        // Here are the pragmas we recognize:
        // use variable range 0xE000 0xEFFF;
        // use maximum backup 16;
        // use nfd rules;
        int p = Utility.parsePattern(rule, pos, limit, "~variable range # #~;", array);
        if (p >= 0) {
            setVariableRange(array[0], array[1]);
            return p;
        }

        p = Utility.parsePattern(rule, pos, limit, "~maximum backup #~;", array);
        if (p >= 0) {
            pragmaMaximumBackup(array[0]);
            return p;
        }

        p = Utility.parsePattern(rule, pos, limit, "~nfd rules~;", null);
        if (p >= 0) {
            pragmaNormalizeRules(Normalizer.DECOMP);
            return p;
        }

        p = Utility.parsePattern(rule, pos, limit, "~nfc rules~;", null);
        if (p >= 0) {
            pragmaNormalizeRules(Normalizer.COMPOSE);
            return p;
        }

        // Syntax error: unable to parse pragma
        return -1;
    }

    /**
     * Throw an exception indicating a syntax error.  Search the rule string
     * for the probable end of the rule.  Of course, if the error is that
     * the end of rule marker is missing, then the rule end will not be found.
     * In any case the rule start will be correctly reported.
     * @param msg error description
     * @param rule pattern string
     * @param start position of first character of current rule
     */
    static final void syntaxError(String msg, String rule, int start) {
        int end = ruleEnd(rule, start, rule.length());
        throw new IllegalArgumentException(msg + " in \"" +
                                           Utility.escape(rule.substring(start, end)) + '"');
    }

    static final int ruleEnd(String rule, int start, int limit) {
        int end = quotedIndexOf(rule, start, limit, ";");
        if (end < 0) {
            end = limit;
        }
        return end;
    }

    /**
     * Parse a UnicodeSet out, store it, and return the stand-in character
     * used to represent it.
     */
    private final char parseSet(String rule, ParsePosition pos) {
        UnicodeSet set = new UnicodeSet(rule, pos, parseData);
        if (variableNext >= variableLimit) {
            throw new RuntimeException("Private use variables exhausted");
        }
        set.compact();
        return generateStandInFor(set);
    }

    /**
     * Generate and return a stand-in for a new UnicodeMatcher.  Store
     * the matcher.
     */
    char generateStandInFor(UnicodeMatcher matcher) {
        // assert(matcher != null);
        if (variableNext >= variableLimit) {
            throw new RuntimeException("Private use variables exhausted");
        }
        variablesVector.addElement(matcher);
        return variableNext++;
    }

    /**
     * Return the stand-in for the dot set.  It is allocated the first
     * time and reused thereafter.
     */
    char getDotStandIn() {
        if (dotStandIn == -1) {
            dotStandIn = generateStandInFor(new UnicodeSet(DOT_SET));
        }
        return (char) dotStandIn;
    }
    
    /**
     * Append the value of the given variable name to the given
     * StringBuffer.
     * @exception IllegalArgumentException if the name is unknown.
     */
    private void appendVariableDef(String name, StringBuffer buf) {
        char[] ch = (char[]) data.variableNames.get(name);
        if (ch == null) {
            // We allow one undefined variable so that variable definition
            // statements work.  For the first undefined variable we return
            // the special placeholder variableLimit-1, and save the variable
            // name.
            if (undefinedVariableName == null) {
                undefinedVariableName = name;
                if (variableNext >= variableLimit) {
                    throw new RuntimeException("Private use variables exhausted");
                }
                buf.append((char) --variableLimit);
            } else {
                throw new IllegalArgumentException("Undefined variable $"
                                                   + name);
            }
        } else {
            buf.append(ch);
        }
    }

    char getSegmentStandin(int r) {
        // assert(r>=1);
        if (r > data.segmentCount) {
            data.segmentCount = r;
            variableLimit = (char) (data.segmentBase - r + 1);
            if (variableNext >= variableLimit) {
                throw new IllegalArgumentException("Too many variables / segments");
            }
        }
        return data.getSegmentStandin(r);
    }

    /**
     * Returns the index of the first character in a set, ignoring quoted text.
     * For example, in the string "abc'hide'h", the 'h' in "hide" will not be
     * found by a search for "h".  Unlike String.indexOf(), this method searches
     * not for a single character, but for any character of the string
     * <code>setOfChars</code>.
     * @param text text to be searched
     * @param start the beginning index, inclusive; <code>0 <= start
     * <= limit</code>.
     * @param limit the ending index, exclusive; <code>start <= limit
     * <= text.length()</code>.
     * @param setOfChars string with one or more distinct characters
     * @return Offset of the first character in <code>setOfChars</code>
     * found, or -1 if not found.
     * @see #indexOf
     */
    private static int quotedIndexOf(String text, int start, int limit,
                                     String setOfChars) {
        for (int i=start; i<limit; ++i) {
            char c = text.charAt(i);
            if (c == ESCAPE) {
                ++i;
            } else if (c == QUOTE) {
                while (++i < limit
                       && text.charAt(i) != QUOTE) {}
            } else if (setOfChars.indexOf(c) >= 0) {
                return i;
            }
        }
        return -1;
    }
}

//eof