| /* |
| ********************************************************************** |
| * Copyright (C) 1999, International Business Machines |
| * Corporation and others. All Rights Reserved. |
| ********************************************************************** |
| * Date Name Description |
| * 11/17/99 aliu Creation. |
| ********************************************************************** |
| */ |
| #include "rbt_pars.h" |
| #include "unicode/rbt.h" |
| #include "rbt_rule.h" |
| #include "unirange.h" |
| #include "rbt_data.h" |
| #include "unicode/uniset.h" |
| #include "cstring.h" |
| #include "unicode/parsepos.h" |
| #include "symtable.h" |
| #include "unicode/parseerr.h" |
| |
| // Operators |
| const UChar TransliterationRuleParser::VARIABLE_DEF_OP = 0x003D/*=*/; |
| const UChar TransliterationRuleParser::FORWARD_RULE_OP = 0x003E/*>*/; |
| const UChar TransliterationRuleParser::REVERSE_RULE_OP = 0x003C/*<*/; |
| const UChar TransliterationRuleParser::FWDREV_RULE_OP = 0x007E/*~*/; // internal rep of <> op |
| const UnicodeString TransliterationRuleParser::OPERATORS = UNICODE_STRING("=><", 3); |
| |
| // Other special characters |
| const UChar TransliterationRuleParser::QUOTE = 0x0027/*'*/; |
| const UChar TransliterationRuleParser::ESCAPE = 0x005C/*\*/; |
| const UChar TransliterationRuleParser::END_OF_RULE = 0x003B/*;*/; |
| const UChar TransliterationRuleParser::RULE_COMMENT_CHAR = 0x0023/*#*/; |
| |
| const UChar TransliterationRuleParser::VARIABLE_REF_OPEN = 0x007B/*{*/; |
| const UChar TransliterationRuleParser::VARIABLE_REF_CLOSE = 0x007D/*}*/; |
| const UChar TransliterationRuleParser::CONTEXT_OPEN = 0x0028/*(*/; |
| const UChar TransliterationRuleParser::CONTEXT_CLOSE = 0x0029/*)*/; |
| const UChar TransliterationRuleParser::SET_OPEN = 0x005B/*[*/; |
| const UChar TransliterationRuleParser::SET_CLOSE = 0x005D/*]*/; |
| const UChar TransliterationRuleParser::CURSOR_POS = 0x007C/*|*/; |
| |
| //---------------------------------------------------------------------- |
| // BEGIN ParseData |
| //---------------------------------------------------------------------- |
| |
| /** |
| * This class implements the SymbolTable interface. It is used |
| * during parsing to give UnicodeSet access to variables that |
| * have been defined so far. Note that it uses setVariablesVector, |
| * _not_ data.setVariables. |
| */ |
| class ParseData : public SymbolTable { |
| public: |
| const TransliterationRuleData* data; // alias |
| |
| const UVector* setVariablesVector; // alias |
| |
| ParseData(const TransliterationRuleData* data = 0, |
| const UVector* setVariablesVector = 0); |
| |
| /** |
| * Lookup the object associated with this string and return it. |
| * Return U_ILLEGAL_ARGUMENT_ERROR status if the name does not |
| * exist. Return a non-NULL set if the name is mapped to a set; |
| * otherwise return a NULL set. |
| */ |
| virtual void lookup(const UnicodeString& name, UChar& c, UnicodeSet*& set, |
| UErrorCode& status) const; |
| }; |
| |
| ParseData::ParseData(const TransliterationRuleData* d, |
| const UVector* sets) : |
| data(d), setVariablesVector(sets) {} |
| |
| /** |
| * Implement SymbolTable API. Lookup a variable, returning |
| * either a Character, a UnicodeSet, or null. |
| */ |
| void ParseData::lookup(const UnicodeString& name, UChar& c, UnicodeSet*& set, |
| UErrorCode& status) const { |
| c = data->lookupVariable(name, status); |
| if (U_SUCCESS(status)) { |
| int32_t i = c - data->setVariablesBase; |
| set = (i < setVariablesVector->size()) ? |
| (UnicodeSet*) setVariablesVector->elementAt(i) : 0; |
| } |
| } |
| |
| //---------------------------------------------------------------------- |
| // END ParseData |
| //---------------------------------------------------------------------- |
| |
| TransliterationRuleData* |
| TransliterationRuleParser::parse(const UnicodeString& rules, |
| RuleBasedTransliterator::Direction direction, |
| ParseError* parseError) { |
| TransliterationRuleParser parser(rules, direction, parseError); |
| parser.parseRules(); |
| if (U_FAILURE(parser.status)) { |
| delete parser.data; |
| parser.data = 0; |
| } |
| return parser.data; |
| } |
| |
| /** |
| * @param rules list of rules, separated by newline characters |
| * @exception IllegalArgumentException if there is a syntax error in the |
| * rules |
| */ |
| TransliterationRuleParser::TransliterationRuleParser( |
| const UnicodeString& theRules, |
| RuleBasedTransliterator::Direction theDirection, |
| ParseError* theParseError) : |
| rules(theRules), direction(theDirection), data(0), parseError(theParseError) { |
| parseData = new ParseData(0, &setVariablesVector); |
| } |
| |
| /** |
| * Destructor. |
| */ |
| TransliterationRuleParser::~TransliterationRuleParser() { |
| delete parseData; |
| } |
| |
| /** |
| * Parse the given string as a sequence of rules, separated by newline |
| * characters ('\n'), and cause this object to implement those rules. Any |
| * previous rules are discarded. Typically this method is called exactly |
| * once, during construction. |
| * @exception IllegalArgumentException if there is a syntax error in the |
| * rules |
| */ |
| void TransliterationRuleParser::parseRules(void) { |
| status = U_ZERO_ERROR; |
| |
| delete data; |
| data = new TransliterationRuleData(status); |
| if (U_FAILURE(status)) { |
| return; |
| } |
| |
| parseData->data = data; |
| setVariablesVector.removeAllElements(); |
| if (parseError != 0) { |
| parseError->code = 0; |
| } |
| determineVariableRange(); |
| |
| int32_t pos = 0; |
| int32_t limit = rules.length(); |
| while (pos < limit && U_SUCCESS(status)) { |
| UChar c = rules.charAt(pos++); |
| if (Unicode::isWhitespace(c)) { |
| // Ignore leading whitespace. Note that this is not |
| // Unicode spaces, but Java spaces -- a subset, |
| // representing whitespace likely to be seen in code. |
| continue; |
| } |
| // Skip lines starting with the comment character |
| if (c == RULE_COMMENT_CHAR) { |
| pos = rules.indexOf((UChar)0x000A /*\n*/, pos) + 1; |
| if (pos == 0) { |
| break; // No "\n" found; rest of rule is a commnet |
| } |
| continue; // Either fall out or restart with next line |
| } |
| // We've found the start of a rule. c is its first |
| // character, and pos points past c. Lexically parse the |
| // rule into component pieces. |
| pos = parseRule(--pos, limit); |
| } |
| |
| // Convert the set vector to an array |
| data->setVariablesLength = setVariablesVector.size(); |
| data->setVariables = new UnicodeSet*[data->setVariablesLength]; |
| // orphanElement removes the given element and shifts all other |
| // elements down. For performance (and code clarity) we work from |
| // the end back to index 0. |
| for (int32_t i=data->setVariablesLength; i>0; ) { |
| --i; |
| data->setVariables[i] = |
| (UnicodeSet*) setVariablesVector.orphanElementAt(i); |
| } |
| |
| // Index the rules |
| if (U_SUCCESS(status)) { |
| data->ruleSet.freeze(*data, status); |
| } |
| } |
| |
| /** |
| * MAIN PARSER. Parse the next rule in the given rule string, starting |
| * at pos. Return the index after the last character parsed. Do not |
| * parse characters at or after limit. |
| * |
| * Important: The character at pos must be a non-whitespace character |
| * that is not the comment character. |
| * |
| * This method handles quoting, escaping, and whitespace removal. It |
| * parses the end-of-rule character. It recognizes context and cursor |
| * indicators. Once it does a lexical breakdown of the rule at pos, it |
| * creates a rule object and adds it to our rule list. |
| */ |
| int32_t TransliterationRuleParser::parseRule(int32_t pos, int32_t limit) { |
| // Locate the left side, operator, and right side |
| int32_t start = pos; |
| UChar op = 0; |
| |
| UnicodeString buf; |
| int32_t cursor = -1; // position of cursor in buf |
| int32_t ante = -1; // position of ante context marker ')' in buf |
| int32_t post = -1; // position of post context marker '(' in buf |
| int32_t postClose = -1; // position of post context close ')' in buf |
| |
| // Assigned to buf and its adjuncts after the LHS has been |
| // parsed. Thereafter, buf etc. refer to the RHS. |
| UnicodeString left; |
| int32_t leftCursor = -1, leftAnte = -1, leftPost = -1, leftPostClose = -1; |
| |
| UnicodeString scratch; |
| |
| while (pos < limit) { |
| UChar c = rules.charAt(pos++); |
| if (Unicode::isWhitespace(c)) { |
| // Ignore whitespace. Note that this is not Unicode |
| // spaces, but Java spaces -- a subset, representing |
| // whitespace likely to be seen in code. |
| continue; |
| } |
| // Handle escapes |
| if (c == ESCAPE) { |
| if (pos == limit) { |
| return syntaxError(RuleBasedTransliterator::TRAILING_BACKSLASH, rules, start); |
| } |
| // Parse \uXXXX escapes |
| c = rules.charAt(pos++); |
| if (c == 0x0075/*u*/) { |
| if ((pos+4) > limit) { |
| return syntaxError(RuleBasedTransliterator::MALFORMED_UNICODE_ESCAPE, rules, start); |
| } |
| c = (UChar)0x0000; |
| for (int32_t plim=pos+4; pos<plim; ++pos) { // [sic] |
| int32_t digit = Unicode::digit(rules.charAt(pos), 16); |
| if (digit<0) { |
| return syntaxError(RuleBasedTransliterator::MALFORMED_UNICODE_ESCAPE, rules, start); |
| } |
| c = (UChar) ((c << 4) | digit); |
| } |
| } |
| |
| buf.append(c); |
| continue; |
| } |
| // Handle quoted matter |
| if (c == QUOTE) { |
| int32_t iq = rules.indexOf(QUOTE, pos); |
| if (iq == pos) { |
| buf.append(c); // Parse [''] outside quotes as ['] |
| ++pos; |
| } else { |
| /* This loop picks up a segment of quoted text of the |
| * form 'aaaa' each time through. If this segment |
| * hasn't really ended ('aaaa''bbbb') then it keeps |
| * looping, each time adding on a new segment. When it |
| * reaches the final quote it breaks. |
| */ |
| for (;;) { |
| if (iq < 0) { |
| return syntaxError(RuleBasedTransliterator::UNTERMINATED_QUOTE, rules, start); |
| } |
| scratch.truncate(0); |
| rules.extractBetween(pos, iq, scratch); |
| buf.append(scratch); |
| pos = iq+1; |
| if (pos < limit && rules.charAt(pos) == QUOTE) { |
| // Parse [''] inside quotes as ['] |
| iq = rules.indexOf(QUOTE, pos+1); |
| // Continue looping |
| } else { |
| break; |
| } |
| } |
| } |
| continue; |
| } |
| if (OPERATORS.indexOf(c) >= 0) { |
| if (op != 0) { |
| return syntaxError(RuleBasedTransliterator::UNQUOTED_SPECIAL, rules, start); |
| } |
| // Found an operator char. Check for forward-reverse operator. |
| if (c == REVERSE_RULE_OP && |
| (pos < limit && rules.charAt(pos) == FORWARD_RULE_OP)) { |
| ++pos; |
| op = FWDREV_RULE_OP; |
| } else { |
| op = c; |
| } |
| left = buf; // lhs |
| leftCursor = cursor; |
| leftAnte = ante; |
| leftPost = post; |
| leftPostClose = postClose; |
| |
| buf.truncate(0); |
| cursor = ante = post = postClose = -1; |
| continue; |
| } |
| if (c == END_OF_RULE) { |
| break; |
| } |
| switch (c) { |
| case VARIABLE_REF_OPEN: |
| { |
| int32_t j = rules.indexOf(VARIABLE_REF_CLOSE, pos); |
| if (pos == j || j < 0) { // empty or unterminated |
| return syntaxError(RuleBasedTransliterator::MALFORMED_VARIABLE_REFERENCE, rules, start); |
| } |
| scratch.truncate(0); |
| rules.extractBetween(pos, j, scratch); |
| pos = j+1; |
| UChar v = data->lookupVariable(scratch, status); |
| if (U_FAILURE(status)) { |
| return syntaxError(RuleBasedTransliterator::UNDEFINED_VARIABLE, rules, start); |
| } |
| buf.append(v); |
| } |
| break; |
| case CONTEXT_OPEN: |
| if (post >= 0) { |
| return syntaxError(RuleBasedTransliterator::MULTIPLE_POST_CONTEXTS, rules, start); |
| } |
| // Ignore CONTEXT_OPEN if buffer length is zero -- that means |
| // this is the optional opening delimiter for the ante context. |
| if (buf.length() > 0) { |
| post = buf.length(); |
| } |
| break; |
| case CONTEXT_CLOSE: |
| if (postClose >= 0) { |
| return syntaxError(RuleBasedTransliterator::UNEXPECTED_CLOSE_CONTEXT, rules, start); |
| } |
| if (post >= 0) { |
| // This is probably the optional closing delimiter |
| // for the post context; save the pos and check later. |
| postClose = buf.length(); |
| } else if (ante >= 0) { |
| return syntaxError(RuleBasedTransliterator::MULTIPLE_ANTE_CONTEXTS, rules, start); |
| } else { |
| ante = buf.length(); |
| } |
| break; |
| case SET_OPEN: { |
| ParsePosition pp(pos-1); // Backup to opening '[' |
| buf.append(registerSet(new UnicodeSet(rules, pp, *parseData, status))); |
| if (U_FAILURE(status)) { |
| return syntaxError(RuleBasedTransliterator::MALFORMED_SET, rules, start); |
| } |
| pos = pp.getIndex(); } |
| break; |
| case VARIABLE_REF_CLOSE: |
| case SET_CLOSE: |
| return syntaxError(RuleBasedTransliterator::UNQUOTED_SPECIAL, rules, start); |
| case CURSOR_POS: |
| if (cursor >= 0) { |
| return syntaxError(RuleBasedTransliterator::MULTIPLE_CURSORS, rules, start); |
| } |
| cursor = buf.length(); |
| break; |
| default: |
| buf.append(c); |
| break; |
| } |
| } |
| if (op == 0) { |
| return syntaxError(RuleBasedTransliterator::MISSING_OPERATOR, rules, start); |
| } |
| |
| // Check context close parameters |
| if ((leftPostClose >= 0 && leftPostClose != left.length()) || |
| (postClose >= 0 && postClose != buf.length())) { |
| return syntaxError(RuleBasedTransliterator::TEXT_AFTER_CLOSE_CONTEXT, rules, start); |
| } |
| |
| // Context is only allowed on the input side; that is, the left side |
| // for forward rules. Cursors are only allowed on the output side; |
| // that is, the right side for forward rules. Bidirectional rules |
| // ignore elements that do not apply. |
| |
| switch (op) { |
| case VARIABLE_DEF_OP: |
| // LHS is the name. RHS is a single character, either a literal |
| // or a set (already parsed). If RHS is longer than one |
| // character, it is either a multi-character string, or multiple |
| // sets, or a mixture of chars and sets -- syntax error. |
| if (buf.length() != 1) { |
| return syntaxError(RuleBasedTransliterator::MALFORMED_RHS, rules, start); |
| } |
| if (data->isVariableDefined(left)) { |
| return syntaxError(RuleBasedTransliterator::DUPLICATE_VARIABLE_DEFINITION, rules, start); |
| } |
| data->defineVariable(left, buf.charAt(0), status); |
| break; |
| |
| case FORWARD_RULE_OP: |
| if (direction == RuleBasedTransliterator::FORWARD) { |
| if (ante >= 0 || post >= 0 || leftCursor >= 0) { |
| return syntaxError(RuleBasedTransliterator::MALFORMED_RULE, rules, start); |
| } |
| data->ruleSet.addRule(new TransliterationRule( |
| left, leftAnte, leftPost, |
| buf, cursor, status), status); |
| } // otherwise ignore the rule; it's not the direction we want |
| break; |
| |
| case REVERSE_RULE_OP: |
| if (direction == RuleBasedTransliterator::REVERSE) { |
| if (leftAnte >= 0 || leftPost >= 0 || cursor >= 0) { |
| return syntaxError(RuleBasedTransliterator::MALFORMED_RULE, rules, start); |
| } |
| data->ruleSet.addRule(new TransliterationRule( |
| buf, ante, post, |
| left, leftCursor, status), status); |
| } // otherwise ignore the rule; it's not the direction we want |
| break; |
| |
| case FWDREV_RULE_OP: |
| if (direction == RuleBasedTransliterator::FORWARD) { |
| // The output side is the right; trim off any context |
| if (post >= 0) { |
| buf.remove(post); |
| } |
| if (ante >= 0) { |
| buf.removeBetween(0, ante); |
| } |
| data->ruleSet.addRule(new TransliterationRule( |
| left, leftAnte, leftPost, |
| buf, cursor, status), status); |
| } else { |
| // The output side is the left; trim off any context |
| if (leftPost >= 0) { |
| left.remove(leftPost); |
| } |
| if (leftAnte >= 0) { |
| left.removeBetween(0, leftAnte); |
| } |
| data->ruleSet.addRule(new TransliterationRule( |
| buf, ante, post, |
| left, leftCursor, status), status); |
| } |
| break; |
| } |
| |
| return pos; |
| } |
| |
| /** |
| * Called by main parser upon syntax error. Search the rule string |
| * for the probable end of the rule. Of course, if the error is that |
| * the end of rule marker is missing, then the rule end will not be found. |
| * In any case the rule start will be correctly reported. |
| * @param msg error description |
| * @param rule pattern string |
| * @param start position of first character of current rule |
| */ |
| int32_t TransliterationRuleParser::syntaxError(int32_t parseErrorCode, |
| const UnicodeString& rule, |
| int32_t start) { |
| if (parseError != 0) { |
| parseError->code = parseErrorCode; |
| parseError->line = 0; // We don't return a line # |
| parseError->offset = start; // Character offset from rule start |
| int32_t end = quotedIndexOf(rule, start, rule.length(), END_OF_RULE); |
| if (end < 0) { |
| end = rule.length(); |
| } |
| rule.extractBetween(start, end, parseError->context); // Current rule |
| } |
| status = U_ILLEGAL_ARGUMENT_ERROR; |
| return start; |
| } |
| |
| /** |
| * Allocate a private-use substitution character for the given set, |
| * register it in the setVariables hash, and return the substitution |
| * character. |
| */ |
| UChar TransliterationRuleParser::registerSet(UnicodeSet* adoptedSet) { |
| if (variableNext >= variableLimit) { |
| // throw new RuntimeException("Private use variables exhausted"); |
| status = U_ILLEGAL_ARGUMENT_ERROR; |
| return 0; |
| } |
| setVariablesVector.addElement(adoptedSet); |
| return variableNext++; |
| } |
| |
| /** |
| * Determines what part of the private use region of Unicode we can use for |
| * variable stand-ins. The correct way to do this is as follows: Parse each |
| * rule, and for forward and reverse rules, take the FROM expression, and |
| * make a hash of all characters used. The TO expression should be ignored. |
| * When done, everything not in the hash is available for use. In practice, |
| * this method may employ some other algorithm for improved speed. |
| */ |
| void TransliterationRuleParser::determineVariableRange(void) { |
| UnicodeRange privateUse(0xE000, 0x1900); // Private use area |
| |
| UnicodeRange* r = privateUse.largestUnusedSubrange(rules); |
| |
| data->setVariablesBase = variableNext = variableLimit = (UChar) 0; |
| |
| if (r != 0) { |
| data->setVariablesBase = variableNext = r->start; |
| variableLimit = (UChar) (r->start + r->length); |
| delete r; |
| } |
| |
| if (variableNext >= variableLimit) { |
| status = U_ILLEGAL_ARGUMENT_ERROR; |
| } |
| } |
| |
| /** |
| * Returns the index of a character, ignoring quoted text. |
| * For example, in the string "abc'hide'h", the 'h' in "hide" will not be |
| * found by a search for 'h'. |
| */ |
| int32_t TransliterationRuleParser::quotedIndexOf(const UnicodeString& text, |
| int32_t start, int32_t limit, |
| UChar charToFind) { |
| for (int32_t i=start; i<limit; ++i) { |
| UChar c = text.charAt(i); |
| if (c == ESCAPE) { |
| ++i; |
| } else if (c == QUOTE) { |
| while (++i < limit |
| && text.charAt(i) != QUOTE) {} |
| } else if (c == charToFind) { |
| return i; |
| } |
| } |
| return -1; |
| } |
