/**
*******************************************************************************
* Copyright (C) 1996-2001, International Business Machines Corporation and    *
* others. All Rights Reserved.                                                *
*******************************************************************************
*
* $Source: /xsrl/Nsvn/icu/unicodetools/com/ibm/text/UCD/GenerateBreakTest.java,v $
* $Date: 2006/09/24 23:32:44 $
* $Revision: 1.15 $
*
*******************************************************************************
*/

package com.ibm.text.UCD;

import java.util.*;
import java.io.*;

import org.unicode.cldr.util.Segmenter;

import com.ibm.text.utility.*;
import com.ibm.icu.dev.test.util.UnicodeMap;
import com.ibm.icu.dev.test.util.UnicodeProperty;
import com.ibm.icu.text.UTF16;
import com.ibm.icu.text.UnicodeSet;

abstract public class GenerateBreakTest implements UCD_Types {

    static boolean DEBUG = true;
    static final boolean SHOW_TYPE = false;
    UCD ucd;
    Normalizer nfd;
    Normalizer nfkd;
    
    OldUnicodeMap sampleMap = null;
    OldUnicodeMap map = new OldUnicodeMap();
    UnicodeProperty prop;
   
    // ====================== Main ===========================
    
    public static void main(String[] args) throws IOException {
        System.out.println("Remember to add length marks (half & full) and other punctuation for sentence, with FF61");
        //Default.setUCD();
        new GenerateGraphemeBreakTest(Default.ucd()).run();
        new GenerateWordBreakTest(Default.ucd()).run();
        new GenerateLineBreakTest(Default.ucd()).run();
        new GenerateSentenceBreakTest(Default.ucd()).run();
    }
    
    GenerateBreakTest(UCD ucd) {
        this.ucd = ucd;
        nfd = new Normalizer(Normalizer.NFD, ucd.getVersion());
        nfkd = new Normalizer(Normalizer.NFKD, ucd.getVersion());
        	/*
        public void fillMap(String propName) {
        	List list = y.getAvailableValues();
        	for (Iterator it = list.iterator(); it.hasNext();) {
        		String label = (String) it.next();
        		map.add(label, y.getSet(label));
        	}
        }
        */
    }
    
	ToolUnicodePropertySource unicodePropertySource = ToolUnicodePropertySource.make("");

	Set labels = new HashSet();
    
    int addToMap(String label) {
    	labels.add(label);
    	UnicodeSet s = prop.getSet(label);
    	if (s == null || s.size() == 0) throw new IllegalArgumentException("Bad value: " + prop.getName() + ", " + label);
    	return map.add(label, s);
    }

    int addToMapLast(String label) {
    	int result = addToMap(label);
    	Set values = new HashSet(prop.getAvailableValues());
    	if (!values.equals(labels)) throw new IllegalArgumentException("Missing Property Values: " + prop.getName()
    			+ ": " + values.removeAll(labels));
		return result;
    }

    // COMMON STUFF for Hangul
    /*
    static final byte hNot = -1, hL = 0, hV = 1, hT = 2, hLV = 3, hLVT = 4, hLIMIT = 5;
    static final String[] hNames = {"L", "V", "T", "LV", "LVT"};

    
    static byte getHangulType(int cp) {
        if (ucd.isLeadingJamo(cp)) return hL;
        if (ucd.isVowelJamo(cp)) return hV;
        if (ucd.isTrailingJamo(cp)) return hT;
        if (ucd.isHangulSyllable(cp)) {
            if (ucd.isDoubleHangul(cp)) return hLV;
            return hLVT;
        }
        return hNot;
    }
    */
    
   /* static { 
        setUCD();
    }
    */
    
    public static boolean onCodepointBoundary(String s, int offset) {
        if (offset < 0 || offset > s.length()) return false;
        if (offset == 0 || offset == s.length()) return true;
        if (UTF16.isLeadSurrogate(s.charAt(offset-1))
        && UTF16.isTrailSurrogate(s.charAt(offset))) return false;
        return true;
    }

    // finds the first base character, or the first character if there is no base
    public int findFirstBase(String source, int start, int limit) {
        int cp;
        for (int i = start; i < limit; i += UTF16.getCharCount(cp)) {
            cp = UTF16.charAt(source, i);
            byte cat = ucd.getCategory(cp);
            if (((1<<cat) & MARK_MASK) != 0) continue;
            return cp;
        }
        return UTF16.charAt(source, start);
    }

    // quick & dirty routine
    static String insertEverywhere(String source, String insertion, GenerateBreakTest breaker) {
        String result = insertion;
        for (int i = 0; i < source.length(); ++i) {
            result += source.charAt(i);
            if (breaker.isBreak(source, i)) {
                result += insertion;
            }
        }
        return result + insertion;
    }


    static void checkDecomps(UCD ucd) {
        UCDProperty[]  INFOPROPS = {UnifiedProperty.make(CATEGORY), UnifiedProperty.make(LINE_BREAK)};
        GenerateBreakTest[] tests = {
            new GenerateGraphemeBreakTest(ucd), 
            new GenerateWordBreakTest(ucd),
            new GenerateLineBreakTest(ucd),
        };
        tests[0].isBreak("\u0300\u0903", 1);
        Normalizer nfd = new Normalizer(Normalizer.NFD, ucd.getVersion());
        
        System.out.println("Check Decomps");
        //System.out.println("otherExtendSet: " + ((GenerateGraphemeBreakTest)tests[0]).otherExtendSet.toPattern(true));
        //Utility.showSetNames("", ((GenerateGraphemeBreakTest)tests[0]).otherExtendSet, false, ucd);
        
        for (int k = 0; k < tests.length; ++k) {
            for (int i = 0; i < 0x10FFFF; ++i) {
                if (!ucd.isAllocated(i)) continue;
                if (ucd.isHangulSyllable(i)) continue;
                if (nfd.isNormalized(i)) continue;
                String decomp = nfd.normalize(i);
                boolean shown = false;
                String test = decomp;
                for (int j = 1; j < test.length(); ++j) {
                    if (tests[k].isBreak(test, j)) {
                        if (!shown) {
                            System.out.println(showData(ucd, UTF16.valueOf(i), INFOPROPS, "\r\n\t"));
                            System.out.println(" => " + showData(ucd, decomp, INFOPROPS, "\r\n\t"));
                            shown = true;
                        }
                        System.out.println(j  + ": " + tests[k].fileName);
                    }
                }
            }
        }
    }
    
    static String showData(UCD ucd, String source, UCDProperty[] props, String separator) {
        StringBuffer result = new StringBuffer();
        int cp;
        for (int i = 0; i < source.length(); i += UTF16.getCharCount(cp)) {
            cp = UTF16.charAt(source, i);
            if (i != 0) result.append(separator);
            result.append(ucd.getCodeAndName(cp));
            for (int j = 0; j < props.length; ++j) {
                result.append(", ");
                result.append(props[j].getPropertyName(SHORT)).append('=').append(props[j].getValue(cp,SHORT));
            }
        }
        return result.toString();
    }
    
    void showSet(String title, UnicodeSet set) {
        System.out.println(title + ": " + set.toPattern(true));
        Utility.showSetNames("", set, false, ucd);
    }
    
    // determines if string is of form Base NSM*
    boolean isBaseNSMStar(String source) {
        int cp;
        int status = 0;
        for (int i = 0; i < source.length(); i += UTF16.getCharCount(cp)) {
            cp = UTF16.charAt(source, i);
            byte cat = ucd.getCategory(cp);
            int catMask = 1<<cat;
            switch(status) {
            case 0: if ((catMask & BASE_MASK) == 0) return false;
                    status = 1;
                    break;
            case 1: if ((catMask & NONSPACING_MARK_MASK) == 0) return false;
                    break;
            }
                    
        }
        return true;
    }

    UnicodeSet getClosure(UnicodeSet source) {
        UnicodeSet result = new UnicodeSet(source);
        for (int i = 0; i < 0x10FFFF; ++i) {
            if (!ucd.isAllocated(i)) continue;
            if (nfkd.isNormalized(i)) continue;
            String decomp = nfkd.normalize(i);
            if (source.containsAll(decomp)) result.add(i);
        }
        return result;
    }
    
    
    /*
    static UnicodeSet extraAlpha = new UnicodeSet("[\\u02B9-\\u02BA\\u02C2-\\u02CF\\u02D2-\\u02DF\\u02E5-\\u02ED\\u05F3]");
    static UnicodeSet alphabeticSet = UnifiedBinaryProperty.make(DERIVED | PropAlphabetic).getSet()
        .addAll(extraAlpha);
        
    static UnicodeSet ideographicSet = UnifiedBinaryProperty.make(BINARY_PROPERTIES | Ideographic).getSet();
    
    static {
        if (false) System.out.println("alphabetic: " + alphabeticSet.toPattern(true));
    }
    */


    void generateTerminalClosure() {
        UnicodeSet midLetterSet = new UnicodeSet("[\u0027\u002E\u003A\u00AD\u05F3\u05F4\u2019\uFE52\uFE55\uFF07\uFF0E\uFF1A]");

        UnicodeSet ambigSentPunct = new UnicodeSet("[\u002E\u0589\u06D4]");

        UnicodeSet sentPunct = new UnicodeSet("[\u0021\u003F\u0387\u061F\u0964\u203C\u203D\u2048\u2049"
            + "\u3002\ufe52\ufe57\uff01\uff0e\uff1f\uff61]");
        
        UnicodeSet terminals = UnifiedBinaryProperty.make(BINARY_PROPERTIES | Terminal_Punctuation).getSet();
        UnicodeSet extras = getClosure(terminals).removeAll(terminals);
        System.out.println("Current Terminal_Punctuation");
        Utility.showSetNames("", terminals, true, ucd);

        System.out.println("Missing Terminal_Punctuation");
        Utility.showSetNames("", extras, true, ucd);

        System.out.println("midLetterSet");
        System.out.println(midLetterSet.toPattern(true));
        Utility.showSetNames("", midLetterSet, true, ucd);

        System.out.println("ambigSentPunct");
        System.out.println(ambigSentPunct.toPattern(true));
        Utility.showSetNames("", ambigSentPunct, true, ucd);

        System.out.println("sentPunct");
        System.out.println(sentPunct.toPattern(true));
        Utility.showSetNames("", sentPunct, true, ucd);
        /*

        UnicodeSet sentencePunctuation = new UnicodeSet("[\u0021\003F          ; Terminal_Punctuation # Po       QUESTION MARK
037E          ; Terminal_Punctuation # Po       GREEK QUESTION MARK
061F          ; Terminal_Punctuation # Po       ARABIC QUESTION MARK
06D4          ; Terminal_Punctuation # Po       ARABIC FULL STOP
203C..203D    ; Terminal_Punctuation # Po   [2] DOUBLE EXCLAMATION MARK..INTERROBANG
3002          ; Terminal_Punctuation # Po       IDEOGRAPHIC FULL STOP
2048..2049    ; Terminal_Punctuation # Po   [2] QUESTION EXCLAMATION MARK..EXCLAMATION QUESTION MARK
        */

    }

    //============================

    protected String currentRule;
    protected String fileName;
    protected String[] samples = new String[100];
    protected String[] extraSamples = new String[0];
    protected String[] extraSingleSamples = new String[0];
    protected int sampleLimit = 0;
    protected int tableLimit = -1;
    
    protected int[] skippedSamples = new int[100];
    protected boolean didSkipSamples = false;
    
    private String[] ruleList = new String[100];
    private int ruleListCount = 0;
    protected boolean collectingRules = false;
	protected boolean needsFullBreakSample = true;
    
    public void setRule(String rule) {
        if (collectingRules) {
            ruleList[ruleListCount++] = rule;
        }
        currentRule = rule;
    }

    public String getRule() {
        return currentRule;
    }

    public void run() throws IOException {
        findSamples();

        // test individual cases
        //printLine(out, samples[LB_ZW], "", samples[LB_CL]);
        //printLine(out, samples[LB_ZW], " ", samples[LB_CL]);

        UnicodeDataFile fc = UnicodeDataFile.openHTMLAndWriteHeader("DerivedData\\auxiliary\\", fileName + "BreakTest");
        PrintWriter out = fc.out;

/*        PrintWriter out = Utility.openPrintWriter("auxiliary\\" 
            + fileName + "BreakTest-"
            + ucd.getVersion()
            + ".html", Utility.UTF8_WINDOWS);
*/        
        out.println("<!doctype HTML PUBLIC '-//W3C//DTD HTML 4.0 Transitional//EN' 'http://www.w3.org/TR/REC-html40/loose.dtd'>");
        out.println("<html><head><meta http-equiv='Content-Type' content='text/html; charset=utf-8'>");
        out.println("<title>" + fileName + " Break Chart</title>");
        out.println("<style type='text/css'>");
        out.println("td, th { vertical-align: top }");
        out.println("</style></head>");


        out.println("<body bgcolor='#FFFFFF'>");
        out.println("<h2>" + fileName + " Break Chart</h2>");
        out.println("<p><b>Unicode Version:</b> " + ucd.getVersion() + "</p>");
        out.println("<p><b>Date:</b> " + Default.getDate() + "</p>");
        out.println("<p>This page illustrates the application of the boundary specifications. " +
        		"The first chart shows where breaks would appear between different sample characters or strings. " +
        		"The sample characters are chosen mechanically to represent the different properties used by the specification. " +
        		"Where properties used in the rules have 'overlaps', the samples are given 'composed' names. " +
        		"For example, SentenceBreak uses GCLF_Sep: Sep is the SentenceBreak property, but it overlaps with the GraphemeClusterBreak property LF." +
        		"</p>");
        generateTable(out);
        

        if (false) {
            out.println("<h3>Character Type Breakdown</h3>");
            out.println("<table border='1' cellspacing='0' width='100%'>");
            for (int i = 0; i < sampleMap.size(); ++i) {
                out.println("<tr><th>" + sampleMap.getLabelFromIndex(i) 
                    + "</th><td>" + sampleMap.getSetFromIndex(i)
                    + "</td></tr>");
            }
            out.println("</table>");
        }

        fc.close();
        
        generateTest(false);
        
    }
    
    public void generateTest(boolean shortVersion) throws IOException {
        String[] testCase = new String[50];
        // do main test

        UnicodeDataFile fc = UnicodeDataFile.openAndWriteHeader("DerivedData\\auxiliary\\", fileName + "BreakTest" 
                + (shortVersion ? "_SHORT" : ""));
        PrintWriter out = fc.out;
/*        PrintWriter out = Utility.openPrintWriter("TR29\\" + fileName + "BreakTest" 
            + (shortVersion ? "_SHORT" : "")
            + "-" + ucd.getVersion()
            + ".txt", Utility.UTF8_WINDOWS);
*/        
        int counter = 0;

        out.println("#");
        out.println("# Default " + fileName + " Break Test");
        out.println("#");
        out.println("# Format:");
        out.println("# <string> (# <comment>)? ");
        out.println("#  <string> contains hex Unicode code points, with ");
        out.println("#\t" + BREAK + " wherever there is a break opportunity, and ");
        out.println("#\t" + NOBREAK + " wherever there is not.");
        out.println("#  <comment> the format can change, but currently it shows:");
        out.println("#\t- the sample character name");
        out.println("#\t- (x) the line_break property* for the sample character");
        out.println("#\t- [x] the rule that determines whether there is a break or not");
        out.println("#");
        sampleDescription(out);
        out.println("# These samples may be extended or changed in the future.");
        out.println("#");

        for (int ii = 0; ii < sampleLimit; ++ii) {
            String before = samples[ii];

            for (int jj = 0; jj < sampleLimit; ++jj) {
                Utility.dot(counter);
                String after = samples[jj];

                // do line straight
                int len = genTestItems(before, after, testCase);
                for (int q = 0; q < len; ++q) {
                    printLine(out, testCase[q], !shortVersion && q == 0, false);
                    ++counter;
                }
            }
        }

        for (int ii = 0; ii < extraSingleSamples.length; ++ii) {
            printLine(out, extraSingleSamples[ii], true, false);
        }
        out.println("# Lines: " + counter);
        fc.close();
    }

    public void sampleDescription(PrintWriter out) {}

    abstract public boolean isBreak(String source, int offset);
    
    abstract public String fullBreakSample();

    abstract public byte getType (int cp);
    
    public byte getSampleType (int cp) {
        return getType(cp);
    }
    
    public int mapType(int input) {
        return input;
    }
    
    public boolean highlightTableEntry(int x, int y, String s) {
        return false;
    }

    abstract public String getTypeID(int s);

    public String getTypeID(String s) {
        if (s == null) return "<null>";
        if (s.length() == 1) return getTypeID(s.charAt(0));
        StringBuffer result = new StringBuffer();
        int cp;
        for (int i = 0; i < s.length(); i += UTF32.count16(cp)) {
            cp = UTF32.char32At(s, i);
            if (i > 0) result.append(" ");
            result.append(getTypeID(cp));
        }
        return result.toString();
    }

    static final int DONE = -1;

    public int next(String source, int offset) {
        for (int i = offset + 1; i <= source.length(); ++i) {
            if (isBreak(source, i)) return i;
        }
        return DONE;
    }

    public int previous(String source, int offset) {
        for (int i = offset - 1; i >= 0; --i) {
            if (isBreak(source, i)) return i;
        }
        return DONE;
    }

    public int genTestItems(String before, String after, String[] results) {
        results[0] = before + after;
        return 1;
    }

    public String getTableEntry(String before, String after, String[] ruleOut) {
        boolean normalBreak = isBreak(before + after, before.length());
        String normalRule = getRule();
        ruleOut[0] = normalRule;
        return normalBreak ? BREAK : NOBREAK;
    }

    public byte getResolvedType(int cp) {
        return getType(cp);
    }

    boolean skipType(int type) {
        return false;
    }

    String getInfo(String s) {
        if (s == null || s.length() == 0) return "NULL";
        StringBuffer result = new StringBuffer();
        int cp;
        for (int i = 0; i < s.length(); i += UTF32.count16(cp)) {
            cp = UTF32.char32At(s, i);
            if (i > 0) result.append(", ");
            result.append(ucd.getCodeAndName(cp));
            result.append(", gc=" + ucd.getCategoryID_fromIndex(ucd.getCategory(cp),SHORT));
            result.append(", sc=" + ucd.getScriptID_fromIndex(ucd.getScript(cp),SHORT));
            //result.append(", lb=" + ucd.getLineBreakID_fromIndex(ucd.getLineBreak(cp))
            //    + "=" + ucd.getLineBreakID_fromIndex(ucd.getLineBreak(cp), LONG));
        }
        return result.toString();
    }

    public void generateTable(PrintWriter out) {
        String width = "width='" + (100 / (tableLimit + 1)) + "%'";
        out.print("<table border='1' cellspacing='0' width='100%'>");
        String types = "";
        String codes = "";
        for (int type = 0; type < tableLimit; ++type) {
            String after = samples[type];
            if (after == null) continue;

            String h = getTypeID(after);
            types += "<th " + width + " class='lbclass' title='" + getInfo(after) + "'>" + h + "</th>";
            
            
            //codes += "<th " + width + " title='" + getInfo(after) + "'>" + Utility.hex(after) + "</th>";
        }

        out.println("<tr><th " + width + "></th>" + types + "</tr>");
        // out.println("<tr><th " + width + "></th><th " + width + "></th>" + codes + "</tr>");

        String[] rule = new String[1];
        String[] rule2 = new String[1];
        for (int type = 0; type < sampleLimit; ++type) {
            if (type == tableLimit) {
                out.println("<tr><td bgcolor='#0000FF' colSpan='" + (tableLimit + 1) + "' style='font-size: 1px'>&nbsp;</td></tr>");
            }
            String before = samples[type];
            if (before == null) continue;

            String h = getTypeID(before);
            String line = "<tr><th class='lbclass' title='" + ucd.getCodeAndName(before) + "'>" + h + "</th>";

            for (int type2 = 0; type2 < tableLimit; ++type2) {
                
                String after = samples[type2];
                if (after == null) continue;

                String t = getTableEntry(before, after, rule);
                String background = "";
                String t2 = getTableEntry(before, after, rule2);
                if (highlightTableEntry(type, type2, t)) {
                    background = " bgcolor='#FFFF00'";
                }
                if (!t.equals(t2)) {
                    if (t.equals(NOBREAK)) {
                        background = " bgcolor='#CCFFFF'";
                    } else {
                        background = " bgcolor='#FFFF00'";
                    }
                } else if (t.equals(NOBREAK)) {
                    background = " bgcolor='#CCCCFF'";
                }
                line += "<th title='" + rule[0] + "'" + background + " class='pairItem'>" + t + "</th>";
            }
            out.println(line + "</tr>");
        }
        out.println("</table>");
            
        if (didSkipSamples) {
            out.println("<p><b>Suppressed:</b> ");
            for (int i = 0; i < skippedSamples.length; ++i) {
                if (skippedSamples[i] > 0) {
                    String tmp = UTF16.valueOf(skippedSamples[i]);
                    out.println("<span title='" + getInfo(tmp) + "'>" + getTypeID(tmp) + "</span>");
                }
            }
            out.println("</p>");
        }
        
        // gather the data for the rules
        if (needsFullBreakSample ) {
        collectingRules = true;
        isBreak(fullBreakSample(), 1);
        collectingRules = false;
        }
        
        out.println("<h3>Rules</h3>");
        out.println("<p>Due to the way they have been mechanically processed for generation, " +
        		"the following rules do not match the UAX rules precisely. " +
        		"In particular:</p>"+
        		"<ol>" +
        		"<li>The rules are cast into a more regex-style.</li>"+
        		"<li>The rules \"sot ÷\", \"÷ eot\", and \"÷ Any\" are added mechanically, and have artificial numbers.</li>"+
        		"<li>The rules are given decimal numbers, so rules such as 11a are given a number using tenths, such as 11.1.</li>"+
        		"<li>Where a rule has multiple parts (lines), each one is numbered using hundredths, such as 21.01) × BA, 21.02) × HY,...</li>"+
        		"<li>Any 'treat as' or 'ignore' rules are handled as discussed in Unicode Standard Annex #29, and thus" +
        		"reflected in a transformation of the rules not visible here.</li>" +
        		"</ol>" +
        		"<p>For the original rules, see the UAX.</p>"

        		);
        out.println("<ul style='list-style-type: none'>");
            for (int ii = 0; ii < ruleListCount; ++ii) {
                out.println("<li>" + ruleList[ii].replaceAll("[$]","") + "</li>");
            }
        out.println("</ul>");
        
        if (extraSingleSamples.length > 0) {
            out.println("<h3>Sample Strings</h3>");
            out.println("<p>" +
            		"The following samples illustrate the application of the rules. " +
            		"The blue lines indicate possible break points. " +
            		"If your browser supports titles, then positioning the mouse over each character will show its name, " +
            		"white positioning between characters shows the rule number of the rule responsible for the break-status." +
            		"</p>");
            out.println("<ol>");
                for (int ii = 0; ii < extraSingleSamples.length; ++ii) {
                    out.println("<li><font size='5'>");
                    printLine(out, extraSingleSamples[ii], true, true);
                    out.println("</font></li>");
                }
            out.println("</ol>");
        }
    }

    static final String BREAK = "\u00F7";
    static final String NOBREAK = "\u00D7";

    public void printLine(PrintWriter out, String source, boolean comments, boolean html) {
        int cp;
        StringBuffer string = new StringBuffer();
        StringBuffer comment = new StringBuffer("\t# ");
        boolean hasBreak = isBreak(source, 0);
        String status;
        if (html) {
            status = hasBreak ? " style='border-right: 1px solid blue'" : "";
            string.append("<span title='" + getRule() + "'><span" + status + ">&nbsp;</span>&nbsp;</span>");
        } else {
            status = hasBreak ? BREAK : NOBREAK;
            string.append(status);
        }
        comment.append(' ').append(status).append(" [").append(getRule()).append(']');

        for (int offset = 0; offset < source.length(); offset += UTF16.getCharCount(cp)) {

            cp = UTF16.charAt(source, offset);
            hasBreak = isBreak(source, offset + UTF16.getCharCount(cp));

            if (html) {
                status = hasBreak ? " style='border-right: 1px solid blue'" : "";
                string.append("<span title='" +
                    Utility.quoteXML(ucd.getCodeAndName(cp) + " (" + getTypeID(cp) + ")", true)
                    + "'>"
                    + Utility.quoteXML(Utility.getDisplay(cp), true)
                    + "</span>");
                string.append("<span title='" + getRule() + "'><span" + status + ">&nbsp;</span>&nbsp;</span>");
            } else {
                if (string.length() > 0) {
                    string.append(' ');
                    comment.append(' ');
                }

                status = hasBreak ? BREAK : NOBREAK;

                string.append(Utility.hex(cp));
                comment.append(ucd.getName(cp) + " (" + getTypeID(cp) + ")");
                string.append(' ').append(status);
                comment.append(' ').append(status).append(" [").append(getRule()).append(']');
            }
        }

        if (comments && !html) string.append(comment);
        out.println(string);
        if (DEBUG) System.out.println("*" + string);
    }

    public void findSamples() {

        // what we want is a list of sample characters. In the simple case, this is just one per type.
        // However, if there are characters that have different types (when recommended or not), then
        // we want a type for each cross-section

        BitSet bitset = new BitSet();
        Map list = new TreeMap();

        for (int i = 1; i <= 0xFFFF; ++i) {
            if (!ucd.isAllocated(i)) continue;
            if (0xD800 <= i && i <= 0xDFFF) continue;
            if (DEBUG && i == 0x1100) {
                System.out.println("debug");
            }
            byte lb = getSampleType(i);
            byte lb2 = lb; // HACK
            if (lb == lb2 && skipType(lb)) {
                skippedSamples[lb] = i;
                didSkipSamples = true;
                continue;
            }

            int combined = (mapType(lb) << 7) + mapType(lb2);
            if (combined < 0) {
            	throw new IllegalArgumentException("should never happen");
            }
            if (!bitset.get(combined)) {
                bitset.set(combined);
                list.put(new Integer(combined), UTF16.valueOf(i));
            }
            /*
            // if the sample slot is full OR
            if (samples[lb] == null) {
                samples[lb] = UTF16.valueOf(i);
                if (sampleLimit <= lb) sampleLimit = lb + 1;
                // byte lb2 = getType(i, true);
                // if (lb2 != lb) bs.set(lb);
            }
            */
        }

        Iterator it = list.keySet().iterator();
        while (it.hasNext()) {
            String sample = (String)list.get(it.next());
            samples[sampleLimit++] = sample;
            if (DEBUG) System.out.println(getTypeID(sample) + ":\t" + ucd.getCodeAndName(sample));
        }

        tableLimit = sampleLimit;

        // now add values that are different
        /*

        for (int i = 1; i <= 0x10FFFF; ++i) {
            if (!ucd.isAllocated(i)) continue;
            if (0xD800 <= i && i <= 0xDFFF) continue;
            byte lb = getType(i);
            byte lb2 = getType(i, true);
            if (lb == lb2) continue;
            // pick some different ones
            if (!bs.get(lb)) {
                samples[sampleLimit++] = UTF16.valueOf(i);
                bs.set(lb);
            }
            if (!bs2.get(lb2)) {
                samples[sampleLimit++] = UTF16.valueOf(i);
                bs.set(lb2);
            }
        }
        */

        if (extraSamples.length > 0) {
            System.arraycopy(extraSamples, 0, samples, sampleLimit, extraSamples.length);
            sampleLimit += extraSamples.length;
        }
    }

    public int findLastNon(String source, int offset, byte notLBType) {
        int cp;
        for (int i = offset-1; i >= 0; i -= UTF16.getCharCount(cp)) {
            cp = UTF16.charAt(source, i);
            byte f = getResolvedType(cp);
            if (f != notLBType) return i;
        }
        return -1;
    }

    public static UnicodeSet getSet(UCD ucd, int prop, byte propValue) {
        return UnifiedBinaryProperty.make(prop | propValue, ucd).getSet();
    }

    static public class Context {
        public int cpBefore2, cpBefore, cpAfter, cpAfter2;
        public byte tBefore2, tBefore, tAfter, tAfter2;
        public String toString() {
            return "[" 
            + Utility.hex(cpBefore2) + "(" + tBefore2 + "), "
            + Utility.hex(cpBefore) + "(" + tBefore + "), "
            + Utility.hex(cpAfter) + "(" + tAfter + "), "
            + Utility.hex(cpAfter2) + "(" + tAfter2 + ")]";
        }
    }

    public void getGraphemeBases(MyBreakIterator graphemeIterator, String source, int offset, int ignoreType, Context context) {
        context.cpBefore2 = context.cpBefore = context.cpAfter = context.cpAfter2 = -1;
        context.tBefore2 = context.tBefore = context.tAfter = context.tAfter2 = -1;
        //if (DEBUG_GRAPHEMES) System.out.println(Utility.hex(source) + "; " + offset + "; " + ignoreType);
            
        //MyBreakIterator graphemeIterator = new MyBreakIterator(new GenerateGraphemeBreakTest(ucd));

        graphemeIterator.set(source, offset);
        while (true) {
            int cp = graphemeIterator.previousBase();
            if (cp == -1) break;
            byte t = getResolvedType(cp);
            if (t == ignoreType) continue;
                
            if (context.cpBefore == -1) {
                context.cpBefore = cp;
                context.tBefore = t;
            } else {
                context.cpBefore2 = cp;
                context.tBefore2 = t;
                break;
            }
        }
        graphemeIterator.set(source, offset);
        while (true) {
            int cp = graphemeIterator.nextBase();
            if (cp == -1) break;
            byte t = getResolvedType(cp);
            if (t == ignoreType) continue;
                
            if (context.cpAfter == -1) {
                context.cpAfter = cp;
                context.tAfter = t;
            } else {
                context.cpAfter2 = cp;
                context.tAfter2 = t;
                break;
            }
        }
    }


    //==============================================
    
    static class XGenerateBreakTest extends GenerateBreakTest {
    	Segmenter seg;
    	String sample;
    	{
    		needsFullBreakSample = false;
    	}
    	
    	public XGenerateBreakTest(UCD ucd, Segmenter.Builder segBuilder, String sample, String filename, String[] extraSamples, String[] extraSingleSamples) {
    		super(ucd);
    		this.seg = segBuilder.make();
    		this.sample = sample;
    		List rules = segBuilder.getRules();
    		collectingRules = true;
    		for (Iterator it = rules.iterator(); it.hasNext();) {
    			String rule = (String)it.next();
    			setRule(rule);
    		}
    		collectingRules = false;
    		map.add("Other", new UnicodeSet(0,0x10FFFF));
    		UnicodeMap segSamples = seg.getSamples();
    		Collection x = segSamples.getAvailableValues();
    		for (Iterator it = x.iterator(); it.hasNext();) {
    			String label = (String)it.next();
    			map.add(label, segSamples.getSet(label), true, false);
    		}
            this.fileName = filename;
            sampleMap = map;
            this.extraSamples = extraSamples;
            this.extraSingleSamples = extraSingleSamples;
    	}

		public boolean isBreak(String source, int offset) {
			boolean result = seg.breaksAt(source, offset);
			setRule(String.valueOf(seg.getBreakRule()));
			return result;
		}

		public String fullBreakSample() {
			return sample;
		}

        // stuff that subclasses need to override
        public String getTypeID(int cp) {
            return map.getLabel(cp);
        }

        // stuff that subclasses need to override
        public byte getType(int cp) {
            return (byte) map.getIndex(cp);
        }    
    }
    
    static class GenerateGraphemeBreakTest extends XGenerateBreakTest {
		public GenerateGraphemeBreakTest(UCD ucd) {
	        super(ucd, Segmenter.make(ToolUnicodePropertySource.make(ucd.getVersion()),"GraphemeClusterBreak"), "aa", "Grapheme",
	        		new String[]{}, new String[]{});
		}	
    }

    static class GenerateLineBreakTest extends XGenerateBreakTest {
		public GenerateLineBreakTest(UCD ucd) {
	        super(ucd, Segmenter.make(ToolUnicodePropertySource.make(ucd.getVersion()),"LineBreak"), "aa", "Line",
	        	new String[]{}, new String[] {
	        	"can't", "can\u2019t", "ab\u00ADby",
	             "-3",
	             "e.g.",
	             "\u4e00.\u4e00.",
	              "a  b",
	              "a  \u200bb",
	              "a \u0308b",
	              "1\u0308b(a)-(b)",
	              });
		}	
    }
    
    static class GenerateSentenceBreakTest extends XGenerateBreakTest {
		public GenerateSentenceBreakTest(UCD ucd) {
	        super(ucd, Segmenter.make(ToolUnicodePropertySource.make(ucd.getVersion()),"SentenceBreak"), "aa", "Sentence",
	        		new String[]{},
	        		getExtraSamples());
		}	
		static String[] getExtraSamples() {
            GenerateBreakTest grapheme = new GenerateGraphemeBreakTest(Default.ucd());
	        String[] extraSingleSamples = new String[] {
	                "(\"Go.\") (He did.)", 
	                "(\u201CGo?\u201D) (He did.)", 
	                "U.S.A\u0300. is", 
	                "U.S.A\u0300? He", 
	                "U.S.A\u0300.", 
	                "3.4", 
	                "c.d",
	                "etc.)\u2019 \u2018(the",
	                "etc.)\u2019 \u2018(The",
	                "the resp. leaders are",
	                "\u5B57.\u5B57",
	                "etc.\u5B83",
	                "etc.\u3002",
	                "\u5B57\u3002\u5B83",
	            };
	            String[] temp = new String [extraSingleSamples.length * 2];
	            System.arraycopy(extraSingleSamples, 0, temp, 0, extraSingleSamples.length);
	            for (int i = 0; i < extraSingleSamples.length; ++i) {
	                temp[i+extraSingleSamples.length] = insertEverywhere(extraSingleSamples[i], "\u2060", grapheme);
	            }
	            extraSingleSamples = temp;
	            return extraSingleSamples;
		}
    }

    static class GenerateWordBreakTest extends XGenerateBreakTest {
		public GenerateWordBreakTest(UCD ucd) {
	        super(ucd, Segmenter.make(ToolUnicodePropertySource.make(ucd.getVersion()),"WordBreak"), "aa", "Word",
	    	        new String[] {
                /*"\uFF70", "\uFF65", "\u30FD", */ "a\u2060", "a:", "a'", "a'\u2060", "a,", "1:", "1'", "1,",  "1.\u2060"
            	},


	        	getExtraSamples());
		}	
		static String[] getExtraSamples() {
            GenerateBreakTest grapheme = new GenerateGraphemeBreakTest(Default.ucd());
	                String [] temp = {"can't", "can\u2019t", "ab\u00ADby", "a$-34,567.14%b", "3a" };
	                String[] extraSingleSamples = new String [temp.length * 2];
	                System.arraycopy(temp, 0, extraSingleSamples, 0, temp.length);
	                for (int i = 0; i < temp.length; ++i) {
	                    extraSingleSamples[i+temp.length] = insertEverywhere(temp[i], "\u2060", grapheme);
	                }

	            return extraSingleSamples;
		}
    }

    static class OLDGenerateGraphemeBreakTest extends GenerateBreakTest {

    	OLDGenerateGraphemeBreakTest(UCD ucd) {
            super(ucd);
            fileName = "Grapheme";
            sampleMap = map;
        }

        Object foo = prop = unicodePropertySource.getProperty("Grapheme_Cluster_Break");

        final int
            CR =    addToMap("CR"),
            LF =    addToMap("LF"),
            Control = addToMap("Control"),
            Extend = addToMap("Extend"),
            L =     addToMap("L"),
            V =     addToMap("V"),
            T =     addToMap("T"),
            LV =    addToMap("LV"),
            LVT =   addToMap("LVT"),
            Other = addToMapLast("Other");            
                
        // stuff that subclasses need to override
        public String getTypeID(int cp) {
            return map.getLabel(cp);
        }

        // stuff that subclasses need to override
        public byte getType(int cp) {
            return (byte) map.getIndex(cp);
        }
        
        public String fullBreakSample() {
            return "aa";
        }

        public boolean isBreak(String source, int offset) {
            
            setRule("1: sot ÷");
            if (offset < 0 || offset > source.length()) return false;
            if (offset == 0) return true;

            setRule("2: ÷ eot");
            if (offset == source.length()) return true;

            // UTF-16: never break in the middle of a code point
            if (!onCodepointBoundary(source, offset)) return false;

            // now get the character before and after, and their types


            int cpBefore = UTF16.charAt(source, offset-1);
            int cpAfter = UTF16.charAt(source, offset);

            byte before = getResolvedType(cpBefore);
            byte after = getResolvedType(cpAfter);

            setRule("3: CR × LF");
            if (before == CR && after == LF) return false;

            setRule("4: ( Control | CR | LF ) ÷");
            if (before == CR || before == LF || before == Control) return true;

            setRule("5: ÷ ( Control | CR | LF )");
            if (after == Control || after == LF || after == CR) return true;

            setRule("6: L × ( L | V | LV | LVT )");
            if (before == L && (after == L || after == V || after == LV || after == LVT)) return false;

            setRule("7: ( LV | V ) × ( V | T )");
            if ((before == LV || before == V) && (after == V || after == T)) return false;

            setRule("8: ( LVT | T ) × T");
            if ((before == LVT || before == T) && (after == T)) return false;

            setRule("9: × Extend");
            if (after == Extend) return false;

            // Otherwise break after all characters.
            setRule("10: Any ÷ Any");
            return true;

        }

    }

    //==============================================

    static class XGenerateWordBreakTest extends GenerateBreakTest {
        
        GenerateGraphemeBreakTest grapheme;
        MyBreakIterator breaker;
        Context context = new Context();

        XGenerateWordBreakTest(UCD ucd) {
            super(ucd);
            grapheme = new GenerateGraphemeBreakTest(ucd);
            breaker = new MyBreakIterator(grapheme);
            fileName = "Word";
            sampleMap = map;
            extraSamples = new String[] {
                /*"\uFF70", "\uFF65", "\u30FD", */ "a\u2060", "a:", "a'", "a'\u2060", "a,", "1:", "1'", "1,",  "1.\u2060"
            };

            String [] temp = {"can't", "can\u2019t", "ab\u00ADby", "a$-34,567.14%b", "3a" };
            extraSingleSamples = new String [temp.length * 2];
            System.arraycopy(temp, 0, extraSingleSamples, 0, temp.length);
            for (int i = 0; i < temp.length; ++i) {
                extraSingleSamples[i+temp.length] = insertEverywhere(temp[i], "\u2060", grapheme);
            }
        
            if (false) Utility.showSetDifferences("Katakana", map.getSetFromIndex(Katakana), 
                "Script=Katakana", getSet(ucd, SCRIPT, KATAKANA_SCRIPT), false, ucd);

        }
        
        Object foo = prop = unicodePropertySource.getProperty("Word_Break");

        //static String LENGTH = "[\u30FC\uFF70]";
        //static String HALFWIDTH_KATAKANA = "[\uFF66-\uFF9F]";
        //static String KATAKANA_ITERATION = "[\u30FD\u30FE]";
        //static String HIRAGANA_ITERATION = "[\u309D\u309E]";
        
        final int
            Format =    addToMap("Format"),
            Katakana =    addToMap("Katakana"),
            ALetter = addToMap("ALetter"),
            MidLetter = addToMap("MidLetter"),
            //MidNumLet =     addToMap("MidNumLet"),
            MidNum =     addToMap("MidNum"),
            Numeric =     addToMap("Numeric"),
            ExtendNumLet =     addToMap("ExtendNumLet"),
            Other = addToMapLast("Other");      

        // stuff that subclasses need to override
        public String getTypeID(int cp) {
            return map.getLabel(cp);
        }

        // stuff that subclasses need to override
        public byte getType(int cp) {
            return (byte) map.getIndex(cp);
        }
        
        public String fullBreakSample() {
            return " a";
        }

        public int genTestItems(String before, String after, String[] results) {
            results[0] = before + after;
            results[1] = 'a' + before + "\u0301\u0308" + after + "\u0301\u0308" + 'a';
            results[2] = 'a' + before + "\u0301\u0308" + samples[MidLetter] + after + "\u0301\u0308" + 'a';
            results[3] = 'a' + before + "\u0301\u0308" + samples[MidNum] + after + "\u0301\u0308" + 'a';
            return 3;
        }

        public boolean isBreak(String source, int offset) {

            setRule("1: sot ÷");
            if (offset < 0 || offset > source.length()) return false;
  
            if (offset == 0) return true;

            setRule("2: ÷ eot");
            if (offset == source.length()) return true;

            // Treat a grapheme cluster as if it were a single character:
            // the first base character, if there is one; otherwise the first character.

            setRule("3: GC -> FC");
            if (!grapheme.isBreak( source,  offset)) return false;

            setRule("4: X Format* -> X");
            byte afterChar = getResolvedType(source.charAt(offset));
            if (afterChar == Format) return false;

            // now get the base character before and after, and their types

            getGraphemeBases(breaker, source, offset, Format, context);

            byte before = context.tBefore;
            byte after = context.tAfter;
            byte before2 = context.tBefore2;
            byte after2 = context.tAfter2;

            //Don't break between most letters

            setRule("5: ALetter × ALetter");
            if (before == ALetter && after == ALetter) return false;

            // Don’t break letters across certain punctuation

            setRule("6: ALetter × MidLetter ALetter");
            if (before == ALetter && after == MidLetter && after2 == ALetter) return false;

            setRule("7: ALetter (MidLetter | MidNumLet) × ALetter");
            if (before2 == ALetter && before == MidLetter && after == ALetter) return false;

            // Don’t break within sequences of digits, or digits adjacent to letters.

            setRule("8: Numeric × Numeric");
            if (before == Numeric && after == Numeric) return false;

            setRule("9: ALetter × Numeric");
            if (before == ALetter && after == Numeric) return false;

            setRule("10: Numeric × ALetter");
            if (before == Numeric && after == ALetter) return false;


            // Don’t break within sequences like: '-3.2'
            setRule("11: Numeric (MidNum | MidNumLet) × Numeric");
            if (before2 == Numeric && before == MidNum && after == Numeric) return false;

            setRule("12: Numeric × (MidNum | MidNumLet) Numeric");
            if (before == Numeric && after == MidNum && after2 == Numeric) return false;

            // Don't break between Katakana

            setRule("13: Katakana × Katakana");
            if (before == Katakana && after == Katakana) return false;
            
            // Do not break from extenders
            setRule("13a: (ALetter | Numeric | Katakana | ExtendNumLet)  	×  	ExtendNumLet");
            if ((before == ALetter || before == Numeric || before == Katakana || before == ExtendNumLet) && after == ExtendNumLet) return false;

            setRule("13b: ExtendNumLet 	× 	(ALetter | Numeric | Katakana)");
            if (before == ExtendNumLet && (after == ALetter || after == Numeric || after == Katakana)) return false;

            // Otherwise break always.
            setRule("14: Any ÷ Any");
            return true;

        }

    }

    // ========================================

    static class XGenerateLineBreakTest extends GenerateBreakTest {

        GenerateGraphemeBreakTest grapheme;
        MyBreakIterator breaker;
        Context context = new Context();

        XGenerateLineBreakTest(UCD ucd) {
            super(ucd);
            grapheme = new GenerateGraphemeBreakTest(ucd);
            breaker = new MyBreakIterator(grapheme);

            sampleMap = map;
            fileName = "Line";
            extraSingleSamples = new String[] {"can't", "can\u2019t", "ab\u00ADby",
                 "-3",
                 "e.g.",
                 "\u4e00.\u4e00.",
                  "a  b",
                  "a  \u200bb",
                  "a \u0308b",
                  "1\u0308b(a)-(b)",
                  };
        }
        
        // all the other items are supplied in UCD_TYPES
        
        /*static byte LB_L = LB_LIMIT + hL, LB_V = LB_LIMIT + hV, LB_T = LB_LIMIT + hT,
            LB_LV = LB_LIMIT + hLV, LB_LVT = LB_LIMIT + hLVT, LB_SUP = LB_LIMIT + hLIMIT,
            LB2_LIMIT = (byte)(LB_SUP + 1);
        */    

        /*
        private byte[] AsmusOrderToMyOrder = {
                    LB_OP, LB_CL, LB_QU, LB_GL, LB_NS, LB_EX, LB_SY, LB_IS, LB_PR, LB_PO,
                    LB_NU, LB_AL, LB_ID, LB_IN, LB_HY, LB_BA, LB_BB, LB_B2, LB_ZW, LB_CM,
                    // missing from Pair Table
                    LB_SP, LB_BK, LB_CR, LB_LF,
                    // resolved types below
                    LB_CB, LB_AI, LB_SA, LB_SG, LB_XX,
                    // 3 JAMO CLASSES, plus supplementary
                    LB_L, LB_V, LB_T, LB_LV, LB_LVT, LB_SUP
                };

        private byte[] MyOrderToAsmusOrder = new byte[AsmusOrderToMyOrder.length];
        {
            for (byte i = 0; i < AsmusOrderToMyOrder.length; ++i) {
                MyOrderToAsmusOrder[AsmusOrderToMyOrder[i]] = i;
            }
        */
            
        {
            //System.out.println("Adding Linebreak");
            for (int i = 0; i <= 0x10FFFF; ++i) {
                map.put(i, ucd.getLineBreak(i));
            }
            for (int i = 0; i < LB_LIMIT; ++i) {
                map.setLabel(i, ucd.getLineBreakID_fromIndex((byte)i, SHORT));
            }
            //System.out.println(map.getSetFromIndex(LB_CL));
            //System.out.println("Done adding Linebreak");
        }
        
        public int mapType(int input) {
            int old = input;
            switch (input) {
                case LB_BA: input = 16; break;
                case LB_BB: input = 17; break;
                case LB_B2: input = 18; break;
                case LB_ZW: input = 19; break;
                case LB_CM: input = 20; break;
                case LB_WJ: input = 21; break;
                
                case LB_SP: input = 22; break;
                case LB_BK: input = 23; break;
                case LB_NL: input = 24; break;
                case LB_CR: input = 25; break;
                case LB_LF: input = 26; break;
                
                case LB_CB: input = 27; break;
                case LB_SA: input = 28; break;
                case LB_AI: input = 29; break;
                case LB_SG: input = 30; break;
            }
            //if (old != input) System.out.println(old + " => " + input);
            return input;
        }


        public void sampleDescription(PrintWriter out) {
            out.println("# Samples:");
            out.println("# The test currently takes all pairs of linebreak types*,");
            out.println("# picks a sample for each type, and generates three strings: ");
            out.println("#\t- the pair alone");
            out.println("#\t- the pair alone with an imbeded space");
            out.println("#\t- the pair alone with embedded combining marks");
            out.println("# The sample for each type is simply the first code point (above NULL)");
            out.println("# with that property.");
            out.println("# * Note:");
            out.println("#\t- SG is omitted");
            out.println("#\t- 3 different Jamo characters and a supplementary character are added");
            out.println("#\t  The syllable types for the Jamo (L, V, T) are displayed in comments");
            out.println("#\t  instead of the linebreak property");
            out.println("#");
        }

        // stuff that subclasses need to override
        public int genTestItems(String before, String after, String[] results) {
            results[0] = before + after;
            results[1] = before + " " + after;
            results[2] = before + "\u0301\u0308" + after;
            return 3;
        }

        // stuff that subclasses need to override
        boolean skipType(int type) {
            return type == LB_AI || type == LB_SA || type == LB_SG || type == LB_XX
                || type == LB_CB || type == LB_CR || type == LB_BK || type == LB_LF
                || type == LB_NL || type == LB_SP;
        }

        // stuff that subclasses need to override
        public String getTypeID(int cp) {
            /*
            byte result = getType(cp);
            if (result == LB_SUP) return "SUP";
            if (result >= LB_LIMIT) return hNames[result - LB_LIMIT];
            */
            // return ucd.getLineBreakID_fromIndex(cp); // AsmusOrderToMyOrder[result]);
            return ucd.getLineBreakID(cp); // AsmusOrderToMyOrder[result]);
        }

        public String fullBreakSample() {
            return ")a";
        }

        // stuff that subclasses need to override
        public byte getType(int cp) {
            /*if (cp > 0xFFFF) return LB_SUP;
            byte result = getHangulType(cp);
            if (result != hNot) return (byte)(result + LB_LIMIT);
            */
            // return MyOrderToAsmusOrder[ucd.getLineBreak(cp)];
            return ucd.getLineBreak(cp);
        }

        public String getTableEntry(String before, String after, String[] ruleOut) {
            String t = "_"; // break
            boolean spaceBreak = isBreak(before + " " + after, before.length()+1);
            String spaceRule = getRule();

            boolean spaceBreak2 = isBreak(before + " " + after, before.length());
            String spaceRule2 = getRule();

            boolean normalBreak = isBreak(before + after, before.length());
            String normalRule = getRule();

            ruleOut[0] = normalRule;
            if (!normalBreak) {
                if (!spaceBreak && !spaceBreak2) {
                    t = "^"; // don't break, even with intervening spaces
                } else {
                    t = "%"; // don't break, but break with intervening spaces
                }
                if (!spaceRule2.equals(normalRule)) {
                    ruleOut[0] += " [" + spaceRule2 + "]";
                }
                if (!spaceRule.equals(normalRule) && !spaceRule.equals(spaceRule2)) {
                    ruleOut[0] += " {" + spaceRule + "}";
                }
            }
            return t;
        }
        
        public boolean highlightTableEntry(int x, int y, String s) {
            return false;
            /*
            try {
                return !oldLineBreak[x][y].equals(s);
            } catch (Exception e) {}
            return true;
            */
        }

/*
        String[][] oldLineBreak = {
{"^",	"^",	"^",	"^",	"^",	"^",	"^",	"^",	"^",	"^",	"^",	"^",	"^",	"^",	"^",	"^",	"^",	"^",	"^",	"%"},
{"_",	"^",	"%",	"%",	"^",	"^",	"^",	"^",	" ",	"%",	"_",	"_",	"_",	"_",	"%",	"%",	"_",	"_",	"^",	"%"},
{"^",	"^",	"%",	"%",	"%",	"^",	"^",	"^",	"%",	"%",	"%",	"%",	"%",	"%",	"%",	"%",	"%",	"%",	"^",	"%"},
{"%",	"^",	"%",	"%",	"%",	"^",	"^",	"^",	"%",	"%",	"%",	"%",	"%",	"%",	"%",	"%",	"%",	"%",	"^",	"%"},
{"_",	"^",	"%",	"%",	"%",	"^",	"^",	"^",	"_",	"_",	"_",	"_",	"_",	"_",	"%",	"%",	"_",	"_",	"^",	"%"},
{"_",	"^",	"%",	"%",	"%",	"^",	"^",	"^",	"_",	"_",	"_",	"_",	"_",	"_",	"%",	"%",	"_",	"_",	"^",	"%"},
{"_",	"^",	"%",	"%",	"%",	"^",	"^",	"^",	"_",	"_",	"%",	"_",	"_",	"_",	"%",	"%",	"_",	"_",	"^",	"%"},
{"_",	"^",	"%",	"%",	"%",	"^",	"^",	"^",	"_",	"_",	"%",	"_",	"_",	"_",	"%",	"%",	"_",	"_",	"^",	"%"},
{"%",	"^",	"%",	"%",	"%",	"^",	"^",	"^",	"_",	"_",	"%",	"%",	"%",	"_",	"%",	"%",	"_",	"_",	"^",	"%"},
{"_",	"^",	"%",	"%",	"%",	"^",	"^",	"^",	"_",	"_",	"_",	"_",	"_",	"_",	"%",	"%",	"_",	"_",	"^",	"%"},
{"_",	"^",	"%",	"%",	"%",	"^",	"^",	"^",	"_",	"%",	"%",	"%",	"_",	"%",	"%",	"%",	"_",	"_",	"^",	"%"},
{"_",	"^",	"%",	"%",	"%",	"^",	"^",	"^",	"_",	"_",	"%",	"%",	"_",	"%",	"%",	"%",	"_",	"_",	"^",	"%"},
{"_",	"^",	"%",	"%",	"%",	"^",	"^",	"^",	"_",	"%",	"_",	"_",	"_",	"%",	"%",	"%",	"_",	"_",	"^",	"%"},
{"_",	"^",	"%",	"%",	"%",	"^",	"^",	"^",	"_",	"_",	"_",	"_",	"_",	"%",	"%",	"%",	"_",	"_",	"^",	"%"},
{"_",	"^",	"%",	"%",	"%",	"^",	"^",	"^",	"_",	"_",	"_",	"_",	"_",	"_",	"%",	"%",	"_",	"_",	"^",	"%"},
{"_",	"^",	"%",	"%",	"%",	"^",	"^",	"^",	"_",	"_",	"_",	"_",	"_",	"_",	"%",	"%",	"_",	"_",	"^",	"%"},
{"%",	"^",	"%",	"%",	"%",	"^",	"^",	"^",	"%",	"%",	"%",	"%",	"%",	"%",	"%",	"%",	"%",	"%",	"^",	"%"},
{"_",	"^",	"%",	"%",	"%",	"^",	"^",	"^",	"_",	"_",	"_",	"_",	"_",	"_",	"%",	"%",	"_",	"^",	"^",	"%"},
{"_",	"_",	"_",	"_",	"_",	"_",	"_",	"_",	"_",	"_",	"_",	"_",	"_",	"_",	"_",	"_",	"_",	"_",	"^",	"%"},
{"_",	"^",	"%",	"%",	"%",	"^",	"^",	"^",	"_",	"_",	"%",	"%",	"_",	"%",	"%",	"%",	"_",	"_",	"^",	"%"}
        };
*/

        public byte getResolvedType (int cp) {
            // LB 1  Assign a line break category to each character of the input.
            // Resolve AI, CB, SA, SG, XX into other line break classes depending on criteria outside this algorithm.
            byte result = getType(cp);
            switch (result) {
                case LB_AI: result = LB_AI; break;
                // case LB_CB: result = LB_ID; break;
                case LB_SA: result = LB_AL; break;
                // case LB_SG: result = LB_XX; break; Surrogates; will never occur
                case LB_XX: result = LB_AL; break;
            }
            /*
            if (recommended) {
                if (getHangulType(cp) != hNot) {
                        result = LB_ID;
                }
            }
            */
            
            return result;
        }
        
        public byte getSampleType (int cp) {
            if (ucd.getHangulSyllableType(cp) != NA) return LB_XX;
            return getType(cp);
        }


        // find out whether there is a break at offset
        // WARNING: as a side effect, sets "rule"

        public boolean isBreak(String source, int offset) {

            // LB 1  Assign a line break category to each character of the input.
            // Resolve AI, CB, SA, SG, XX into other line break classes depending on criteria outside this algorithm.
            // this is taken care of in the getResolvedType function

            // LB 2a  Never break at the start of text

            setRule("2a: × sot");
            if (offset <= 0) return false;

            // LB 2b  Always break at the end of text

            setRule("2b: ! eot");
            if (offset >= source.length()) return true;


            // UTF-16: never break in the middle of a code point
            
            // now get the base character before and after, and their types

            getGraphemeBases(breaker, source, offset, -1, context);

            byte before = context.tBefore;
            byte after = context.tAfter;
            byte before2 = context.tBefore2;
            byte after2 = context.tAfter2;
            
            
            //if (!onCodepointBoundary(source, offset)) return false;


            // now get the character before and after, and their types


            //int cpBefore = UTF16.charAt(source, offset-1);
            //int cpAfter = UTF16.charAt(source, offset);

            //byte before = getResolvedType(cpBefore);
            //byte after = getResolvedType(cpAfter);


            setRule("3a: CR × LF ; ( BK | CR | LF | NL ) !");
            
            // Always break after hard line breaks (but never between CR and LF).
            // CR ^ LF
            if (before == LB_CR && after == LB_LF) return false;
            if (before == LB_BK || before == LB_LF || before == LB_CR) return true;

            //LB 3b  Don’t break before hard line breaks.
            setRule("3b: × ( BK | CR | LF )");
            if (after == LB_BK || after == LB_LF || after == LB_CR) return false;

            // LB 4  Don’t break before spaces or zero-width space.
            setRule("4: × ( SP | ZW )");
            if (after == LB_SP || after == LB_ZW) return false;

            // LB 5 Break after zero-width space.
            setRule("5: ZW ÷");
            if (before == LB_ZW) return true;

            // LB 6  Don’t break graphemes (before combining marks, around virama or on sequences of conjoining Jamos.
            setRule("6: DGC -> FC");
            if (!grapheme.isBreak( source,  offset)) return false;
            
            /*
            if (before == LB_L && (after == LB_L || after == LB_V || after == LB_LV || after == LB_LVT)) return false;
            if ((before == LB_LV || before == LB_V) && (after == LB_V || after == LB_T)) return false;
            if ((before == LB_LVT || before == LB_T) && (after == LB_T)) return false;
            */
            
            byte backBase = -1;
            boolean setBase = false;
            if (before == LB_CM) {
                setBase = true;
                int backOffset = findLastNon(source, offset, LB_CM);
                if (backOffset >= 0) {
                    backBase = getResolvedType(UTF16.charAt(source, backOffset));
                }
            }
            
            
            // LB 7  In all of the following rules, if a space is the base character for a combining mark,
            // the space is changed to type ID. In other words, break before SP CM* in the same cases as
            // one would break before an ID.
            setRule("7: SP CM* -> ID");
            if (setBase && backBase == LB_SP) before = LB_ID;
            if (after == LB_SP && after2 == LB_CM) after = LB_ID;

            setRule("7a: X CM* -> X");
            if (after == LB_CM) return false;
            if (setBase && backBase != -1) before = LB_ID;

            setRule("7b: CM -> AL");
            if (setBase && backBase == -1) before = LB_AL;

            
            // LB 8  Don’t break before ‘]’ or ‘!’ or ‘;’ or ‘/’,  even after spaces.
            // × CL, × EX, × IS, × SY
            setRule("8: × ( CL | EX | IS | SY )");
            if (after == LB_CL || after == LB_EX || after == LB_SY | after == LB_IS) return false;


            // find the last non-space character; we will need it
            byte lastNonSpace = before;
            if (lastNonSpace == LB_SP) {
                int backOffset = findLastNon(source, offset, LB_SP);
                if (backOffset >= 0) {
                    lastNonSpace = getResolvedType(UTF16.charAt(source, backOffset));
                }
            }

            // LB 9  Don’t break after ‘[’, even after spaces.
            // OP SP* ×
            setRule("9: OP SP* ×");
            if (lastNonSpace == LB_OP) return false;

            // LB 10  Don’t break within ‘�?[’, , even with intervening spaces.
            // QU SP* × OP
            setRule("10: QU SP* × OP");
            if (lastNonSpace == LB_QU && after == LB_OP) return false;

            // LB 11  Don’t break within ‘]h’, even with intervening spaces.
            // CL SP* × NS
            setRule("11: CL SP* × NS");
            if (lastNonSpace == LB_CL && after == LB_NS) return false;

            // LB 11a  Don’t break within ‘——’, even with intervening spaces.
            // B2 × B2
            setRule("11a: B2 × B2");
            if (lastNonSpace == LB_B2 && after == LB_B2) return false;


            // LB 13  Don’t break before or after NBSP or WORD JOINER
            // × GL
            // GL ×

            setRule("11b: × WJ ; WJ ×");
            if (after == LB_WJ || before == LB_WJ) return false;

            // [Note: by this time, all of the "X" in the table are accounted for. We can safely break after spaces.]

            // LB 12  Break after spaces
            setRule("12: SP ÷");
            if (before == LB_SP) return true;

            // LB 13  Don’t break before or after NBSP or WORD JOINER
            setRule("13: × GL ; GL ×");
            if (after == LB_GL || before == LB_GL) return false;

            // LB 14  Don’t break before or after ‘�?’
            setRule("14: × QU ; QU ×");
            if (before == LB_QU || after == LB_QU) return false;

            // LB 14a  Break before and after CB
            setRule("14a: ÷ CB ; CB ÷");
            if (before == LB_CB || after == LB_CB) return true;

            // LB 15  Don’t break before hyphen-minus, other hyphens, fixed-width spaces,
            // small kana and other non- starters,  or after acute accents:

            setRule("15: × ( BA | HY | NS ) ; BB ×");
            if (after == LB_NS) return false;
            if (after == LB_HY) return false;
            if (after == LB_BA) return false;
            if (before == LB_BB) return false;


            //setRule("15a: HY × NU"); // NEW
            //if (before == LB_HY && after == LB_NU) return false;

            // LB 16  Don’t break between two ellipses, or between letters or numbers and ellipsis:
            // Examples: ’9...’, ‘a...’, ‘H...’
            setRule("16: ( AL | ID | IN | NU ) × IN");
            if ((before == LB_NU || before == LB_AL || before == LB_ID) && after == LB_IN) return false;
            if (before == LB_IN && after == LB_IN) return false;

            // Don't break alphanumerics.
            // LB 17  Don’t break within ‘a9’, ‘3a’, or ‘H%’
            // Numbers are of the form PR ? ( OP | HY ) ? NU (NU | IS) * CL ?  PO ?
            // Examples:   $(12.35)    2,1234    (12)¢    12.54¢
            // This is approximated with the following rules. (Some cases already handled above,
            // like ‘9,’, ‘[9’.)
            setRule("17: ID × PO ; AL × NU; NU × AL");
            if (before == LB_ID && after == LB_PO) return false;
            if (before == LB_AL && after == LB_NU) return false;
            if (before == LB_NU && after == LB_AL) return false;

            // LB 18  Don’t break between the following pairs of classes.
            // CL × PO
            // HY × NU
            // IS × NU
            // NU × NU
            // NU × PO
            // PR × AL
            // PR × HY
            // PR × ID
            // PR × NU
            // PR × OP
            // SY × NU
            // Example pairs: ‘$9’, ‘$[’, ‘$-‘, ‘-9’, ‘/9’, ‘99’, ‘,9’,  ‘9%’ ‘]%’

            setRule("18: CL × PO ; NU × PO ; ( IS | NU | HY | PR | SY ) × NU ; PR × ( AL | HY | ID | OP )");
            if (before == LB_CL && after == LB_PO) return false;
            if (before == LB_IS && after == LB_NU) return false;
            if (before == LB_NU && after == LB_NU) return false;
            if (before == LB_NU && after == LB_PO) return false;
            
            if (before == LB_HY && after == LB_NU) return false;

            if (before == LB_PR && after == LB_AL) return false;
            if (before == LB_PR && after == LB_HY) return false;
            if (before == LB_PR && after == LB_ID) return false;
            if (before == LB_PR && after == LB_NU) return false;
            if (before == LB_PR && after == LB_OP) return false;

            if (before == LB_SY && after == LB_NU) return false;

            // LB 15b  Break after hyphen-minus, and before acute accents:
            setRule("18b: HY ÷ ; ÷ BB");
            if (before == LB_HY) return true;
            if (after == LB_BB) return true;

            // LB 19  Don’t break between alphabetics (“at�?)
            // AL × AL

            setRule("19: AL × AL");
            if (before == LB_AL && after == LB_AL) return false;

            // LB 20  Break everywhere else
            // ALL ÷
            // ÷ ALL

            if (ucd.getCompositeVersion() > 0x040000) {
                setRule("19b: IS × AL");
                if (before == LB_IS && after == LB_AL) return false;
            }

            // LB 20  Break everywhere else
            // ALL ÷
            // ÷ ALL

            setRule("20: ALL ÷ ; ÷ ALL");
            return true;
        }
    }

    //==============================================

    static class XGenerateSentenceBreakTest extends GenerateBreakTest {
        
        GenerateGraphemeBreakTest grapheme;
        MyBreakIterator breaker;
        
        XGenerateSentenceBreakTest(UCD ucd) {
            super(ucd);
            grapheme = new GenerateGraphemeBreakTest(ucd);
            breaker = new MyBreakIterator(grapheme);

            fileName = "Sentence";
            extraSamples = new String[] {
            };
            
            extraSingleSamples = new String[] {
                "(\"Go.\") (He did.)", 
                "(\u201CGo?\u201D) (He did.)", 
                "U.S.A\u0300. is", 
                "U.S.A\u0300? He", 
                "U.S.A\u0300.", 
                "3.4", 
                "c.d",
                "etc.)\u2019 \u2018(the",
                "etc.)\u2019 \u2018(The",
                "the resp. leaders are",
                "\u5B57.\u5B57",
                "etc.\u5B83",
                "etc.\u3002",
                "\u5B57\u3002\u5B83",
            };
            String[] temp = new String [extraSingleSamples.length * 2];
            System.arraycopy(extraSingleSamples, 0, temp, 0, extraSingleSamples.length);
            for (int i = 0; i < extraSingleSamples.length; ++i) {
                temp[i+extraSingleSamples.length] = insertEverywhere(extraSingleSamples[i], "\u2060", grapheme);
            }
            extraSingleSamples = temp;

        }

        Object foo = prop = unicodePropertySource.getProperty("Sentence_Break");

        final int
            Sep =    addToMap("Sep"),
            Format =    addToMap("Format"),
            Sp = addToMap("Sp"),
            Lower = addToMap("Lower"),
            Upper = addToMap("Upper"),
            OLetter = addToMap("OLetter"),
            Numeric =     addToMap("Numeric"),
            ATerm =     addToMap("ATerm"),
            STerm =    addToMap("STerm"),
            Close =     addToMap("Close"),
            Other = addToMapLast("Other");            
                
        // stuff that subclasses need to override
        public String getTypeID(int cp) {
            return map.getLabel(cp);
        }

        public String fullBreakSample() {
            return "!a";
        }

       // stuff that subclasses need to override
        public byte getType(int cp) {
            return (byte) map.getIndex(cp);
        }

        /*LB_XX = 0, LB_OP = 1, LB_CL = 2, LB_QU = 3, LB_GL = 4, LB_NS = 5, LB_EX = 6, LB_SY = 7,
        LB_IS = 8, LB_PR = 9, LB_PO = 10, LB_NU = 11, LB_AL = 12, LB_ID = 13, LB_IN = 14, LB_HY = 15,
        LB_CM = 16, LB_BB = 17, LB_BA = 18, LB_SP = 19, LB_BK = 20, LB_CR = 21, LB_LF = 22, LB_CB = 23,
        LB_SA = 24, LB_AI = 25, LB_B2 = 26, LB_SG = 27, LB_ZW = 28,
        LB_NL = 29,
        LB_WJ = 30,
        */
        /*
        static final byte Format = 0, Sep = 1, Sp = 2, OLetter = 3, Lower = 4, Upper = 5,
            Numeric = 6, Close = 7, ATerm = 8, Term = 9, Other = 10,
            LIMIT = Other + 1;

        static final String[] Names = {"Format", "Sep", "Sp", "OLetter", "Lower", "Upper", "Numeric",
            "Close", "ATerm", "Term", "Other" };


        static UnicodeSet sepSet = new UnicodeSet("[\\u000a\\u000d\\u0085\\u2029\\u2028]");
        static UnicodeSet atermSet = new UnicodeSet("[\\u002E]");
        static UnicodeSet termSet = new UnicodeSet(
            "[\\u0021\\u003F\\u0589\\u061f\\u06d4\\u0700-\\u0702\\u0934"
            + "\\u1362\\u1367\\u1368\\u104A\\u104B\\u166E"
            + "\\u1803\\u1809\\u203c\\u203d"
            + "\\u2048\\u2049\\u3002\\ufe52\\ufe57\\uff01\\uff0e\\uff1f\\uff61]");
        
        static UnicodeProperty lowercaseProp = UnifiedBinaryProperty.make(DERIVED | PropLowercase);
        static UnicodeProperty uppercaseProp = UnifiedBinaryProperty.make(DERIVED | PropUppercase);
        
        UnicodeSet linebreakNS = UnifiedBinaryProperty.make(LINE_BREAK | LB_NU).getSet();
        */
        
        /*
        // stuff that subclasses need to override
        public String getTypeID(int cp) {
            byte type = getType(cp);
            return Names[type];
        }

        // stuff that subclasses need to override
        public byte getType(int cp) {
            byte cat = ucd.getCategory(cp);
            
            if (cat == Cf) return Format;
            if (sepSet.contains(cp)) return Sep;
            if (ucd.getBinaryProperty(cp, White_space)) return Sp;
            if (linebreakNS.contains(cp)) return Numeric;
            if (lowercaseProp.hasValue(cp)) return Lower;
            if (uppercaseProp.hasValue(cp) || cat == Lt) return Upper;
            if (alphabeticSet.contains(cp)) return OLetter;
            if (atermSet.contains(cp)) return ATerm;
            if (termSet.contains(cp)) return Term;
            if (cat == Po || cat == Pe
                || ucd.getLineBreak(cp) == LB_QU) return Close;
            return Other;
        }
        */
        
        public int genTestItems(String before, String after, String[] results) {
            results[0] = before + after;
            /*
            results[1] = 'a' + before + "\u0301\u0308" + after + "\u0301\u0308" + 'a';
            results[2] = 'a' + before + "\u0301\u0308" + samples[MidLetter] + after + "\u0301\u0308" + 'a';
            results[3] = 'a' + before + "\u0301\u0308" + samples[MidNum] + after + "\u0301\u0308" + 'a';
            */
            return 1;
        }

        static Context context = new Context();
        
        public boolean isBreak(String source, int offset) {
    
            // Break at the start and end of text.
            setRule("1: sot ÷");
            if (offset < 0 || offset > source.length()) return false;
  
            if (offset == 0) return true;

            setRule("2: ÷ eot");
            if (offset == source.length()) return true;

            setRule("3: Sep ÷");
            byte beforeChar = getResolvedType(source.charAt(offset-1));
            if (beforeChar == Sep) return true;
            
            // Treat a grapheme cluster as if it were a single character:
            // the first base character, if there is one; otherwise the first character.

            setRule("4: GC -> FC");
            if (!grapheme.isBreak( source,  offset)) return false;
            
            // Ignore interior Format characters. That is, ignore Format characters in all subsequent rules.
            setRule("5: X Format* -> X");
            byte afterChar = getResolvedType(source.charAt(offset));
            if (afterChar == Format) return false;

            getGraphemeBases(breaker, source, offset, Format, context);
            byte before = context.tBefore;
            byte after = context.tAfter;
            byte before2 = context.tBefore2;
            byte after2 = context.tAfter2;
            
            // HACK COPY for rule collection!
            if (collectingRules) {
                setRule("6: ATerm × ( Numeric | Lower )");
                setRule("7: Upper ATerm × Upper");
                setRule("8: ATerm Close* Sp* × ( ¬(OLetter | Upper | Lower) )* Lower");
                setRule("9: ( Term | ATerm ) Close* × ( Close | Sp | Sep )");
                setRule("10: ( Term | ATerm ) Close* Sp × ( Sp | Sep )");
                setRule("11: ( Term | ATerm ) Close* Sp* ÷");
                setRule("12: Any × Any");
                collectingRules = false;
            }
            
            // Do not break after ambiguous terminators like period, if immediately followed by a number or lowercase letter, is between uppercase letters, or if the first following letter (optionally after certain punctuation) is lowercase. For example, a period may be an abbreviation or numeric period, and not mark the end of a sentence.
            
            if (before == ATerm) {
                setRule("6: ATerm × ( Numeric | Lower )");
                if (after == Lower || after == Numeric) return false;
                setRule("7: Upper ATerm × Upper");
                if (DEBUG_GRAPHEMES) System.out.println(context + ", " + Upper);
                if (before2 == Upper && after == Upper) return false;
            }
            
            // The following cases are all handled together.
            
            // First we loop backwards, checking for the different types.
            
            MyBreakIterator graphemeIterator = new MyBreakIterator(grapheme);
            graphemeIterator.set(source, offset);
            
            int state = 0;
            int lookAfter = -1;
            int cp;
            byte t;
            boolean gotSpace = false;
            boolean gotClose = false;
            
            behindLoop:
            while (true) {
                cp = graphemeIterator.previousBase();
                if (cp == -1) break;
                t = getResolvedType(cp);
                if (SHOW_TYPE) System.out.println(ucd.getCodeAndName(cp) + ", " + getTypeID(cp));
                
                if (t == Format) continue;  // ignore all formats!
                
                switch (state) {
                    case 0:
                        if (t == Sp) {
                            // loop as long as we have Space
                            gotSpace = true;
                            continue behindLoop;
                        } else if (t == Close) {
                            gotClose = true;
                            state = 1;    // go to close loop
                            continue behindLoop;
                        }
                        break;
                    case 1:
                        if (t == Close) {
                            // loop as long as we have Close
                            continue behindLoop;
                        }
                        break;
                }
                if (t == ATerm) {
                    lookAfter = ATerm;
                } else if (t == STerm) {
                    lookAfter = STerm;
                }
                break;
            }
            
            // if we didn't find ATerm or Term, bail
            
            if (lookAfter == -1) {
                // Otherwise, do not break
                // Any × Any (11)
                setRule("12: Any × Any");
                return false;
            }
                
            // ATerm Close* Sp*×(¬( OLetter))* Lower(8)
            
            // Break after sentence terminators, but include closing punctuation, trailing spaces, and (optionally) a paragraph separator.
            // ( Term | ATerm ) Close*×( Close | Sp | Sep )(9)
            // ( Term | ATerm ) Close* Sp×( Sp | Sep )(10)
            // ( Term | ATerm ) Close* Sp*÷(11)

                        
            // We DID find one. Loop to see if the right side is ok.

            graphemeIterator.set(source, offset);
            boolean isFirst = true;
            while (true) {
                cp = graphemeIterator.nextBase();
                if (cp == -1) break;
                t = getResolvedType(cp);
                if (SHOW_TYPE) System.out.println(ucd.getCodeAndName(cp) + ", " + getTypeID(cp));
                    
                if (t == Format) continue;  // skip format characters!
                    
                if (isFirst) {
                    isFirst = false;
                    if (lookAfter == ATerm && t == Upper) {
                        setRule("8: ATerm Close* Sp* × ( ¬(OLetter | Upper | Lower) )* Lower");
                        return false;
                    }
                    if (gotSpace) {
                        if (t == Sp || t == Sep) {
                            setRule("10: ( Term | ATerm ) Close* Sp × ( Sp | Sep )");
                            return false;
                        }
                    } else if (t == Close || t == Sp || t == Sep) {
                        setRule("9: ( Term | ATerm ) Close* × ( Close | Sp | Sep )");
                        return false;
                    }
                    if (lookAfter == STerm) break;
                }
                    
                // at this point, we have an ATerm. All other conditions are ok, but we need to verify 6
                if (t != OLetter && t != Upper && t != Lower) continue;
                if (t == Lower) {
                    setRule("8: ATerm Close* Sp* × ( ¬(OLetter | Upper | Lower) )* Lower");
                    return false;
                }
                break;
            }
            setRule("11: ( Term | ATerm ) Close* Sp* ÷");
            return true;
        }
    }
    
    static final boolean DEBUG_GRAPHEMES = false;
    
    static class MyBreakIterator {
        int offset = 0;
        String string = "";
        GenerateBreakTest breaker;
        boolean recommended = true;
        
        MyBreakIterator(GenerateBreakTest breaker) {
            this.breaker = breaker; //  = new GenerateGraphemeBreakTest()
        }
        public MyBreakIterator set(String source, int offset) {
            //if (DEBUG_GRAPHEMES) System.out.println(Utility.hex(string) + "; " + offset);
            string = source;
            this.offset = offset;
            return this;
        }
        
        public int nextBase() {
            if (offset >= string.length()) return -1;
            int result = UTF16.charAt(string, offset);
            for (++offset; offset < string.length(); ++offset) {
                if (breaker.isBreak(string, offset)) break;
            }
            //if (DEBUG_GRAPHEMES) System.out.println(Utility.hex(result));
            return result;
        }
        
        public int previousBase() {
            if (offset <= 0) return -1;
            for (--offset; offset >= 0; --offset) {
                if (breaker.isBreak(string, offset)) break;
            }
            int result = UTF16.charAt(string, offset);
            //if (DEBUG_GRAPHEMES) System.out.println(Utility.hex(result));
            return result;
        }
    }
    /*
     * 
     *         if (false) {
            
            PrintWriter log = Utility.openPrintWriter("Diff.txt", Utility.UTF8_WINDOWS);
            UnicodeSet Term = new UnicodeSet(
                "[\\u0021\\u003F\\u0589\\u061F\\u06D4\\u0700\\u0701\\u0702\\u0964\\u1362\\u1367"
                + "\\u1368\\u104A\\u104B\\u166E\\u1803\\u1809\\u203C\\u203D\\u2047\\u2048\\u2049"
                + "\\u3002\\uFE52\\uFE57\\uFF01\\uFF0E\\uFF1F\\uFF61]");
            UnicodeSet terminal_punctuation = getSet(BINARY_PROPERTIES, Terminal_Punctuation);
            UnicodeMap names = new UnicodeMap();
            names.add("Pd", getSet(CATEGORY, Pd));
            names.add("Ps", getSet(CATEGORY, Ps));
            names.add("Pe", getSet(CATEGORY, Pe));
            names.add("Pc", getSet(CATEGORY, Pc));
            names.add("Po", getSet(CATEGORY, Po));
            names.add("Pi", getSet(CATEGORY, Pi));
            names.add("Pf", getSet(CATEGORY, Pf));
            
            Utility.showSetDifferences(log, "Term", Term, "Terminal_Punctuation", terminal_punctuation, true, true, names, ucd);
            Utility.showSetDifferences(log, "Po", getSet(CATEGORY, Po), "Terminal_Punctuation", terminal_punctuation, true, true, names, ucd);
            log.close();
            
            if (true) return;
                
            UnicodeSet whitespace = getSet(BINARY_PROPERTIES, White_space);
            UnicodeSet space = getSet(CATEGORY, Zs).addAll(getSet(CATEGORY, Zp)).addAll(getSet(CATEGORY, Zl));
            Utility.showSetDifferences("White_Space", whitespace, "Z", space, true, ucd);
            
            UnicodeSet isSpace = new UnicodeSet();
            UnicodeSet isSpaceChar = new UnicodeSet();
            UnicodeSet isWhitespace = new UnicodeSet();
            for (int i = 0; i <= 0xFFFF; ++i) {
                if (Character.isSpace((char)i)) isSpace.add(i);
                if (Character.isSpaceChar((char)i)) isSpaceChar.add(i);
                if (Character.isWhitespace((char)i)) isWhitespace.add(i);
            }
            Utility.showSetDifferences("White_Space", whitespace, "isSpace", isSpace, true, ucd);
            Utility.showSetDifferences("White_Space", whitespace, "isSpaceChar", isSpaceChar, true, ucd);
            Utility.showSetDifferences("White_Space", whitespace, "isWhitespace", isWhitespace, true, ucd);
            return;
        }
        
        if (DEBUG) {
            checkDecomps();

            Utility.showSetNames("", new UnicodeSet("[\u034F\u00AD\u1806[:DI:]-[:Cs:]-[:Cn:]]"), true, ucd);

            System.out.println("*** Extend - Cf");

            generateTerminalClosure();

            GenerateWordBreakTest gwb = new GenerateWordBreakTest();
            PrintWriter systemPrintWriter = new PrintWriter(System.out);
            gwb.printLine(systemPrintWriter, "n\u0308't", true, true, false);
            systemPrintWriter.flush();
            //showSet("sepSet", GenerateSentenceBreakTest.sepSet);
            //showSet("atermSet", GenerateSentenceBreakTest.atermSet);
            //showSet("termSet", GenerateSentenceBreakTest.termSet);
        }
        
        if (true) {
            GenerateBreakTest foo = new GenerateLineBreakTest();
            //foo.isBreak("(\"Go.\") (He did)", 5, true);
            foo.isBreak("\u4e00\u4300", 1, true);
            /*
            GenerateSentenceBreakTest foo = new GenerateSentenceBreakTest();
            //foo.isBreak("(\"Go.\") (He did)", 5, true);
            foo.isBreak("3.4", 2, true);
            * /
        }

        new GenerateGraphemeBreakTest().run();
        new GenerateWordBreakTest().run();
        new GenerateLineBreakTest().run();
        new GenerateSentenceBreakTest().run();
        
        //if (true) return; // cut short for now
        
    }

     */
}