/*
 *******************************************************************************
 * Copyright (C) 2009-2011, International Business Machines Corporation and    *
 * others. All Rights Reserved.                                                *
 *******************************************************************************
 */
package com.ibm.icu.dev.test.text;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.Reader;
import java.text.ParseException;
import java.util.LinkedHashSet;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.ibm.icu.dev.test.TestFmwk;
import com.ibm.icu.dev.test.TestUtil;
import com.ibm.icu.dev.test.TestUtil.JavaVendor;
import com.ibm.icu.impl.Utility;
import com.ibm.icu.text.Normalizer2;
import com.ibm.icu.text.SpoofChecker;
import com.ibm.icu.text.UnicodeSet;
import com.ibm.icu.util.ULocale;

public class SpoofCheckerTest extends TestFmwk {

    public static void main(String[] args) throws Exception {
        new SpoofCheckerTest().run(args);
    }

    /*
     * Identifiers for verifying that spoof checking is minimally alive and working.
     */
    char[] goodLatinChars = { (char) 0x75, (char) 0x7a };
    String goodLatin = new String(goodLatinChars); /* "uz", all ASCII */
    /* (not confusable) */
    char[] scMixedChars = { (char) 0x73, (char) 0x0441 };
    String scMixed = new String(scMixedChars); /* "sc", with Cyrillic 'c' */
    /* (mixed script, confusable */

    String scLatin = "sc";   /* "sc", plain ascii. */
    String goodCyrl = "\u0438\u043B";    // "Cyrillic small letter i and el"  Plain lower case Cyrillic letters, no latin confusables 
    String goodGreek = "\u03c0\u03c6";   // "Greek small letter pi and phi"  Plain lower case Greek letters

    // Various 1 l I look-alikes
    String lll_Latin_a = "lI1";   // small letter l, cap I, digit 1, all ASCII
    //  "\uFF29\u217C\u0196"  Full-width I, Small Roman Numeral fifty, Latin Cap Letter IOTA 
    String lll_Latin_b = "\uff29\u217c\u0196";
    String lll_Cyrl = "\u0406\u04C0\u0031";  // "\u0406\u04C01"
    /* The skeleton transform for all of the 'lll' lookalikes is ascii lower case letter l. */
    String lll_Skel = "lll";

    String han_Hiragana = "\u3086\u308A \u77F3\u7530";  // Hiragana, space, Han


    /*
     * Test basic constructor.
     */
    public void TestUSpoof() {
        SpoofChecker sc = new SpoofChecker.Builder().build();
        if (sc == null) {
            errln("FAIL: null SpoofChecker");
        }
    }

    /*
     * Test build from source rules.
     */
    public void TestOpenFromSourceRules() {
        if (TestUtil.getJavaVendor() == JavaVendor.IBM && TestUtil.getJavaVersion() == 5) {
            // Note: IBM Java 5 has a bug reading a large UTF-8 text contents
            logln("Skip this test case because of the IBM Java 5 bug");
            return;
        }
        String fileName;
        Reader confusables;
        Reader confusablesWholeScript;

        try {
            fileName = "unicode/confusables.txt";
            confusables = TestUtil.getDataReader(fileName, "UTF-8");
            fileName = "unicode/confusablesWholeScript.txt";
            confusablesWholeScript = TestUtil.getDataReader(fileName, "UTF-8");

            SpoofChecker rsc = new SpoofChecker.Builder().setData(confusables, confusablesWholeScript).build();
            if (rsc == null) {
                errln("FAIL: null SpoofChecker");
                return;
            }            
            // Check that newly built-from-rules SpoofChecker is able to function.
            checkSkeleton(rsc, "TestOpenFromSourceRules");
        } catch (java.io.IOException e) {
            errln(e.toString());
        } catch (ParseException e) {
            errln(e.toString());
        }
    }

    /*
     * Set & Get Check Flags
     */
    public void TestGetSetChecks1() {
        SpoofChecker sc = new SpoofChecker.Builder().setChecks(SpoofChecker.ALL_CHECKS).build();
        int t;
        t = sc.getChecks();
        assertEquals("", SpoofChecker.ALL_CHECKS, t);

        sc = new SpoofChecker.Builder().setChecks(0).build();
        t = sc.getChecks();
        assertEquals("", 0, t);

        int checks = SpoofChecker.WHOLE_SCRIPT_CONFUSABLE | SpoofChecker.MIXED_SCRIPT_CONFUSABLE
                | SpoofChecker.ANY_CASE;
        sc = new SpoofChecker.Builder().setChecks(checks).build();
        t = sc.getChecks();
        assertEquals("", checks, t);
    }

    /*
     * get & setAllowedChars
     */
    public void TestGetSetAllowedChars() {
        SpoofChecker sc = new SpoofChecker.Builder().build();
        UnicodeSet us;
        UnicodeSet uset;

        uset = sc.getAllowedChars();
        assertTrue("", uset.isFrozen());
        us = new UnicodeSet((int) 0x41, (int) 0x5A); /* [A-Z] */
        sc = new SpoofChecker.Builder().setAllowedChars(us).build();
        assertEquals("", us, sc.getAllowedChars());
    }

    /*
     * get & set Checks
     */
    public void TestGetSetChecks() {
        SpoofChecker sc = new SpoofChecker.Builder().build();
        int checks;
        int checks2;
        boolean checkResults;

        checks = sc.getChecks();
        assertEquals("", SpoofChecker.ALL_CHECKS, checks);

        checks &= ~(SpoofChecker.SINGLE_SCRIPT | SpoofChecker.MIXED_SCRIPT_CONFUSABLE);
        sc = new SpoofChecker.Builder().setChecks(checks).build();
        checks2 = sc.getChecks();
        assertEquals("", checks, checks2);

        /*
         * The checks that were disabled just above are the same ones that the "scMixed" test fails. So with those tests
         * gone checking that Identifier should now succeed
         */
        checkResults = sc.failsChecks(scMixed);
        assertFalse("", checkResults);
    }

    /*
     * AllowedLocales
     */
    public void TestAllowedLocales() {
        SpoofChecker sc = new SpoofChecker.Builder().build();
        Set<ULocale> allowedLocales = new LinkedHashSet<ULocale>();
        boolean checkResults;

        /* Default allowed locales list should be empty */
        allowedLocales = sc.getAllowedLocales();
        assertTrue("", allowedLocales.isEmpty());

        /* Allow en and ru, which should enable Latin and Cyrillic only to pass */
        ULocale enloc = new ULocale("en");
        ULocale ruloc = new ULocale("ru_RU");
        allowedLocales.add(enloc);
        allowedLocales.add(ruloc);
        sc = new SpoofChecker.Builder().setAllowedLocales(allowedLocales).build();
        allowedLocales = sc.getAllowedLocales();
        assertTrue("", allowedLocales.contains(enloc));
        assertTrue("", allowedLocales.contains(ruloc));

        /*
         * Limit checks to SpoofChecker.CHAR_LIMIT. Some of the test data has whole script confusables also, which we
         * don't want to see in this test.
         */
        sc = new SpoofChecker.Builder().setChecks(SpoofChecker.CHAR_LIMIT).setAllowedLocales(allowedLocales).build();
        
        SpoofChecker.CheckResult result = new SpoofChecker.CheckResult();
        checkResults = sc.failsChecks(goodLatin);
        assertFalse("", checkResults);

        checkResults = sc.failsChecks(goodGreek, result);
        assertEquals("", SpoofChecker.CHAR_LIMIT, result.checks);

        checkResults = sc.failsChecks(goodCyrl);
        assertFalse("", checkResults);

        /* Reset with an empty locale list, which should allow all characters to pass */
        allowedLocales = new LinkedHashSet<ULocale>();
        sc = new SpoofChecker.Builder().setChecks(SpoofChecker.CHAR_LIMIT).setAllowedLocales(allowedLocales).build();

        checkResults = sc.failsChecks(goodGreek);
        assertFalse("", checkResults);
    }

    /*
     * AllowedChars set/get the UnicodeSet of allowed characters.
     */
    public void TestAllowedChars() {
        SpoofChecker sc = new SpoofChecker.Builder().build();
        UnicodeSet set;
        UnicodeSet tmpSet;
        boolean checkResults;

        /* By default, we should see no restriction; the UnicodeSet should allow all characters. */
        set = sc.getAllowedChars();
        tmpSet = new UnicodeSet(0, 0x10ffff);
        assertEquals("", tmpSet, set);

        /* Setting the allowed chars should enable the check. */
        sc = new SpoofChecker.Builder().setChecks(SpoofChecker.ALL_CHECKS & ~SpoofChecker.CHAR_LIMIT).build();

        /* Remove a character that is in our good Latin test identifier from the allowed chars set. */
        tmpSet.remove(goodLatin.charAt(1));
        sc = new SpoofChecker.Builder().setAllowedChars(tmpSet).build();

        /* Latin Identifier should now fail; other non-latin test cases should still be OK */
        SpoofChecker.CheckResult result = new SpoofChecker.CheckResult();
        checkResults = sc.failsChecks(goodLatin, result);
        assertTrue("", checkResults);
        assertEquals("", SpoofChecker.CHAR_LIMIT, result.checks);

        checkResults = sc.failsChecks(goodGreek, result);
        assertTrue("", checkResults);
        assertEquals("", SpoofChecker.WHOLE_SCRIPT_CONFUSABLE, result.checks);
    }

    public void TestCheck() {
        SpoofChecker sc = new SpoofChecker.Builder().build();
        SpoofChecker.CheckResult result = new SpoofChecker.CheckResult();
        boolean checkResults;

        result.position = 666;
        checkResults = sc.failsChecks(goodLatin, result);
        assertFalse("", checkResults);
        assertEquals("", 666, result.position);

        checkResults = sc.failsChecks(goodCyrl, result);
        assertFalse("", checkResults);

        result.position = 666;
        checkResults = sc.failsChecks(scMixed, result);
        assertTrue("", checkResults);
        assertEquals("", SpoofChecker.MIXED_SCRIPT_CONFUSABLE | SpoofChecker.SINGLE_SCRIPT, result.checks);
        assertEquals("", 2, result.position);
        
        result.position = 666;
        checkResults = sc.failsChecks(han_Hiragana, result);
        assertFalse("", checkResults);
        assertEquals("", 666, result.position);
        assertEquals("", 0, result.checks);
    }

    public void TestAreConfusable1() {
        SpoofChecker sc = new SpoofChecker.Builder().build();
        int checkResults;
        checkResults = sc.areConfusable(scLatin, scMixed);
        assertEquals("", SpoofChecker.MIXED_SCRIPT_CONFUSABLE, checkResults);

        checkResults = sc.areConfusable(goodGreek, scLatin);
        assertEquals("", 0, checkResults);

        checkResults = sc.areConfusable(lll_Latin_a, lll_Latin_b);
        assertEquals("", SpoofChecker.SINGLE_SCRIPT_CONFUSABLE, checkResults);
    }

    public void TestGetSkeleton() {
        SpoofChecker sc = new SpoofChecker.Builder().build();
        String dest;
        dest = sc.getSkeleton(SpoofChecker.ANY_CASE, lll_Latin_a);
        assertEquals("", lll_Skel, dest);
    }

    /**
     * IntlTestSpoof is the top level test class for the Unicode Spoof detection tests
     */

    // Test the USpoofDetector API functions that require C++
    // The pure C part of the API, which is most of it, is tested in cintltst
    /**
     * IntlTestSpoof tests for USpoofDetector
     */
    public void TestSpoofAPI() {
        SpoofChecker sc = new SpoofChecker.Builder().build();
        String s = "xyz";  // Many latin ranges are whole-script confusable with other scripts.
                           // If this test starts failing, consult confusablesWholeScript.txt
        SpoofChecker.CheckResult result = new SpoofChecker.CheckResult();
        result.position = 666;
        boolean checkResults = sc.failsChecks(s, result);
        assertFalse("", checkResults);
        assertEquals("", 666, result.position);   // not changed

        sc = new SpoofChecker.Builder().build();
        String s1 = "cxs";
        String s2 = Utility.unescape("\\u0441\\u0445\\u0455"); // Cyrillic "cxs"
        int checkResult = sc.areConfusable(s1, s2);
        assertEquals("", SpoofChecker.MIXED_SCRIPT_CONFUSABLE | SpoofChecker.WHOLE_SCRIPT_CONFUSABLE, checkResult);

        sc = new SpoofChecker.Builder().build();
        s = "I1l0O";
        String dest = sc.getSkeleton(SpoofChecker.ANY_CASE, s);
        assertEquals("", dest, "lllOO");
    }

    public void TestSkeleton() {
        SpoofChecker sc = new SpoofChecker.Builder().build();
        checkSkeleton(sc, "TestSkeleton");
    }
    
    // testSkeleton. Spot check a number of confusable skeleton substitutions from the
    // Unicode data file confusables.txt
    // Test cases chosen for substitutions of various lengths, and
    // membership in different mapping tables.
    public void checkSkeleton(SpoofChecker sc, String testName) {
        int ML = 0;
        int SL = SpoofChecker.SINGLE_SCRIPT_CONFUSABLE;
        int MA = SpoofChecker.ANY_CASE;
        int SA = SpoofChecker.SINGLE_SCRIPT_CONFUSABLE | SpoofChecker.ANY_CASE;

        // A long "identifier" that will overflow implementation stack buffers, forcing heap allocations.
        //    (in the C implementation)
        checkSkeleton(
                sc,
                SL,
                " A 1ong \\u02b9identifier' that will overflow implementation stack buffers, forcing heap allocations."
                        + " A 1ong 'identifier' that will overflow implementation stack buffers, forcing heap allocations."
                        + " A 1ong 'identifier' that will overflow implementation stack buffers, forcing heap allocations."
                        + " A 1ong 'identifier' that will overflow implementation stack buffers, forcing heap allocations.",
                " A long 'identifier' that vvill overflovv irnplernentation stack buffers, forcing heap allocations."
                        + " A long 'identifier' that vvill overflovv irnplernentation stack buffers, forcing heap allocations."
                        + " A long 'identifier' that vvill overflovv irnplernentation stack buffers, forcing heap allocations."
                        + " A long 'identifier' that vvill overflovv irnplernentation stack buffers, forcing heap allocations.",
                testName);

        checkSkeleton(sc, SL, "nochange", "nochange", testName);
        checkSkeleton(sc, MA, "love", "love", testName);
        checkSkeleton(sc, MA, "1ove", "love", testName);   // Digit 1 to letter l
        checkSkeleton(sc, ML, "OOPS", "OOPS", testName);
        checkSkeleton(sc, ML, "00PS", "00PS", testName);   // Digit 0 unchanged in lower case mode.
        checkSkeleton(sc, MA, "OOPS", "OOPS", testName);
        checkSkeleton(sc, MA, "00PS", "OOPS", testName);   // Digit 0 to letter O in any case mode only
        checkSkeleton(sc, SL, "\\u059c", "\\u0301", testName);
        checkSkeleton(sc, SL, "\\u2A74", "\\u003A\\u003A\\u003D", testName);
        checkSkeleton(sc, SL, "\\u247E", "\\u0028\\u006c\\u006c\\u0029", testName);  // "(ll)"
        checkSkeleton(sc, SL, "\\uFDFB", "\\u062C\\u0644\\u0020\\u062C\\u0644\\u0627\\u0644\\u0647", testName);

        // This mapping exists in the ML and MA tables, does not exist in SL, SA
        // 0C83 ; 0983 ; ML #  KANNADA SIGN VISARGA to 
        checkSkeleton(sc, SL, "\\u0C83", "\\u0C83", testName);
        checkSkeleton(sc, SA, "\\u0C83", "\\u0C83", testName);
        checkSkeleton(sc, ML, "\\u0C83", "\\u0983", testName);
        checkSkeleton(sc, MA, "\\u0C83", "\\u0983", testName);

        // 0391 ; 0041 ; MA # GREEK CAPITAL LETTER ALPHA to LATIN CAPITAL LETTER A
        // This mapping exists only in the MA table.
        checkSkeleton(sc, MA, "\\u0391", "A", testName);
        checkSkeleton(sc, SA, "\\u0391", "\\u0391", testName);
        checkSkeleton(sc, ML, "\\u0391", "\\u0391", testName);
        checkSkeleton(sc, SL, "\\u0391", "\\u0391", testName);

        // 13CF ; 0062 ; MA # CHEROKEE LETTER SI to LATIN SMALL LETTER B
        // This mapping exists in the ML and MA tables
        checkSkeleton(sc, ML, "\\u13CF", "b", testName);
        checkSkeleton(sc, MA, "\\u13CF", "b", testName);
        checkSkeleton(sc, SL, "\\u13CF", "\\u13CF", testName);
        checkSkeleton(sc, SA, "\\u13CF", "\\u13CF", testName);

        // 0022 ; 0027 0027 ; 
        // all tables
        checkSkeleton(sc, SL, "\"", "\\u0027\\u0027", testName);
        checkSkeleton(sc, SA, "\"", "\\u0027\\u0027", testName);
        checkSkeleton(sc, ML, "\"", "\\u0027\\u0027", testName);
        checkSkeleton(sc, MA, "\"", "\\u0027\\u0027", testName);
    }

    // Internal function to run a single skeleton test case.
    //
    // Run a single confusable skeleton transformation test case.
    //
    void checkSkeleton(SpoofChecker sc, int type, String input, String expected, String testName) {
        String uInput = Utility.unescape(input);
        String uExpected = Utility.unescape(expected);
        String actual;
        actual = sc.getSkeleton(type, uInput);
        assertEquals(testName + "Expected (escaped): " + expected, uExpected, actual);
    }

    public void TestAreConfusable() {
        SpoofChecker sc = new SpoofChecker.Builder().build();
        String s1 = "A long string that will overflow stack buffers.  A long string that will overflow stack buffers. "
                + "A long string that will overflow stack buffers.  A long string that will overflow stack buffers. ";
        String s2 = "A long string that wi11 overflow stack buffers.  A long string that will overflow stack buffers. "
                + "A long string that wi11 overflow stack buffers.  A long string that will overflow stack buffers. ";
        assertEquals("", SpoofChecker.SINGLE_SCRIPT_CONFUSABLE, sc.areConfusable(s1, s2));
    }

    public void TestInvisible() {
        SpoofChecker sc = new SpoofChecker.Builder().build();
        String s = Utility.unescape("abcd\\u0301ef");
        SpoofChecker.CheckResult result = new SpoofChecker.CheckResult();
        result.position = -42;
        assertFalse("", sc.failsChecks(s, result));
        assertEquals("", 0, result.checks);
        assertEquals("", result.position, -42); // unchanged

        String s2 = Utility.unescape("abcd\\u0301\\u0302\\u0301ef");
        assertTrue("", sc.failsChecks(s2, result));
        assertEquals("", SpoofChecker.INVISIBLE, result.checks);
        assertEquals("", 7, result.position);

        // Two acute accents, one from the composed a with acute accent, \u00e1,
        // and one separate.
        result.position = -42;
        String s3 = Utility.unescape("abcd\\u00e1\\u0301xyz");
        assertTrue("", sc.failsChecks(s3, result));
        assertEquals("", SpoofChecker.INVISIBLE, result.checks);
        assertEquals("", 7, result.position);
    }

    private String parseHex(String in) {
        StringBuilder sb = new StringBuilder();
        for (String oneCharAsHexString : in.split("\\s+")) {
            if (oneCharAsHexString.length() > 0) {
                sb.appendCodePoint(Integer.parseInt(oneCharAsHexString, 16));
            }
        }
        return sb.toString();
    }

    private String escapeString(String in) {
        StringBuilder out = new StringBuilder();
        for (int i = 0; i < in.length(); i++) {
            int c = in.codePointAt(i);
            if (c <= 0x7f) {
                out.append((char) c);
            } else if (c <= 0xffff) {
                out.append(String.format("\\u%04x", c));
            } else {
                out.append(String.format("\\U%06x", c));
                i++;
            }
        }
        return out.toString();
    }

    // Verify that each item from the Unicode confusables.txt file
    // transforms into the expected skeleton.
    public void testConfData() {
        if (TestUtil.getJavaVendor() == JavaVendor.IBM && TestUtil.getJavaVersion() == 5) {
            // Note: IBM Java 5 has a bug reading a large UTF-8 text contents
            logln("Skip this test case because of the IBM Java 5 bug");
            return;
        }
        try {
            // Read in the confusables.txt file. (Distributed by Unicode.org)
            String fileName = "unicode/confusables.txt";
            BufferedReader confusablesRdr = TestUtil.getDataReader(fileName, "UTF-8");

            // Create a default spoof checker to use in this test.
            SpoofChecker sc = new SpoofChecker.Builder().build();

            // Parse lines from the confusables.txt file. Example Line:
            // FF44 ; 0064 ; SL # ( d -> d ) FULLWIDTH ....
            // Lines have three fields. The hex fields can contain more than one character,
            // and each character may be more than 4 digits (for supplemntals)
            // This regular expression matches lines and splits the fields into capture groups.
            // Capture group 1: map from chars
            // 2: map to chars
            // 3: table type, SL, ML, SA or MA
            // 4: Comment Lines Only
            // 5: Error Lines Only
            Matcher parseLine = Pattern.compile(
                    "\\ufeff?" + "(?:([0-9A-F\\s]+);([0-9A-F\\s]+);\\s*(SL|ML|SA|MA)\\s*(?:#.*?)?$)"
                            + "|\\ufeff?(\\s*(?:#.*)?)"). // Comment line
                    matcher("");
            Normalizer2 normalizer = Normalizer2.getNFDInstance();
            int lineNum = 0;
            String inputLine;
            while ((inputLine = confusablesRdr.readLine()) != null) {
                lineNum++;
                parseLine.reset(inputLine);
                if (!parseLine.matches()) {
                    errln("Syntax error in confusable data file at line " + lineNum);
                    errln(inputLine);
                    break;
                }
                if (parseLine.group(4) != null) {
                    continue; // comment line
                }
                String from = parseHex(parseLine.group(1));

                if (!normalizer.isNormalized(from)) {
                    // The source character was not NFD.
                    // Skip this case; the first step in obtaining a skeleton is to NFD the input,
                    // so the mapping in this line of confusables.txt will never be applied.
                    continue;
                }

                String rawExpected = parseHex(parseLine.group(2));
                String expected = normalizer.normalize(rawExpected);

                int skeletonType = 0;
                String tableType = parseLine.group(3);
                if (tableType.equals("SL")) {
                    skeletonType = SpoofChecker.SINGLE_SCRIPT_CONFUSABLE;
                } else if (tableType.indexOf("SA") >= 0) {
                    skeletonType = SpoofChecker.SINGLE_SCRIPT_CONFUSABLE | SpoofChecker.ANY_CASE;
                } else if (tableType.indexOf("ML") >= 0) {
                    skeletonType = 0;
                } else if (tableType.indexOf("MA") >= 0) {
                    skeletonType = SpoofChecker.ANY_CASE;
                }

                String actual;
                actual = sc.getSkeleton(skeletonType, from);

                if (!actual.equals(expected)) {
                    errln("confusables.txt: " + lineNum + ": " + parseLine.group(0));
                    errln("Actual: " + escapeString(actual));
                }
            }
            confusablesRdr.close();
        } catch (IOException e) {
            errln(e.toString());
        }
    }
}